1eb8dc403SDave Cobbley""" 2eb8dc403SDave CobbleyBitBake 'Fetch' git submodules implementation 3eb8dc403SDave Cobbley 4eb8dc403SDave CobbleyInherits from and extends the Git fetcher to retrieve submodules of a git repository 5eb8dc403SDave Cobbleyafter cloning. 6eb8dc403SDave Cobbley 7eb8dc403SDave CobbleySRC_URI = "gitsm://<see Git fetcher for syntax>" 8eb8dc403SDave Cobbley 9eb8dc403SDave CobbleySee the Git fetcher, git://, for usage documentation. 10eb8dc403SDave Cobbley 11eb8dc403SDave CobbleyNOTE: Switching a SRC_URI from "git://" to "gitsm://" requires a clean of your recipe. 12eb8dc403SDave Cobbley 13eb8dc403SDave Cobbley""" 14eb8dc403SDave Cobbley 15eb8dc403SDave Cobbley# Copyright (C) 2013 Richard Purdie 16eb8dc403SDave Cobbley# 17c342db35SBrad Bishop# SPDX-License-Identifier: GPL-2.0-only 18eb8dc403SDave Cobbley# 19eb8dc403SDave Cobbley 20eb8dc403SDave Cobbleyimport os 21eb8dc403SDave Cobbleyimport bb 221a4b7ee2SBrad Bishopimport copy 2382c905dcSAndrew Geisslerimport shutil 2482c905dcSAndrew Geisslerimport tempfile 25eb8dc403SDave Cobbleyfrom bb.fetch2.git import Git 26eb8dc403SDave Cobbleyfrom bb.fetch2 import runfetchcmd 27eb8dc403SDave Cobbleyfrom bb.fetch2 import logger 281a4b7ee2SBrad Bishopfrom bb.fetch2 import Fetch 29eb8dc403SDave Cobbley 30eb8dc403SDave Cobbleyclass GitSM(Git): 31eb8dc403SDave Cobbley def supports(self, ud, d): 32eb8dc403SDave Cobbley """ 33eb8dc403SDave Cobbley Check to see if a given url can be fetched with git. 34eb8dc403SDave Cobbley """ 35eb8dc403SDave Cobbley return ud.type in ['gitsm'] 36eb8dc403SDave Cobbley 37f8caae30SBrad Bishop def process_submodules(self, ud, workdir, function, d): 38f8caae30SBrad Bishop """ 39f8caae30SBrad Bishop Iterate over all of the submodules in this repository and execute 40f8caae30SBrad Bishop the 'function' for each of them. 41f8caae30SBrad Bishop """ 42f8caae30SBrad Bishop 43f8caae30SBrad Bishop submodules = [] 44f8caae30SBrad Bishop paths = {} 45f8caae30SBrad Bishop revision = {} 46f8caae30SBrad Bishop uris = {} 47f8caae30SBrad Bishop subrevision = {} 48f8caae30SBrad Bishop 491a4b7ee2SBrad Bishop def parse_gitmodules(gitmodules): 501a4b7ee2SBrad Bishop modules = {} 511a4b7ee2SBrad Bishop module = "" 521a4b7ee2SBrad Bishop for line in gitmodules.splitlines(): 53eb8dc403SDave Cobbley if line.startswith('[submodule'): 541a4b7ee2SBrad Bishop module = line.split('"')[1] 551a4b7ee2SBrad Bishop modules[module] = {} 561a4b7ee2SBrad Bishop elif module and line.strip().startswith('path'): 571a4b7ee2SBrad Bishop path = line.split('=')[1].strip() 581a4b7ee2SBrad Bishop modules[module]['path'] = path 591a4b7ee2SBrad Bishop elif module and line.strip().startswith('url'): 601a4b7ee2SBrad Bishop url = line.split('=')[1].strip() 611a4b7ee2SBrad Bishop modules[module]['url'] = url 621a4b7ee2SBrad Bishop return modules 63eb8dc403SDave Cobbley 64f8caae30SBrad Bishop # Collect the defined submodules, and their attributes 651a4b7ee2SBrad Bishop for name in ud.names: 661a4b7ee2SBrad Bishop try: 67f8caae30SBrad Bishop gitmodules = runfetchcmd("%s show %s:.gitmodules" % (ud.basecmd, ud.revisions[name]), d, quiet=True, workdir=workdir) 681a4b7ee2SBrad Bishop except: 691a4b7ee2SBrad Bishop # No submodules to update 701a4b7ee2SBrad Bishop continue 711a4b7ee2SBrad Bishop 72f8caae30SBrad Bishop for m, md in parse_gitmodules(gitmodules).items(): 73f8caae30SBrad Bishop try: 74f8caae30SBrad Bishop module_hash = runfetchcmd("%s ls-tree -z -d %s %s" % (ud.basecmd, ud.revisions[name], md['path']), d, quiet=True, workdir=workdir) 75f8caae30SBrad Bishop except: 76f8caae30SBrad Bishop # If the command fails, we don't have a valid file to check. If it doesn't 77f8caae30SBrad Bishop # fail -- it still might be a failure, see next check... 78f8caae30SBrad Bishop module_hash = "" 79f8caae30SBrad Bishop 80f8caae30SBrad Bishop if not module_hash: 81d1e89497SAndrew Geissler logger.debug("submodule %s is defined, but is not initialized in the repository. Skipping", m) 82f8caae30SBrad Bishop continue 83f8caae30SBrad Bishop 841a4b7ee2SBrad Bishop submodules.append(m) 851a4b7ee2SBrad Bishop paths[m] = md['path'] 86f8caae30SBrad Bishop revision[m] = ud.revisions[name] 871a4b7ee2SBrad Bishop uris[m] = md['url'] 88f8caae30SBrad Bishop subrevision[m] = module_hash.split()[2] 89f8caae30SBrad Bishop 90f8caae30SBrad Bishop # Convert relative to absolute uri based on parent uri 91615f2f11SAndrew Geissler if uris[m].startswith('..') or uris[m].startswith('./'): 921a4b7ee2SBrad Bishop newud = copy.copy(ud) 936aa7eec5SAndrew Geissler newud.path = os.path.normpath(os.path.join(newud.path, uris[m])) 941a4b7ee2SBrad Bishop uris[m] = Git._get_repo_url(self, newud) 951a4b7ee2SBrad Bishop 961a4b7ee2SBrad Bishop for module in submodules: 97f8caae30SBrad Bishop # Translate the module url into a SRC_URI 981a4b7ee2SBrad Bishop 99f8caae30SBrad Bishop if "://" in uris[module]: 100f8caae30SBrad Bishop # Properly formated URL already 1011a4b7ee2SBrad Bishop proto = uris[module].split(':', 1)[0] 1021a4b7ee2SBrad Bishop url = uris[module].replace('%s:' % proto, 'gitsm:', 1) 103f8caae30SBrad Bishop else: 104f8caae30SBrad Bishop if ":" in uris[module]: 105f8caae30SBrad Bishop # Most likely an SSH style reference 106f8caae30SBrad Bishop proto = "ssh" 107f8caae30SBrad Bishop if ":/" in uris[module]: 108f8caae30SBrad Bishop # Absolute reference, easy to convert.. 109f8caae30SBrad Bishop url = "gitsm://" + uris[module].replace(':/', '/', 1) 110f8caae30SBrad Bishop else: 111f8caae30SBrad Bishop # Relative reference, no way to know if this is right! 112f8caae30SBrad Bishop logger.warning("Submodule included by %s refers to relative ssh reference %s. References may fail if not absolute." % (ud.url, uris[module])) 113f8caae30SBrad Bishop url = "gitsm://" + uris[module].replace(':', '/', 1) 114f8caae30SBrad Bishop else: 115f8caae30SBrad Bishop # This has to be a file reference 116f8caae30SBrad Bishop proto = "file" 117f8caae30SBrad Bishop url = "gitsm://" + uris[module] 1187784c429SPatrick Williams if url.endswith("{}{}".format(ud.host, ud.path)): 11992b42cb3SPatrick Williams raise bb.fetch2.FetchError("Submodule refers to the parent repository. This will cause deadlock situation in current version of Bitbake." \ 12092b42cb3SPatrick Williams "Consider using git fetcher instead.") 121f8caae30SBrad Bishop 1221a4b7ee2SBrad Bishop url += ';protocol=%s' % proto 1231a4b7ee2SBrad Bishop url += ";name=%s" % module 124393846f1SBrad Bishop url += ";subpath=%s" % module 1257784c429SPatrick Williams url += ";nobranch=1" 1265082cc7fSAndrew Geissler url += ";lfs=%s" % self._need_lfs(ud) 1272a25492cSPatrick Williams # Note that adding "user=" here to give credentials to the 1282a25492cSPatrick Williams # submodule is not supported. Since using SRC_URI to give git:// 1292a25492cSPatrick Williams # URL a password is not supported, one have to use one of the 1302a25492cSPatrick Williams # recommended way (eg. ~/.netrc or SSH config) which does specify 1312a25492cSPatrick Williams # the user (See comment in git.py). 1322a25492cSPatrick Williams # So, we will not take patches adding "user=" support here. 1331a4b7ee2SBrad Bishop 1341a4b7ee2SBrad Bishop ld = d.createCopy() 1351a4b7ee2SBrad Bishop # Not necessary to set SRC_URI, since we're passing the URI to 1361a4b7ee2SBrad Bishop # Fetch. 1371a4b7ee2SBrad Bishop #ld.setVar('SRC_URI', url) 138f8caae30SBrad Bishop ld.setVar('SRCREV_%s' % module, subrevision[module]) 1391a4b7ee2SBrad Bishop 1401a4b7ee2SBrad Bishop # Workaround for issues with SRCPV/SRCREV_FORMAT errors 1411a4b7ee2SBrad Bishop # error refer to 'multiple' repositories. Only the repository 1421a4b7ee2SBrad Bishop # in the original SRC_URI actually matters... 1431a4b7ee2SBrad Bishop ld.setVar('SRCPV', d.getVar('SRCPV')) 1441a4b7ee2SBrad Bishop ld.setVar('SRCREV_FORMAT', module) 1451a4b7ee2SBrad Bishop 14682c905dcSAndrew Geissler function(ud, url, module, paths[module], workdir, ld) 1471a4b7ee2SBrad Bishop 148f8caae30SBrad Bishop return submodules != [] 149eb8dc403SDave Cobbley 15019323693SBrad Bishop def need_update(self, ud, d): 15119323693SBrad Bishop if Git.need_update(self, ud, d): 15219323693SBrad Bishop return True 15319323693SBrad Bishop 154d25ed324SAndrew Geissler need_update_list = [] 155d25ed324SAndrew Geissler def need_update_submodule(ud, url, module, modpath, workdir, d): 156d25ed324SAndrew Geissler url += ";bareclone=1;nobranch=1" 157d25ed324SAndrew Geissler 158d25ed324SAndrew Geissler try: 159d25ed324SAndrew Geissler newfetch = Fetch([url], d, cache=False) 160d25ed324SAndrew Geissler new_ud = newfetch.ud[url] 161d25ed324SAndrew Geissler if new_ud.method.need_update(new_ud, d): 162d25ed324SAndrew Geissler need_update_list.append(modpath) 163d25ed324SAndrew Geissler except Exception as e: 164d25ed324SAndrew Geissler logger.error('gitsm: submodule update check failed: %s %s' % (type(e).__name__, str(e))) 165d25ed324SAndrew Geissler need_update_result = True 166d25ed324SAndrew Geissler 167d25ed324SAndrew Geissler # If we're using a shallow mirror tarball it needs to be unpacked 168d25ed324SAndrew Geissler # temporarily so that we can examine the .gitmodules file 169d25ed324SAndrew Geissler if ud.shallow and os.path.exists(ud.fullshallow) and not os.path.exists(ud.clonedir): 170d25ed324SAndrew Geissler tmpdir = tempfile.mkdtemp(dir=d.getVar("DL_DIR")) 171d25ed324SAndrew Geissler runfetchcmd("tar -xzf %s" % ud.fullshallow, d, workdir=tmpdir) 172d25ed324SAndrew Geissler self.process_submodules(ud, tmpdir, need_update_submodule, d) 173d25ed324SAndrew Geissler shutil.rmtree(tmpdir) 174d25ed324SAndrew Geissler else: 175d25ed324SAndrew Geissler self.process_submodules(ud, ud.clonedir, need_update_submodule, d) 176d25ed324SAndrew Geissler 177595f6308SAndrew Geissler if need_update_list: 178d1e89497SAndrew Geissler logger.debug('gitsm: Submodules requiring update: %s' % (' '.join(need_update_list))) 17919323693SBrad Bishop return True 18019323693SBrad Bishop 18119323693SBrad Bishop return False 18219323693SBrad Bishop 183eb8dc403SDave Cobbley def download(self, ud, d): 18482c905dcSAndrew Geissler def download_submodule(ud, url, module, modpath, workdir, d): 185f8caae30SBrad Bishop url += ";bareclone=1;nobranch=1" 186eb8dc403SDave Cobbley 187f8caae30SBrad Bishop # Is the following still needed? 188f8caae30SBrad Bishop #url += ";nocheckout=1" 189eb8dc403SDave Cobbley 1901a4b7ee2SBrad Bishop try: 191f8caae30SBrad Bishop newfetch = Fetch([url], d, cache=False) 192f8caae30SBrad Bishop newfetch.download() 193f8caae30SBrad Bishop except Exception as e: 194f8caae30SBrad Bishop logger.error('gitsm: submodule download failed: %s %s' % (type(e).__name__, str(e))) 195f8caae30SBrad Bishop raise 1961a4b7ee2SBrad Bishop 197f8caae30SBrad Bishop Git.download(self, ud, d) 19882c905dcSAndrew Geissler 19982c905dcSAndrew Geissler # If we're using a shallow mirror tarball it needs to be unpacked 20082c905dcSAndrew Geissler # temporarily so that we can examine the .gitmodules file 20182c905dcSAndrew Geissler if ud.shallow and os.path.exists(ud.fullshallow) and self.need_update(ud, d): 20282c905dcSAndrew Geissler tmpdir = tempfile.mkdtemp(dir=d.getVar("DL_DIR")) 20382c905dcSAndrew Geissler runfetchcmd("tar -xzf %s" % ud.fullshallow, d, workdir=tmpdir) 20482c905dcSAndrew Geissler self.process_submodules(ud, tmpdir, download_submodule, d) 20582c905dcSAndrew Geissler shutil.rmtree(tmpdir) 20682c905dcSAndrew Geissler else: 207f8caae30SBrad Bishop self.process_submodules(ud, ud.clonedir, download_submodule, d) 208eb8dc403SDave Cobbley 209eb8dc403SDave Cobbley def unpack(self, ud, destdir, d): 21082c905dcSAndrew Geissler def unpack_submodules(ud, url, module, modpath, workdir, d): 211f8caae30SBrad Bishop url += ";bareclone=1;nobranch=1" 212eb8dc403SDave Cobbley 213f8caae30SBrad Bishop # Figure out where we clone over the bare submodules... 2141a4b7ee2SBrad Bishop if ud.bareclone: 2151a4b7ee2SBrad Bishop repo_conf = ud.destdir 2161a4b7ee2SBrad Bishop else: 2171a4b7ee2SBrad Bishop repo_conf = os.path.join(ud.destdir, '.git') 2181a4b7ee2SBrad Bishop 2191a4b7ee2SBrad Bishop try: 220f8caae30SBrad Bishop newfetch = Fetch([url], d, cache=False) 221*ac13d5f3SPatrick Williams # modpath is needed by unpack tracer to calculate submodule 222*ac13d5f3SPatrick Williams # checkout dir 223*ac13d5f3SPatrick Williams new_ud = newfetch.ud[url] 224*ac13d5f3SPatrick Williams new_ud.modpath = modpath 225393846f1SBrad Bishop newfetch.unpack(root=os.path.dirname(os.path.join(repo_conf, 'modules', module))) 226f8caae30SBrad Bishop except Exception as e: 227f8caae30SBrad Bishop logger.error('gitsm: submodule unpack failed: %s %s' % (type(e).__name__, str(e))) 228f8caae30SBrad Bishop raise 2291a4b7ee2SBrad Bishop 230f8caae30SBrad Bishop local_path = newfetch.localpath(url) 2311a4b7ee2SBrad Bishop 232f8caae30SBrad Bishop # Correct the submodule references to the local download version... 233f8caae30SBrad Bishop runfetchcmd("%(basecmd)s config submodule.%(module)s.url %(url)s" % {'basecmd': ud.basecmd, 'module': module, 'url' : local_path}, d, workdir=ud.destdir) 2341a4b7ee2SBrad Bishop 235f8caae30SBrad Bishop if ud.shallow: 236f8caae30SBrad Bishop runfetchcmd("%(basecmd)s config submodule.%(module)s.shallow true" % {'basecmd': ud.basecmd, 'module': module}, d, workdir=ud.destdir) 2371a4b7ee2SBrad Bishop 2381a4b7ee2SBrad Bishop # Ensure the submodule repository is NOT set to bare, since we're checking it out... 239f8caae30SBrad Bishop try: 240393846f1SBrad Bishop runfetchcmd("%s config core.bare false" % (ud.basecmd), d, quiet=True, workdir=os.path.join(repo_conf, 'modules', module)) 241f8caae30SBrad Bishop except: 242393846f1SBrad Bishop logger.error("Unable to set git config core.bare to false for %s" % os.path.join(repo_conf, 'modules', module)) 243f8caae30SBrad Bishop raise 2441a4b7ee2SBrad Bishop 245f8caae30SBrad Bishop Git.unpack(self, ud, destdir, d) 246f8caae30SBrad Bishop 247f8caae30SBrad Bishop ret = self.process_submodules(ud, ud.destdir, unpack_submodules, d) 248f8caae30SBrad Bishop 249f8caae30SBrad Bishop if not ud.bareclone and ret: 2505082cc7fSAndrew Geissler # All submodules should already be downloaded and configured in the tree. This simply 2515082cc7fSAndrew Geissler # sets up the configuration and checks out the files. The main project config should 2525082cc7fSAndrew Geissler # remain unmodified, and no download from the internet should occur. As such, lfs smudge 2535082cc7fSAndrew Geissler # should also be skipped as these files were already smudged in the fetch stage if lfs 2545082cc7fSAndrew Geissler # was enabled. 2555082cc7fSAndrew Geissler runfetchcmd("GIT_LFS_SKIP_SMUDGE=1 %s submodule update --recursive --no-fetch" % (ud.basecmd), d, quiet=True, workdir=ud.destdir) 2564ed12e16SAndrew Geissler 2574ed12e16SAndrew Geissler def implicit_urldata(self, ud, d): 2584ed12e16SAndrew Geissler import shutil, subprocess, tempfile 2594ed12e16SAndrew Geissler 2604ed12e16SAndrew Geissler urldata = [] 2614ed12e16SAndrew Geissler def add_submodule(ud, url, module, modpath, workdir, d): 2624ed12e16SAndrew Geissler url += ";bareclone=1;nobranch=1" 2634ed12e16SAndrew Geissler newfetch = Fetch([url], d, cache=False) 2644ed12e16SAndrew Geissler urldata.extend(newfetch.expanded_urldata()) 2654ed12e16SAndrew Geissler 2664ed12e16SAndrew Geissler # If we're using a shallow mirror tarball it needs to be unpacked 2674ed12e16SAndrew Geissler # temporarily so that we can examine the .gitmodules file 2684ed12e16SAndrew Geissler if ud.shallow and os.path.exists(ud.fullshallow) and ud.method.need_update(ud, d): 2694ed12e16SAndrew Geissler tmpdir = tempfile.mkdtemp(dir=d.getVar("DL_DIR")) 2704ed12e16SAndrew Geissler subprocess.check_call("tar -xzf %s" % ud.fullshallow, cwd=tmpdir, shell=True) 2714ed12e16SAndrew Geissler self.process_submodules(ud, tmpdir, add_submodule, d) 2724ed12e16SAndrew Geissler shutil.rmtree(tmpdir) 2734ed12e16SAndrew Geissler else: 2744ed12e16SAndrew Geissler self.process_submodules(ud, ud.clonedir, add_submodule, d) 2754ed12e16SAndrew Geissler 2764ed12e16SAndrew Geissler return urldata 277