1eb8dc403SDave Cobbley""" 2eb8dc403SDave CobbleyBitBake 'Fetch' git submodules implementation 3eb8dc403SDave Cobbley 4eb8dc403SDave CobbleyInherits from and extends the Git fetcher to retrieve submodules of a git repository 5eb8dc403SDave Cobbleyafter cloning. 6eb8dc403SDave Cobbley 7eb8dc403SDave CobbleySRC_URI = "gitsm://<see Git fetcher for syntax>" 8eb8dc403SDave Cobbley 9eb8dc403SDave CobbleySee the Git fetcher, git://, for usage documentation. 10eb8dc403SDave Cobbley 11eb8dc403SDave CobbleyNOTE: Switching a SRC_URI from "git://" to "gitsm://" requires a clean of your recipe. 12eb8dc403SDave Cobbley 13eb8dc403SDave Cobbley""" 14eb8dc403SDave Cobbley 15eb8dc403SDave Cobbley# Copyright (C) 2013 Richard Purdie 16eb8dc403SDave Cobbley# 17c342db35SBrad Bishop# SPDX-License-Identifier: GPL-2.0-only 18eb8dc403SDave Cobbley# 19eb8dc403SDave Cobbley 20eb8dc403SDave Cobbleyimport os 21eb8dc403SDave Cobbleyimport bb 221a4b7ee2SBrad Bishopimport copy 23*82c905dcSAndrew Geisslerimport shutil 24*82c905dcSAndrew Geisslerimport tempfile 25eb8dc403SDave Cobbleyfrom bb.fetch2.git import Git 26eb8dc403SDave Cobbleyfrom bb.fetch2 import runfetchcmd 27eb8dc403SDave Cobbleyfrom bb.fetch2 import logger 281a4b7ee2SBrad Bishopfrom bb.fetch2 import Fetch 29eb8dc403SDave Cobbley 30eb8dc403SDave Cobbleyclass GitSM(Git): 31eb8dc403SDave Cobbley def supports(self, ud, d): 32eb8dc403SDave Cobbley """ 33eb8dc403SDave Cobbley Check to see if a given url can be fetched with git. 34eb8dc403SDave Cobbley """ 35eb8dc403SDave Cobbley return ud.type in ['gitsm'] 36eb8dc403SDave Cobbley 37f8caae30SBrad Bishop def process_submodules(self, ud, workdir, function, d): 38f8caae30SBrad Bishop """ 39f8caae30SBrad Bishop Iterate over all of the submodules in this repository and execute 40f8caae30SBrad Bishop the 'function' for each of them. 41f8caae30SBrad Bishop """ 42f8caae30SBrad Bishop 43f8caae30SBrad Bishop submodules = [] 44f8caae30SBrad Bishop paths = {} 45f8caae30SBrad Bishop revision = {} 46f8caae30SBrad Bishop uris = {} 47f8caae30SBrad Bishop subrevision = {} 48f8caae30SBrad Bishop 491a4b7ee2SBrad Bishop def parse_gitmodules(gitmodules): 501a4b7ee2SBrad Bishop modules = {} 511a4b7ee2SBrad Bishop module = "" 521a4b7ee2SBrad Bishop for line in gitmodules.splitlines(): 53eb8dc403SDave Cobbley if line.startswith('[submodule'): 541a4b7ee2SBrad Bishop module = line.split('"')[1] 551a4b7ee2SBrad Bishop modules[module] = {} 561a4b7ee2SBrad Bishop elif module and line.strip().startswith('path'): 571a4b7ee2SBrad Bishop path = line.split('=')[1].strip() 581a4b7ee2SBrad Bishop modules[module]['path'] = path 591a4b7ee2SBrad Bishop elif module and line.strip().startswith('url'): 601a4b7ee2SBrad Bishop url = line.split('=')[1].strip() 611a4b7ee2SBrad Bishop modules[module]['url'] = url 621a4b7ee2SBrad Bishop return modules 63eb8dc403SDave Cobbley 64f8caae30SBrad Bishop # Collect the defined submodules, and their attributes 651a4b7ee2SBrad Bishop for name in ud.names: 661a4b7ee2SBrad Bishop try: 67f8caae30SBrad Bishop gitmodules = runfetchcmd("%s show %s:.gitmodules" % (ud.basecmd, ud.revisions[name]), d, quiet=True, workdir=workdir) 681a4b7ee2SBrad Bishop except: 691a4b7ee2SBrad Bishop # No submodules to update 701a4b7ee2SBrad Bishop continue 711a4b7ee2SBrad Bishop 72f8caae30SBrad Bishop for m, md in parse_gitmodules(gitmodules).items(): 73f8caae30SBrad Bishop try: 74f8caae30SBrad Bishop module_hash = runfetchcmd("%s ls-tree -z -d %s %s" % (ud.basecmd, ud.revisions[name], md['path']), d, quiet=True, workdir=workdir) 75f8caae30SBrad Bishop except: 76f8caae30SBrad Bishop # If the command fails, we don't have a valid file to check. If it doesn't 77f8caae30SBrad Bishop # fail -- it still might be a failure, see next check... 78f8caae30SBrad Bishop module_hash = "" 79f8caae30SBrad Bishop 80f8caae30SBrad Bishop if not module_hash: 81f8caae30SBrad Bishop logger.debug(1, "submodule %s is defined, but is not initialized in the repository. Skipping", m) 82f8caae30SBrad Bishop continue 83f8caae30SBrad Bishop 841a4b7ee2SBrad Bishop submodules.append(m) 851a4b7ee2SBrad Bishop paths[m] = md['path'] 86f8caae30SBrad Bishop revision[m] = ud.revisions[name] 871a4b7ee2SBrad Bishop uris[m] = md['url'] 88f8caae30SBrad Bishop subrevision[m] = module_hash.split()[2] 89f8caae30SBrad Bishop 90f8caae30SBrad Bishop # Convert relative to absolute uri based on parent uri 911a4b7ee2SBrad Bishop if uris[m].startswith('..'): 921a4b7ee2SBrad Bishop newud = copy.copy(ud) 93f8caae30SBrad Bishop newud.path = os.path.realpath(os.path.join(newud.path, uris[m])) 941a4b7ee2SBrad Bishop uris[m] = Git._get_repo_url(self, newud) 951a4b7ee2SBrad Bishop 961a4b7ee2SBrad Bishop for module in submodules: 97f8caae30SBrad Bishop # Translate the module url into a SRC_URI 981a4b7ee2SBrad Bishop 99f8caae30SBrad Bishop if "://" in uris[module]: 100f8caae30SBrad Bishop # Properly formated URL already 1011a4b7ee2SBrad Bishop proto = uris[module].split(':', 1)[0] 1021a4b7ee2SBrad Bishop url = uris[module].replace('%s:' % proto, 'gitsm:', 1) 103f8caae30SBrad Bishop else: 104f8caae30SBrad Bishop if ":" in uris[module]: 105f8caae30SBrad Bishop # Most likely an SSH style reference 106f8caae30SBrad Bishop proto = "ssh" 107f8caae30SBrad Bishop if ":/" in uris[module]: 108f8caae30SBrad Bishop # Absolute reference, easy to convert.. 109f8caae30SBrad Bishop url = "gitsm://" + uris[module].replace(':/', '/', 1) 110f8caae30SBrad Bishop else: 111f8caae30SBrad Bishop # Relative reference, no way to know if this is right! 112f8caae30SBrad Bishop logger.warning("Submodule included by %s refers to relative ssh reference %s. References may fail if not absolute." % (ud.url, uris[module])) 113f8caae30SBrad Bishop url = "gitsm://" + uris[module].replace(':', '/', 1) 114f8caae30SBrad Bishop else: 115f8caae30SBrad Bishop # This has to be a file reference 116f8caae30SBrad Bishop proto = "file" 117f8caae30SBrad Bishop url = "gitsm://" + uris[module] 118f8caae30SBrad Bishop 1191a4b7ee2SBrad Bishop url += ';protocol=%s' % proto 1201a4b7ee2SBrad Bishop url += ";name=%s" % module 121393846f1SBrad Bishop url += ";subpath=%s" % module 1221a4b7ee2SBrad Bishop 1231a4b7ee2SBrad Bishop ld = d.createCopy() 1241a4b7ee2SBrad Bishop # Not necessary to set SRC_URI, since we're passing the URI to 1251a4b7ee2SBrad Bishop # Fetch. 1261a4b7ee2SBrad Bishop #ld.setVar('SRC_URI', url) 127f8caae30SBrad Bishop ld.setVar('SRCREV_%s' % module, subrevision[module]) 1281a4b7ee2SBrad Bishop 1291a4b7ee2SBrad Bishop # Workaround for issues with SRCPV/SRCREV_FORMAT errors 1301a4b7ee2SBrad Bishop # error refer to 'multiple' repositories. Only the repository 1311a4b7ee2SBrad Bishop # in the original SRC_URI actually matters... 1321a4b7ee2SBrad Bishop ld.setVar('SRCPV', d.getVar('SRCPV')) 1331a4b7ee2SBrad Bishop ld.setVar('SRCREV_FORMAT', module) 1341a4b7ee2SBrad Bishop 135*82c905dcSAndrew Geissler function(ud, url, module, paths[module], workdir, ld) 1361a4b7ee2SBrad Bishop 137f8caae30SBrad Bishop return submodules != [] 138eb8dc403SDave Cobbley 13919323693SBrad Bishop def need_update(self, ud, d): 14019323693SBrad Bishop if Git.need_update(self, ud, d): 14119323693SBrad Bishop return True 14219323693SBrad Bishop 14319323693SBrad Bishop try: 14419323693SBrad Bishop # Check for the nugget dropped by the download operation 14519323693SBrad Bishop known_srcrevs = runfetchcmd("%s config --get-all bitbake.srcrev" % \ 14619323693SBrad Bishop (ud.basecmd), d, workdir=ud.clonedir) 14719323693SBrad Bishop 14819323693SBrad Bishop if ud.revisions[ud.names[0]] not in known_srcrevs.split(): 14919323693SBrad Bishop return True 15019323693SBrad Bishop except bb.fetch2.FetchError: 15119323693SBrad Bishop # No srcrev nuggets, so this is new and needs to be updated 15219323693SBrad Bishop return True 15319323693SBrad Bishop 15419323693SBrad Bishop return False 15519323693SBrad Bishop 156eb8dc403SDave Cobbley def download(self, ud, d): 157*82c905dcSAndrew Geissler def download_submodule(ud, url, module, modpath, workdir, d): 158f8caae30SBrad Bishop url += ";bareclone=1;nobranch=1" 159eb8dc403SDave Cobbley 160f8caae30SBrad Bishop # Is the following still needed? 161f8caae30SBrad Bishop #url += ";nocheckout=1" 162eb8dc403SDave Cobbley 1631a4b7ee2SBrad Bishop try: 164f8caae30SBrad Bishop newfetch = Fetch([url], d, cache=False) 165f8caae30SBrad Bishop newfetch.download() 16619323693SBrad Bishop # Drop a nugget to add each of the srcrevs we've fetched (used by need_update) 16719323693SBrad Bishop runfetchcmd("%s config --add bitbake.srcrev %s" % \ 168*82c905dcSAndrew Geissler (ud.basecmd, ud.revisions[ud.names[0]]), d, workdir=workdir) 169f8caae30SBrad Bishop except Exception as e: 170f8caae30SBrad Bishop logger.error('gitsm: submodule download failed: %s %s' % (type(e).__name__, str(e))) 171f8caae30SBrad Bishop raise 1721a4b7ee2SBrad Bishop 173f8caae30SBrad Bishop Git.download(self, ud, d) 174*82c905dcSAndrew Geissler 175*82c905dcSAndrew Geissler # If we're using a shallow mirror tarball it needs to be unpacked 176*82c905dcSAndrew Geissler # temporarily so that we can examine the .gitmodules file 177*82c905dcSAndrew Geissler if ud.shallow and os.path.exists(ud.fullshallow) and self.need_update(ud, d): 178*82c905dcSAndrew Geissler tmpdir = tempfile.mkdtemp(dir=d.getVar("DL_DIR")) 179*82c905dcSAndrew Geissler runfetchcmd("tar -xzf %s" % ud.fullshallow, d, workdir=tmpdir) 180*82c905dcSAndrew Geissler self.process_submodules(ud, tmpdir, download_submodule, d) 181*82c905dcSAndrew Geissler shutil.rmtree(tmpdir) 182*82c905dcSAndrew Geissler else: 183f8caae30SBrad Bishop self.process_submodules(ud, ud.clonedir, download_submodule, d) 184eb8dc403SDave Cobbley 185eb8dc403SDave Cobbley def unpack(self, ud, destdir, d): 186*82c905dcSAndrew Geissler def unpack_submodules(ud, url, module, modpath, workdir, d): 187f8caae30SBrad Bishop url += ";bareclone=1;nobranch=1" 188eb8dc403SDave Cobbley 189f8caae30SBrad Bishop # Figure out where we clone over the bare submodules... 1901a4b7ee2SBrad Bishop if ud.bareclone: 1911a4b7ee2SBrad Bishop repo_conf = ud.destdir 1921a4b7ee2SBrad Bishop else: 1931a4b7ee2SBrad Bishop repo_conf = os.path.join(ud.destdir, '.git') 1941a4b7ee2SBrad Bishop 1951a4b7ee2SBrad Bishop try: 196f8caae30SBrad Bishop newfetch = Fetch([url], d, cache=False) 197393846f1SBrad Bishop newfetch.unpack(root=os.path.dirname(os.path.join(repo_conf, 'modules', module))) 198f8caae30SBrad Bishop except Exception as e: 199f8caae30SBrad Bishop logger.error('gitsm: submodule unpack failed: %s %s' % (type(e).__name__, str(e))) 200f8caae30SBrad Bishop raise 2011a4b7ee2SBrad Bishop 202f8caae30SBrad Bishop local_path = newfetch.localpath(url) 2031a4b7ee2SBrad Bishop 204f8caae30SBrad Bishop # Correct the submodule references to the local download version... 205f8caae30SBrad Bishop runfetchcmd("%(basecmd)s config submodule.%(module)s.url %(url)s" % {'basecmd': ud.basecmd, 'module': module, 'url' : local_path}, d, workdir=ud.destdir) 2061a4b7ee2SBrad Bishop 207f8caae30SBrad Bishop if ud.shallow: 208f8caae30SBrad Bishop runfetchcmd("%(basecmd)s config submodule.%(module)s.shallow true" % {'basecmd': ud.basecmd, 'module': module}, d, workdir=ud.destdir) 2091a4b7ee2SBrad Bishop 2101a4b7ee2SBrad Bishop # Ensure the submodule repository is NOT set to bare, since we're checking it out... 211f8caae30SBrad Bishop try: 212393846f1SBrad Bishop runfetchcmd("%s config core.bare false" % (ud.basecmd), d, quiet=True, workdir=os.path.join(repo_conf, 'modules', module)) 213f8caae30SBrad Bishop except: 214393846f1SBrad Bishop logger.error("Unable to set git config core.bare to false for %s" % os.path.join(repo_conf, 'modules', module)) 215f8caae30SBrad Bishop raise 2161a4b7ee2SBrad Bishop 217f8caae30SBrad Bishop Git.unpack(self, ud, destdir, d) 218f8caae30SBrad Bishop 219f8caae30SBrad Bishop ret = self.process_submodules(ud, ud.destdir, unpack_submodules, d) 220f8caae30SBrad Bishop 221f8caae30SBrad Bishop if not ud.bareclone and ret: 222393846f1SBrad Bishop # All submodules should already be downloaded and configured in the tree. This simply sets 223393846f1SBrad Bishop # up the configuration and checks out the files. The main project config should remain 224393846f1SBrad Bishop # unmodified, and no download from the internet should occur. 2251a4b7ee2SBrad Bishop runfetchcmd("%s submodule update --recursive --no-fetch" % (ud.basecmd), d, quiet=True, workdir=ud.destdir) 226