1eb8dc403SDave Cobbley""" 2eb8dc403SDave CobbleyBitBake 'Fetch' git submodules implementation 3eb8dc403SDave Cobbley 4eb8dc403SDave CobbleyInherits from and extends the Git fetcher to retrieve submodules of a git repository 5eb8dc403SDave Cobbleyafter cloning. 6eb8dc403SDave Cobbley 7eb8dc403SDave CobbleySRC_URI = "gitsm://<see Git fetcher for syntax>" 8eb8dc403SDave Cobbley 9eb8dc403SDave CobbleySee the Git fetcher, git://, for usage documentation. 10eb8dc403SDave Cobbley 11eb8dc403SDave CobbleyNOTE: Switching a SRC_URI from "git://" to "gitsm://" requires a clean of your recipe. 12eb8dc403SDave Cobbley 13eb8dc403SDave Cobbley""" 14eb8dc403SDave Cobbley 15eb8dc403SDave Cobbley# Copyright (C) 2013 Richard Purdie 16eb8dc403SDave Cobbley# 17c342db35SBrad Bishop# SPDX-License-Identifier: GPL-2.0-only 18eb8dc403SDave Cobbley# 19eb8dc403SDave Cobbley 20eb8dc403SDave Cobbleyimport os 21eb8dc403SDave Cobbleyimport bb 221a4b7ee2SBrad Bishopimport copy 2382c905dcSAndrew Geisslerimport shutil 2482c905dcSAndrew Geisslerimport tempfile 25eb8dc403SDave Cobbleyfrom bb.fetch2.git import Git 26eb8dc403SDave Cobbleyfrom bb.fetch2 import runfetchcmd 27eb8dc403SDave Cobbleyfrom bb.fetch2 import logger 281a4b7ee2SBrad Bishopfrom bb.fetch2 import Fetch 29eb8dc403SDave Cobbley 30eb8dc403SDave Cobbleyclass GitSM(Git): 31eb8dc403SDave Cobbley def supports(self, ud, d): 32eb8dc403SDave Cobbley """ 33eb8dc403SDave Cobbley Check to see if a given url can be fetched with git. 34eb8dc403SDave Cobbley """ 35eb8dc403SDave Cobbley return ud.type in ['gitsm'] 36eb8dc403SDave Cobbley 37f8caae30SBrad Bishop def process_submodules(self, ud, workdir, function, d): 38f8caae30SBrad Bishop """ 39f8caae30SBrad Bishop Iterate over all of the submodules in this repository and execute 40f8caae30SBrad Bishop the 'function' for each of them. 41f8caae30SBrad Bishop """ 42f8caae30SBrad Bishop 43f8caae30SBrad Bishop submodules = [] 44f8caae30SBrad Bishop paths = {} 45f8caae30SBrad Bishop revision = {} 46f8caae30SBrad Bishop uris = {} 47f8caae30SBrad Bishop subrevision = {} 48f8caae30SBrad Bishop 491a4b7ee2SBrad Bishop def parse_gitmodules(gitmodules): 501a4b7ee2SBrad Bishop modules = {} 511a4b7ee2SBrad Bishop module = "" 521a4b7ee2SBrad Bishop for line in gitmodules.splitlines(): 53eb8dc403SDave Cobbley if line.startswith('[submodule'): 541a4b7ee2SBrad Bishop module = line.split('"')[1] 551a4b7ee2SBrad Bishop modules[module] = {} 561a4b7ee2SBrad Bishop elif module and line.strip().startswith('path'): 571a4b7ee2SBrad Bishop path = line.split('=')[1].strip() 581a4b7ee2SBrad Bishop modules[module]['path'] = path 591a4b7ee2SBrad Bishop elif module and line.strip().startswith('url'): 601a4b7ee2SBrad Bishop url = line.split('=')[1].strip() 611a4b7ee2SBrad Bishop modules[module]['url'] = url 621a4b7ee2SBrad Bishop return modules 63eb8dc403SDave Cobbley 64f8caae30SBrad Bishop # Collect the defined submodules, and their attributes 651a4b7ee2SBrad Bishop for name in ud.names: 661a4b7ee2SBrad Bishop try: 67f8caae30SBrad Bishop gitmodules = runfetchcmd("%s show %s:.gitmodules" % (ud.basecmd, ud.revisions[name]), d, quiet=True, workdir=workdir) 681a4b7ee2SBrad Bishop except: 691a4b7ee2SBrad Bishop # No submodules to update 701a4b7ee2SBrad Bishop continue 711a4b7ee2SBrad Bishop 72f8caae30SBrad Bishop for m, md in parse_gitmodules(gitmodules).items(): 73f8caae30SBrad Bishop try: 74f8caae30SBrad Bishop module_hash = runfetchcmd("%s ls-tree -z -d %s %s" % (ud.basecmd, ud.revisions[name], md['path']), d, quiet=True, workdir=workdir) 75f8caae30SBrad Bishop except: 76f8caae30SBrad Bishop # If the command fails, we don't have a valid file to check. If it doesn't 77f8caae30SBrad Bishop # fail -- it still might be a failure, see next check... 78f8caae30SBrad Bishop module_hash = "" 79f8caae30SBrad Bishop 80f8caae30SBrad Bishop if not module_hash: 81d1e89497SAndrew Geissler logger.debug("submodule %s is defined, but is not initialized in the repository. Skipping", m) 82f8caae30SBrad Bishop continue 83f8caae30SBrad Bishop 841a4b7ee2SBrad Bishop submodules.append(m) 851a4b7ee2SBrad Bishop paths[m] = md['path'] 86f8caae30SBrad Bishop revision[m] = ud.revisions[name] 871a4b7ee2SBrad Bishop uris[m] = md['url'] 88f8caae30SBrad Bishop subrevision[m] = module_hash.split()[2] 89f8caae30SBrad Bishop 90f8caae30SBrad Bishop # Convert relative to absolute uri based on parent uri 911a4b7ee2SBrad Bishop if uris[m].startswith('..'): 921a4b7ee2SBrad Bishop newud = copy.copy(ud) 93f8caae30SBrad Bishop newud.path = os.path.realpath(os.path.join(newud.path, uris[m])) 941a4b7ee2SBrad Bishop uris[m] = Git._get_repo_url(self, newud) 951a4b7ee2SBrad Bishop 961a4b7ee2SBrad Bishop for module in submodules: 97f8caae30SBrad Bishop # Translate the module url into a SRC_URI 981a4b7ee2SBrad Bishop 99f8caae30SBrad Bishop if "://" in uris[module]: 100f8caae30SBrad Bishop # Properly formated URL already 1011a4b7ee2SBrad Bishop proto = uris[module].split(':', 1)[0] 1021a4b7ee2SBrad Bishop url = uris[module].replace('%s:' % proto, 'gitsm:', 1) 103f8caae30SBrad Bishop else: 104f8caae30SBrad Bishop if ":" in uris[module]: 105f8caae30SBrad Bishop # Most likely an SSH style reference 106f8caae30SBrad Bishop proto = "ssh" 107f8caae30SBrad Bishop if ":/" in uris[module]: 108f8caae30SBrad Bishop # Absolute reference, easy to convert.. 109f8caae30SBrad Bishop url = "gitsm://" + uris[module].replace(':/', '/', 1) 110f8caae30SBrad Bishop else: 111f8caae30SBrad Bishop # Relative reference, no way to know if this is right! 112f8caae30SBrad Bishop logger.warning("Submodule included by %s refers to relative ssh reference %s. References may fail if not absolute." % (ud.url, uris[module])) 113f8caae30SBrad Bishop url = "gitsm://" + uris[module].replace(':', '/', 1) 114f8caae30SBrad Bishop else: 115f8caae30SBrad Bishop # This has to be a file reference 116f8caae30SBrad Bishop proto = "file" 117f8caae30SBrad Bishop url = "gitsm://" + uris[module] 118f8caae30SBrad Bishop 1191a4b7ee2SBrad Bishop url += ';protocol=%s' % proto 1201a4b7ee2SBrad Bishop url += ";name=%s" % module 121393846f1SBrad Bishop url += ";subpath=%s" % module 1221a4b7ee2SBrad Bishop 1231a4b7ee2SBrad Bishop ld = d.createCopy() 1241a4b7ee2SBrad Bishop # Not necessary to set SRC_URI, since we're passing the URI to 1251a4b7ee2SBrad Bishop # Fetch. 1261a4b7ee2SBrad Bishop #ld.setVar('SRC_URI', url) 127f8caae30SBrad Bishop ld.setVar('SRCREV_%s' % module, subrevision[module]) 1281a4b7ee2SBrad Bishop 1291a4b7ee2SBrad Bishop # Workaround for issues with SRCPV/SRCREV_FORMAT errors 1301a4b7ee2SBrad Bishop # error refer to 'multiple' repositories. Only the repository 1311a4b7ee2SBrad Bishop # in the original SRC_URI actually matters... 1321a4b7ee2SBrad Bishop ld.setVar('SRCPV', d.getVar('SRCPV')) 1331a4b7ee2SBrad Bishop ld.setVar('SRCREV_FORMAT', module) 1341a4b7ee2SBrad Bishop 13582c905dcSAndrew Geissler function(ud, url, module, paths[module], workdir, ld) 1361a4b7ee2SBrad Bishop 137f8caae30SBrad Bishop return submodules != [] 138eb8dc403SDave Cobbley 13919323693SBrad Bishop def need_update(self, ud, d): 14019323693SBrad Bishop if Git.need_update(self, ud, d): 14119323693SBrad Bishop return True 14219323693SBrad Bishop 143d25ed324SAndrew Geissler need_update_list = [] 144d25ed324SAndrew Geissler def need_update_submodule(ud, url, module, modpath, workdir, d): 145d25ed324SAndrew Geissler url += ";bareclone=1;nobranch=1" 146d25ed324SAndrew Geissler 147d25ed324SAndrew Geissler try: 148d25ed324SAndrew Geissler newfetch = Fetch([url], d, cache=False) 149d25ed324SAndrew Geissler new_ud = newfetch.ud[url] 150d25ed324SAndrew Geissler if new_ud.method.need_update(new_ud, d): 151d25ed324SAndrew Geissler need_update_list.append(modpath) 152d25ed324SAndrew Geissler except Exception as e: 153d25ed324SAndrew Geissler logger.error('gitsm: submodule update check failed: %s %s' % (type(e).__name__, str(e))) 154d25ed324SAndrew Geissler need_update_result = True 155d25ed324SAndrew Geissler 156d25ed324SAndrew Geissler # If we're using a shallow mirror tarball it needs to be unpacked 157d25ed324SAndrew Geissler # temporarily so that we can examine the .gitmodules file 158d25ed324SAndrew Geissler if ud.shallow and os.path.exists(ud.fullshallow) and not os.path.exists(ud.clonedir): 159d25ed324SAndrew Geissler tmpdir = tempfile.mkdtemp(dir=d.getVar("DL_DIR")) 160d25ed324SAndrew Geissler runfetchcmd("tar -xzf %s" % ud.fullshallow, d, workdir=tmpdir) 161d25ed324SAndrew Geissler self.process_submodules(ud, tmpdir, need_update_submodule, d) 162d25ed324SAndrew Geissler shutil.rmtree(tmpdir) 163d25ed324SAndrew Geissler else: 164d25ed324SAndrew Geissler self.process_submodules(ud, ud.clonedir, need_update_submodule, d) 165d25ed324SAndrew Geissler 166*595f6308SAndrew Geissler if need_update_list: 167d1e89497SAndrew Geissler logger.debug('gitsm: Submodules requiring update: %s' % (' '.join(need_update_list))) 16819323693SBrad Bishop return True 16919323693SBrad Bishop 17019323693SBrad Bishop return False 17119323693SBrad Bishop 172eb8dc403SDave Cobbley def download(self, ud, d): 17382c905dcSAndrew Geissler def download_submodule(ud, url, module, modpath, workdir, d): 174f8caae30SBrad Bishop url += ";bareclone=1;nobranch=1" 175eb8dc403SDave Cobbley 176f8caae30SBrad Bishop # Is the following still needed? 177f8caae30SBrad Bishop #url += ";nocheckout=1" 178eb8dc403SDave Cobbley 1791a4b7ee2SBrad Bishop try: 180f8caae30SBrad Bishop newfetch = Fetch([url], d, cache=False) 181f8caae30SBrad Bishop newfetch.download() 182f8caae30SBrad Bishop except Exception as e: 183f8caae30SBrad Bishop logger.error('gitsm: submodule download failed: %s %s' % (type(e).__name__, str(e))) 184f8caae30SBrad Bishop raise 1851a4b7ee2SBrad Bishop 186f8caae30SBrad Bishop Git.download(self, ud, d) 18782c905dcSAndrew Geissler 18882c905dcSAndrew Geissler # If we're using a shallow mirror tarball it needs to be unpacked 18982c905dcSAndrew Geissler # temporarily so that we can examine the .gitmodules file 19082c905dcSAndrew Geissler if ud.shallow and os.path.exists(ud.fullshallow) and self.need_update(ud, d): 19182c905dcSAndrew Geissler tmpdir = tempfile.mkdtemp(dir=d.getVar("DL_DIR")) 19282c905dcSAndrew Geissler runfetchcmd("tar -xzf %s" % ud.fullshallow, d, workdir=tmpdir) 19382c905dcSAndrew Geissler self.process_submodules(ud, tmpdir, download_submodule, d) 19482c905dcSAndrew Geissler shutil.rmtree(tmpdir) 19582c905dcSAndrew Geissler else: 196f8caae30SBrad Bishop self.process_submodules(ud, ud.clonedir, download_submodule, d) 197eb8dc403SDave Cobbley 198eb8dc403SDave Cobbley def unpack(self, ud, destdir, d): 19982c905dcSAndrew Geissler def unpack_submodules(ud, url, module, modpath, workdir, d): 200f8caae30SBrad Bishop url += ";bareclone=1;nobranch=1" 201eb8dc403SDave Cobbley 202f8caae30SBrad Bishop # Figure out where we clone over the bare submodules... 2031a4b7ee2SBrad Bishop if ud.bareclone: 2041a4b7ee2SBrad Bishop repo_conf = ud.destdir 2051a4b7ee2SBrad Bishop else: 2061a4b7ee2SBrad Bishop repo_conf = os.path.join(ud.destdir, '.git') 2071a4b7ee2SBrad Bishop 2081a4b7ee2SBrad Bishop try: 209f8caae30SBrad Bishop newfetch = Fetch([url], d, cache=False) 210393846f1SBrad Bishop newfetch.unpack(root=os.path.dirname(os.path.join(repo_conf, 'modules', module))) 211f8caae30SBrad Bishop except Exception as e: 212f8caae30SBrad Bishop logger.error('gitsm: submodule unpack failed: %s %s' % (type(e).__name__, str(e))) 213f8caae30SBrad Bishop raise 2141a4b7ee2SBrad Bishop 215f8caae30SBrad Bishop local_path = newfetch.localpath(url) 2161a4b7ee2SBrad Bishop 217f8caae30SBrad Bishop # Correct the submodule references to the local download version... 218f8caae30SBrad Bishop runfetchcmd("%(basecmd)s config submodule.%(module)s.url %(url)s" % {'basecmd': ud.basecmd, 'module': module, 'url' : local_path}, d, workdir=ud.destdir) 2191a4b7ee2SBrad Bishop 220f8caae30SBrad Bishop if ud.shallow: 221f8caae30SBrad Bishop runfetchcmd("%(basecmd)s config submodule.%(module)s.shallow true" % {'basecmd': ud.basecmd, 'module': module}, d, workdir=ud.destdir) 2221a4b7ee2SBrad Bishop 2231a4b7ee2SBrad Bishop # Ensure the submodule repository is NOT set to bare, since we're checking it out... 224f8caae30SBrad Bishop try: 225393846f1SBrad Bishop runfetchcmd("%s config core.bare false" % (ud.basecmd), d, quiet=True, workdir=os.path.join(repo_conf, 'modules', module)) 226f8caae30SBrad Bishop except: 227393846f1SBrad Bishop logger.error("Unable to set git config core.bare to false for %s" % os.path.join(repo_conf, 'modules', module)) 228f8caae30SBrad Bishop raise 2291a4b7ee2SBrad Bishop 230f8caae30SBrad Bishop Git.unpack(self, ud, destdir, d) 231f8caae30SBrad Bishop 232f8caae30SBrad Bishop ret = self.process_submodules(ud, ud.destdir, unpack_submodules, d) 233f8caae30SBrad Bishop 234f8caae30SBrad Bishop if not ud.bareclone and ret: 235393846f1SBrad Bishop # All submodules should already be downloaded and configured in the tree. This simply sets 236393846f1SBrad Bishop # up the configuration and checks out the files. The main project config should remain 237393846f1SBrad Bishop # unmodified, and no download from the internet should occur. 2381a4b7ee2SBrad Bishop runfetchcmd("%s submodule update --recursive --no-fetch" % (ud.basecmd), d, quiet=True, workdir=ud.destdir) 2394ed12e16SAndrew Geissler 2404ed12e16SAndrew Geissler def implicit_urldata(self, ud, d): 2414ed12e16SAndrew Geissler import shutil, subprocess, tempfile 2424ed12e16SAndrew Geissler 2434ed12e16SAndrew Geissler urldata = [] 2444ed12e16SAndrew Geissler def add_submodule(ud, url, module, modpath, workdir, d): 2454ed12e16SAndrew Geissler url += ";bareclone=1;nobranch=1" 2464ed12e16SAndrew Geissler newfetch = Fetch([url], d, cache=False) 2474ed12e16SAndrew Geissler urldata.extend(newfetch.expanded_urldata()) 2484ed12e16SAndrew Geissler 2494ed12e16SAndrew Geissler # If we're using a shallow mirror tarball it needs to be unpacked 2504ed12e16SAndrew Geissler # temporarily so that we can examine the .gitmodules file 2514ed12e16SAndrew Geissler if ud.shallow and os.path.exists(ud.fullshallow) and ud.method.need_update(ud, d): 2524ed12e16SAndrew Geissler tmpdir = tempfile.mkdtemp(dir=d.getVar("DL_DIR")) 2534ed12e16SAndrew Geissler subprocess.check_call("tar -xzf %s" % ud.fullshallow, cwd=tmpdir, shell=True) 2544ed12e16SAndrew Geissler self.process_submodules(ud, tmpdir, add_submodule, d) 2554ed12e16SAndrew Geissler shutil.rmtree(tmpdir) 2564ed12e16SAndrew Geissler else: 2574ed12e16SAndrew Geissler self.process_submodules(ud, ud.clonedir, add_submodule, d) 2584ed12e16SAndrew Geissler 2594ed12e16SAndrew Geissler return urldata 260