1eb8dc403SDave Cobbley""" 2eb8dc403SDave CobbleyBitBake 'Fetch' git submodules implementation 3eb8dc403SDave Cobbley 4eb8dc403SDave CobbleyInherits from and extends the Git fetcher to retrieve submodules of a git repository 5eb8dc403SDave Cobbleyafter cloning. 6eb8dc403SDave Cobbley 7eb8dc403SDave CobbleySRC_URI = "gitsm://<see Git fetcher for syntax>" 8eb8dc403SDave Cobbley 9eb8dc403SDave CobbleySee the Git fetcher, git://, for usage documentation. 10eb8dc403SDave Cobbley 11eb8dc403SDave CobbleyNOTE: Switching a SRC_URI from "git://" to "gitsm://" requires a clean of your recipe. 12eb8dc403SDave Cobbley 13eb8dc403SDave Cobbley""" 14eb8dc403SDave Cobbley 15eb8dc403SDave Cobbley# Copyright (C) 2013 Richard Purdie 16eb8dc403SDave Cobbley# 17*c342db35SBrad Bishop# SPDX-License-Identifier: GPL-2.0-only 18eb8dc403SDave Cobbley# 19eb8dc403SDave Cobbley 20eb8dc403SDave Cobbleyimport os 21eb8dc403SDave Cobbleyimport bb 221a4b7ee2SBrad Bishopimport copy 23eb8dc403SDave Cobbleyfrom bb.fetch2.git import Git 24eb8dc403SDave Cobbleyfrom bb.fetch2 import runfetchcmd 25eb8dc403SDave Cobbleyfrom bb.fetch2 import logger 261a4b7ee2SBrad Bishopfrom bb.fetch2 import Fetch 271a4b7ee2SBrad Bishopfrom bb.fetch2 import BBFetchException 28eb8dc403SDave Cobbley 29eb8dc403SDave Cobbleyclass GitSM(Git): 30eb8dc403SDave Cobbley def supports(self, ud, d): 31eb8dc403SDave Cobbley """ 32eb8dc403SDave Cobbley Check to see if a given url can be fetched with git. 33eb8dc403SDave Cobbley """ 34eb8dc403SDave Cobbley return ud.type in ['gitsm'] 35eb8dc403SDave Cobbley 36f8caae30SBrad Bishop def process_submodules(self, ud, workdir, function, d): 37f8caae30SBrad Bishop """ 38f8caae30SBrad Bishop Iterate over all of the submodules in this repository and execute 39f8caae30SBrad Bishop the 'function' for each of them. 40f8caae30SBrad Bishop """ 41f8caae30SBrad Bishop 42f8caae30SBrad Bishop submodules = [] 43f8caae30SBrad Bishop paths = {} 44f8caae30SBrad Bishop revision = {} 45f8caae30SBrad Bishop uris = {} 46f8caae30SBrad Bishop subrevision = {} 47f8caae30SBrad Bishop 481a4b7ee2SBrad Bishop def parse_gitmodules(gitmodules): 491a4b7ee2SBrad Bishop modules = {} 501a4b7ee2SBrad Bishop module = "" 511a4b7ee2SBrad Bishop for line in gitmodules.splitlines(): 52eb8dc403SDave Cobbley if line.startswith('[submodule'): 531a4b7ee2SBrad Bishop module = line.split('"')[1] 541a4b7ee2SBrad Bishop modules[module] = {} 551a4b7ee2SBrad Bishop elif module and line.strip().startswith('path'): 561a4b7ee2SBrad Bishop path = line.split('=')[1].strip() 571a4b7ee2SBrad Bishop modules[module]['path'] = path 581a4b7ee2SBrad Bishop elif module and line.strip().startswith('url'): 591a4b7ee2SBrad Bishop url = line.split('=')[1].strip() 601a4b7ee2SBrad Bishop modules[module]['url'] = url 611a4b7ee2SBrad Bishop return modules 62eb8dc403SDave Cobbley 63f8caae30SBrad Bishop # Collect the defined submodules, and their attributes 641a4b7ee2SBrad Bishop for name in ud.names: 651a4b7ee2SBrad Bishop try: 66f8caae30SBrad Bishop gitmodules = runfetchcmd("%s show %s:.gitmodules" % (ud.basecmd, ud.revisions[name]), d, quiet=True, workdir=workdir) 671a4b7ee2SBrad Bishop except: 681a4b7ee2SBrad Bishop # No submodules to update 691a4b7ee2SBrad Bishop continue 701a4b7ee2SBrad Bishop 71f8caae30SBrad Bishop for m, md in parse_gitmodules(gitmodules).items(): 72f8caae30SBrad Bishop try: 73f8caae30SBrad Bishop module_hash = runfetchcmd("%s ls-tree -z -d %s %s" % (ud.basecmd, ud.revisions[name], md['path']), d, quiet=True, workdir=workdir) 74f8caae30SBrad Bishop except: 75f8caae30SBrad Bishop # If the command fails, we don't have a valid file to check. If it doesn't 76f8caae30SBrad Bishop # fail -- it still might be a failure, see next check... 77f8caae30SBrad Bishop module_hash = "" 78f8caae30SBrad Bishop 79f8caae30SBrad Bishop if not module_hash: 80f8caae30SBrad Bishop logger.debug(1, "submodule %s is defined, but is not initialized in the repository. Skipping", m) 81f8caae30SBrad Bishop continue 82f8caae30SBrad Bishop 831a4b7ee2SBrad Bishop submodules.append(m) 841a4b7ee2SBrad Bishop paths[m] = md['path'] 85f8caae30SBrad Bishop revision[m] = ud.revisions[name] 861a4b7ee2SBrad Bishop uris[m] = md['url'] 87f8caae30SBrad Bishop subrevision[m] = module_hash.split()[2] 88f8caae30SBrad Bishop 89f8caae30SBrad Bishop # Convert relative to absolute uri based on parent uri 901a4b7ee2SBrad Bishop if uris[m].startswith('..'): 911a4b7ee2SBrad Bishop newud = copy.copy(ud) 92f8caae30SBrad Bishop newud.path = os.path.realpath(os.path.join(newud.path, uris[m])) 931a4b7ee2SBrad Bishop uris[m] = Git._get_repo_url(self, newud) 941a4b7ee2SBrad Bishop 951a4b7ee2SBrad Bishop for module in submodules: 96f8caae30SBrad Bishop # Translate the module url into a SRC_URI 971a4b7ee2SBrad Bishop 98f8caae30SBrad Bishop if "://" in uris[module]: 99f8caae30SBrad Bishop # Properly formated URL already 1001a4b7ee2SBrad Bishop proto = uris[module].split(':', 1)[0] 1011a4b7ee2SBrad Bishop url = uris[module].replace('%s:' % proto, 'gitsm:', 1) 102f8caae30SBrad Bishop else: 103f8caae30SBrad Bishop if ":" in uris[module]: 104f8caae30SBrad Bishop # Most likely an SSH style reference 105f8caae30SBrad Bishop proto = "ssh" 106f8caae30SBrad Bishop if ":/" in uris[module]: 107f8caae30SBrad Bishop # Absolute reference, easy to convert.. 108f8caae30SBrad Bishop url = "gitsm://" + uris[module].replace(':/', '/', 1) 109f8caae30SBrad Bishop else: 110f8caae30SBrad Bishop # Relative reference, no way to know if this is right! 111f8caae30SBrad Bishop logger.warning("Submodule included by %s refers to relative ssh reference %s. References may fail if not absolute." % (ud.url, uris[module])) 112f8caae30SBrad Bishop url = "gitsm://" + uris[module].replace(':', '/', 1) 113f8caae30SBrad Bishop else: 114f8caae30SBrad Bishop # This has to be a file reference 115f8caae30SBrad Bishop proto = "file" 116f8caae30SBrad Bishop url = "gitsm://" + uris[module] 117f8caae30SBrad Bishop 1181a4b7ee2SBrad Bishop url += ';protocol=%s' % proto 1191a4b7ee2SBrad Bishop url += ";name=%s" % module 120f8caae30SBrad Bishop url += ";subpath=%s" % paths[module] 1211a4b7ee2SBrad Bishop 1221a4b7ee2SBrad Bishop ld = d.createCopy() 1231a4b7ee2SBrad Bishop # Not necessary to set SRC_URI, since we're passing the URI to 1241a4b7ee2SBrad Bishop # Fetch. 1251a4b7ee2SBrad Bishop #ld.setVar('SRC_URI', url) 126f8caae30SBrad Bishop ld.setVar('SRCREV_%s' % module, subrevision[module]) 1271a4b7ee2SBrad Bishop 1281a4b7ee2SBrad Bishop # Workaround for issues with SRCPV/SRCREV_FORMAT errors 1291a4b7ee2SBrad Bishop # error refer to 'multiple' repositories. Only the repository 1301a4b7ee2SBrad Bishop # in the original SRC_URI actually matters... 1311a4b7ee2SBrad Bishop ld.setVar('SRCPV', d.getVar('SRCPV')) 1321a4b7ee2SBrad Bishop ld.setVar('SRCREV_FORMAT', module) 1331a4b7ee2SBrad Bishop 134f8caae30SBrad Bishop function(ud, url, module, paths[module], ld) 1351a4b7ee2SBrad Bishop 136f8caae30SBrad Bishop return submodules != [] 137eb8dc403SDave Cobbley 13819323693SBrad Bishop def need_update(self, ud, d): 13919323693SBrad Bishop if Git.need_update(self, ud, d): 14019323693SBrad Bishop return True 14119323693SBrad Bishop 14219323693SBrad Bishop try: 14319323693SBrad Bishop # Check for the nugget dropped by the download operation 14419323693SBrad Bishop known_srcrevs = runfetchcmd("%s config --get-all bitbake.srcrev" % \ 14519323693SBrad Bishop (ud.basecmd), d, workdir=ud.clonedir) 14619323693SBrad Bishop 14719323693SBrad Bishop if ud.revisions[ud.names[0]] not in known_srcrevs.split(): 14819323693SBrad Bishop return True 14919323693SBrad Bishop except bb.fetch2.FetchError: 15019323693SBrad Bishop # No srcrev nuggets, so this is new and needs to be updated 15119323693SBrad Bishop return True 15219323693SBrad Bishop 15319323693SBrad Bishop return False 15419323693SBrad Bishop 155eb8dc403SDave Cobbley def download(self, ud, d): 156f8caae30SBrad Bishop def download_submodule(ud, url, module, modpath, d): 157f8caae30SBrad Bishop url += ";bareclone=1;nobranch=1" 158eb8dc403SDave Cobbley 159f8caae30SBrad Bishop # Is the following still needed? 160f8caae30SBrad Bishop #url += ";nocheckout=1" 161eb8dc403SDave Cobbley 1621a4b7ee2SBrad Bishop try: 163f8caae30SBrad Bishop newfetch = Fetch([url], d, cache=False) 164f8caae30SBrad Bishop newfetch.download() 16519323693SBrad Bishop # Drop a nugget to add each of the srcrevs we've fetched (used by need_update) 16619323693SBrad Bishop runfetchcmd("%s config --add bitbake.srcrev %s" % \ 16719323693SBrad Bishop (ud.basecmd, ud.revisions[ud.names[0]]), d, workdir=ud.clonedir) 168f8caae30SBrad Bishop except Exception as e: 169f8caae30SBrad Bishop logger.error('gitsm: submodule download failed: %s %s' % (type(e).__name__, str(e))) 170f8caae30SBrad Bishop raise 1711a4b7ee2SBrad Bishop 172f8caae30SBrad Bishop Git.download(self, ud, d) 173f8caae30SBrad Bishop self.process_submodules(ud, ud.clonedir, download_submodule, d) 174eb8dc403SDave Cobbley 175eb8dc403SDave Cobbley def unpack(self, ud, destdir, d): 176f8caae30SBrad Bishop def unpack_submodules(ud, url, module, modpath, d): 177f8caae30SBrad Bishop url += ";bareclone=1;nobranch=1" 178eb8dc403SDave Cobbley 179f8caae30SBrad Bishop # Figure out where we clone over the bare submodules... 1801a4b7ee2SBrad Bishop if ud.bareclone: 1811a4b7ee2SBrad Bishop repo_conf = ud.destdir 1821a4b7ee2SBrad Bishop else: 1831a4b7ee2SBrad Bishop repo_conf = os.path.join(ud.destdir, '.git') 1841a4b7ee2SBrad Bishop 1851a4b7ee2SBrad Bishop try: 186f8caae30SBrad Bishop newfetch = Fetch([url], d, cache=False) 187f8caae30SBrad Bishop newfetch.unpack(root=os.path.dirname(os.path.join(repo_conf, 'modules', modpath))) 188f8caae30SBrad Bishop except Exception as e: 189f8caae30SBrad Bishop logger.error('gitsm: submodule unpack failed: %s %s' % (type(e).__name__, str(e))) 190f8caae30SBrad Bishop raise 1911a4b7ee2SBrad Bishop 192f8caae30SBrad Bishop local_path = newfetch.localpath(url) 1931a4b7ee2SBrad Bishop 194f8caae30SBrad Bishop # Correct the submodule references to the local download version... 195f8caae30SBrad Bishop runfetchcmd("%(basecmd)s config submodule.%(module)s.url %(url)s" % {'basecmd': ud.basecmd, 'module': module, 'url' : local_path}, d, workdir=ud.destdir) 1961a4b7ee2SBrad Bishop 197f8caae30SBrad Bishop if ud.shallow: 198f8caae30SBrad Bishop runfetchcmd("%(basecmd)s config submodule.%(module)s.shallow true" % {'basecmd': ud.basecmd, 'module': module}, d, workdir=ud.destdir) 1991a4b7ee2SBrad Bishop 2001a4b7ee2SBrad Bishop # Ensure the submodule repository is NOT set to bare, since we're checking it out... 201f8caae30SBrad Bishop try: 202f8caae30SBrad Bishop runfetchcmd("%s config core.bare false" % (ud.basecmd), d, quiet=True, workdir=os.path.join(repo_conf, 'modules', modpath)) 203f8caae30SBrad Bishop except: 204f8caae30SBrad Bishop logger.error("Unable to set git config core.bare to false for %s" % os.path.join(repo_conf, 'modules', modpath)) 205f8caae30SBrad Bishop raise 2061a4b7ee2SBrad Bishop 207f8caae30SBrad Bishop Git.unpack(self, ud, destdir, d) 208f8caae30SBrad Bishop 209f8caae30SBrad Bishop ret = self.process_submodules(ud, ud.destdir, unpack_submodules, d) 210f8caae30SBrad Bishop 211f8caae30SBrad Bishop if not ud.bareclone and ret: 2121a4b7ee2SBrad Bishop # Run submodule update, this sets up the directories -- without touching the config 2131a4b7ee2SBrad Bishop runfetchcmd("%s submodule update --recursive --no-fetch" % (ud.basecmd), d, quiet=True, workdir=ud.destdir) 214