1eb8dc403SDave Cobbley# ex:ts=4:sw=4:sts=4:et 2eb8dc403SDave Cobbley# -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- 3eb8dc403SDave Cobbley""" 4eb8dc403SDave CobbleyBitBake 'Fetch' git submodules implementation 5eb8dc403SDave Cobbley 6eb8dc403SDave CobbleyInherits from and extends the Git fetcher to retrieve submodules of a git repository 7eb8dc403SDave Cobbleyafter cloning. 8eb8dc403SDave Cobbley 9eb8dc403SDave CobbleySRC_URI = "gitsm://<see Git fetcher for syntax>" 10eb8dc403SDave Cobbley 11eb8dc403SDave CobbleySee the Git fetcher, git://, for usage documentation. 12eb8dc403SDave Cobbley 13eb8dc403SDave CobbleyNOTE: Switching a SRC_URI from "git://" to "gitsm://" requires a clean of your recipe. 14eb8dc403SDave Cobbley 15eb8dc403SDave Cobbley""" 16eb8dc403SDave Cobbley 17eb8dc403SDave Cobbley# Copyright (C) 2013 Richard Purdie 18eb8dc403SDave Cobbley# 19eb8dc403SDave Cobbley# This program is free software; you can redistribute it and/or modify 20eb8dc403SDave Cobbley# it under the terms of the GNU General Public License version 2 as 21eb8dc403SDave Cobbley# published by the Free Software Foundation. 22eb8dc403SDave Cobbley# 23eb8dc403SDave Cobbley# This program is distributed in the hope that it will be useful, 24eb8dc403SDave Cobbley# but WITHOUT ANY WARRANTY; without even the implied warranty of 25eb8dc403SDave Cobbley# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 26eb8dc403SDave Cobbley# GNU General Public License for more details. 27eb8dc403SDave Cobbley# 28eb8dc403SDave Cobbley# You should have received a copy of the GNU General Public License along 29eb8dc403SDave Cobbley# with this program; if not, write to the Free Software Foundation, Inc., 30eb8dc403SDave Cobbley# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. 31eb8dc403SDave Cobbley 32eb8dc403SDave Cobbleyimport os 33eb8dc403SDave Cobbleyimport bb 341a4b7ee2SBrad Bishopimport copy 35eb8dc403SDave Cobbleyfrom bb.fetch2.git import Git 36eb8dc403SDave Cobbleyfrom bb.fetch2 import runfetchcmd 37eb8dc403SDave Cobbleyfrom bb.fetch2 import logger 381a4b7ee2SBrad Bishopfrom bb.fetch2 import Fetch 391a4b7ee2SBrad Bishopfrom bb.fetch2 import BBFetchException 40eb8dc403SDave Cobbley 41eb8dc403SDave Cobbleyclass GitSM(Git): 42eb8dc403SDave Cobbley def supports(self, ud, d): 43eb8dc403SDave Cobbley """ 44eb8dc403SDave Cobbley Check to see if a given url can be fetched with git. 45eb8dc403SDave Cobbley """ 46eb8dc403SDave Cobbley return ud.type in ['gitsm'] 47eb8dc403SDave Cobbley 48*f8caae30SBrad Bishop def process_submodules(self, ud, workdir, function, d): 49*f8caae30SBrad Bishop """ 50*f8caae30SBrad Bishop Iterate over all of the submodules in this repository and execute 51*f8caae30SBrad Bishop the 'function' for each of them. 52*f8caae30SBrad Bishop """ 53*f8caae30SBrad Bishop 54*f8caae30SBrad Bishop submodules = [] 55*f8caae30SBrad Bishop paths = {} 56*f8caae30SBrad Bishop revision = {} 57*f8caae30SBrad Bishop uris = {} 58*f8caae30SBrad Bishop subrevision = {} 59*f8caae30SBrad Bishop 601a4b7ee2SBrad Bishop def parse_gitmodules(gitmodules): 611a4b7ee2SBrad Bishop modules = {} 621a4b7ee2SBrad Bishop module = "" 631a4b7ee2SBrad Bishop for line in gitmodules.splitlines(): 64eb8dc403SDave Cobbley if line.startswith('[submodule'): 651a4b7ee2SBrad Bishop module = line.split('"')[1] 661a4b7ee2SBrad Bishop modules[module] = {} 671a4b7ee2SBrad Bishop elif module and line.strip().startswith('path'): 681a4b7ee2SBrad Bishop path = line.split('=')[1].strip() 691a4b7ee2SBrad Bishop modules[module]['path'] = path 701a4b7ee2SBrad Bishop elif module and line.strip().startswith('url'): 711a4b7ee2SBrad Bishop url = line.split('=')[1].strip() 721a4b7ee2SBrad Bishop modules[module]['url'] = url 731a4b7ee2SBrad Bishop return modules 74eb8dc403SDave Cobbley 75*f8caae30SBrad Bishop # Collect the defined submodules, and their attributes 761a4b7ee2SBrad Bishop for name in ud.names: 771a4b7ee2SBrad Bishop try: 78*f8caae30SBrad Bishop gitmodules = runfetchcmd("%s show %s:.gitmodules" % (ud.basecmd, ud.revisions[name]), d, quiet=True, workdir=workdir) 791a4b7ee2SBrad Bishop except: 801a4b7ee2SBrad Bishop # No submodules to update 811a4b7ee2SBrad Bishop continue 821a4b7ee2SBrad Bishop 83*f8caae30SBrad Bishop for m, md in parse_gitmodules(gitmodules).items(): 84*f8caae30SBrad Bishop try: 85*f8caae30SBrad Bishop module_hash = runfetchcmd("%s ls-tree -z -d %s %s" % (ud.basecmd, ud.revisions[name], md['path']), d, quiet=True, workdir=workdir) 86*f8caae30SBrad Bishop except: 87*f8caae30SBrad Bishop # If the command fails, we don't have a valid file to check. If it doesn't 88*f8caae30SBrad Bishop # fail -- it still might be a failure, see next check... 89*f8caae30SBrad Bishop module_hash = "" 90*f8caae30SBrad Bishop 91*f8caae30SBrad Bishop if not module_hash: 92*f8caae30SBrad Bishop logger.debug(1, "submodule %s is defined, but is not initialized in the repository. Skipping", m) 93*f8caae30SBrad Bishop continue 94*f8caae30SBrad Bishop 951a4b7ee2SBrad Bishop submodules.append(m) 961a4b7ee2SBrad Bishop paths[m] = md['path'] 97*f8caae30SBrad Bishop revision[m] = ud.revisions[name] 981a4b7ee2SBrad Bishop uris[m] = md['url'] 99*f8caae30SBrad Bishop subrevision[m] = module_hash.split()[2] 100*f8caae30SBrad Bishop 101*f8caae30SBrad Bishop # Convert relative to absolute uri based on parent uri 1021a4b7ee2SBrad Bishop if uris[m].startswith('..'): 1031a4b7ee2SBrad Bishop newud = copy.copy(ud) 104*f8caae30SBrad Bishop newud.path = os.path.realpath(os.path.join(newud.path, uris[m])) 1051a4b7ee2SBrad Bishop uris[m] = Git._get_repo_url(self, newud) 1061a4b7ee2SBrad Bishop 1071a4b7ee2SBrad Bishop for module in submodules: 108*f8caae30SBrad Bishop # Translate the module url into a SRC_URI 1091a4b7ee2SBrad Bishop 110*f8caae30SBrad Bishop if "://" in uris[module]: 111*f8caae30SBrad Bishop # Properly formated URL already 1121a4b7ee2SBrad Bishop proto = uris[module].split(':', 1)[0] 1131a4b7ee2SBrad Bishop url = uris[module].replace('%s:' % proto, 'gitsm:', 1) 114*f8caae30SBrad Bishop else: 115*f8caae30SBrad Bishop if ":" in uris[module]: 116*f8caae30SBrad Bishop # Most likely an SSH style reference 117*f8caae30SBrad Bishop proto = "ssh" 118*f8caae30SBrad Bishop if ":/" in uris[module]: 119*f8caae30SBrad Bishop # Absolute reference, easy to convert.. 120*f8caae30SBrad Bishop url = "gitsm://" + uris[module].replace(':/', '/', 1) 121*f8caae30SBrad Bishop else: 122*f8caae30SBrad Bishop # Relative reference, no way to know if this is right! 123*f8caae30SBrad Bishop logger.warning("Submodule included by %s refers to relative ssh reference %s. References may fail if not absolute." % (ud.url, uris[module])) 124*f8caae30SBrad Bishop url = "gitsm://" + uris[module].replace(':', '/', 1) 125*f8caae30SBrad Bishop else: 126*f8caae30SBrad Bishop # This has to be a file reference 127*f8caae30SBrad Bishop proto = "file" 128*f8caae30SBrad Bishop url = "gitsm://" + uris[module] 129*f8caae30SBrad Bishop 1301a4b7ee2SBrad Bishop url += ';protocol=%s' % proto 1311a4b7ee2SBrad Bishop url += ";name=%s" % module 132*f8caae30SBrad Bishop url += ";subpath=%s" % paths[module] 1331a4b7ee2SBrad Bishop 1341a4b7ee2SBrad Bishop ld = d.createCopy() 1351a4b7ee2SBrad Bishop # Not necessary to set SRC_URI, since we're passing the URI to 1361a4b7ee2SBrad Bishop # Fetch. 1371a4b7ee2SBrad Bishop #ld.setVar('SRC_URI', url) 138*f8caae30SBrad Bishop ld.setVar('SRCREV_%s' % module, subrevision[module]) 1391a4b7ee2SBrad Bishop 1401a4b7ee2SBrad Bishop # Workaround for issues with SRCPV/SRCREV_FORMAT errors 1411a4b7ee2SBrad Bishop # error refer to 'multiple' repositories. Only the repository 1421a4b7ee2SBrad Bishop # in the original SRC_URI actually matters... 1431a4b7ee2SBrad Bishop ld.setVar('SRCPV', d.getVar('SRCPV')) 1441a4b7ee2SBrad Bishop ld.setVar('SRCREV_FORMAT', module) 1451a4b7ee2SBrad Bishop 146*f8caae30SBrad Bishop function(ud, url, module, paths[module], ld) 1471a4b7ee2SBrad Bishop 148*f8caae30SBrad Bishop return submodules != [] 149eb8dc403SDave Cobbley 150eb8dc403SDave Cobbley def download(self, ud, d): 151*f8caae30SBrad Bishop def download_submodule(ud, url, module, modpath, d): 152*f8caae30SBrad Bishop url += ";bareclone=1;nobranch=1" 153eb8dc403SDave Cobbley 154*f8caae30SBrad Bishop # Is the following still needed? 155*f8caae30SBrad Bishop #url += ";nocheckout=1" 156eb8dc403SDave Cobbley 1571a4b7ee2SBrad Bishop try: 158*f8caae30SBrad Bishop newfetch = Fetch([url], d, cache=False) 159*f8caae30SBrad Bishop newfetch.download() 160*f8caae30SBrad Bishop except Exception as e: 161*f8caae30SBrad Bishop logger.error('gitsm: submodule download failed: %s %s' % (type(e).__name__, str(e))) 162*f8caae30SBrad Bishop raise 1631a4b7ee2SBrad Bishop 164*f8caae30SBrad Bishop Git.download(self, ud, d) 165*f8caae30SBrad Bishop self.process_submodules(ud, ud.clonedir, download_submodule, d) 166eb8dc403SDave Cobbley 167eb8dc403SDave Cobbley def unpack(self, ud, destdir, d): 168*f8caae30SBrad Bishop def unpack_submodules(ud, url, module, modpath, d): 169*f8caae30SBrad Bishop url += ";bareclone=1;nobranch=1" 170eb8dc403SDave Cobbley 171*f8caae30SBrad Bishop # Figure out where we clone over the bare submodules... 1721a4b7ee2SBrad Bishop if ud.bareclone: 1731a4b7ee2SBrad Bishop repo_conf = ud.destdir 1741a4b7ee2SBrad Bishop else: 1751a4b7ee2SBrad Bishop repo_conf = os.path.join(ud.destdir, '.git') 1761a4b7ee2SBrad Bishop 1771a4b7ee2SBrad Bishop try: 178*f8caae30SBrad Bishop newfetch = Fetch([url], d, cache=False) 179*f8caae30SBrad Bishop newfetch.unpack(root=os.path.dirname(os.path.join(repo_conf, 'modules', modpath))) 180*f8caae30SBrad Bishop except Exception as e: 181*f8caae30SBrad Bishop logger.error('gitsm: submodule unpack failed: %s %s' % (type(e).__name__, str(e))) 182*f8caae30SBrad Bishop raise 1831a4b7ee2SBrad Bishop 184*f8caae30SBrad Bishop local_path = newfetch.localpath(url) 1851a4b7ee2SBrad Bishop 186*f8caae30SBrad Bishop # Correct the submodule references to the local download version... 187*f8caae30SBrad Bishop runfetchcmd("%(basecmd)s config submodule.%(module)s.url %(url)s" % {'basecmd': ud.basecmd, 'module': module, 'url' : local_path}, d, workdir=ud.destdir) 1881a4b7ee2SBrad Bishop 189*f8caae30SBrad Bishop if ud.shallow: 190*f8caae30SBrad Bishop runfetchcmd("%(basecmd)s config submodule.%(module)s.shallow true" % {'basecmd': ud.basecmd, 'module': module}, d, workdir=ud.destdir) 1911a4b7ee2SBrad Bishop 1921a4b7ee2SBrad Bishop # Ensure the submodule repository is NOT set to bare, since we're checking it out... 193*f8caae30SBrad Bishop try: 194*f8caae30SBrad Bishop runfetchcmd("%s config core.bare false" % (ud.basecmd), d, quiet=True, workdir=os.path.join(repo_conf, 'modules', modpath)) 195*f8caae30SBrad Bishop except: 196*f8caae30SBrad Bishop logger.error("Unable to set git config core.bare to false for %s" % os.path.join(repo_conf, 'modules', modpath)) 197*f8caae30SBrad Bishop raise 1981a4b7ee2SBrad Bishop 199*f8caae30SBrad Bishop Git.unpack(self, ud, destdir, d) 200*f8caae30SBrad Bishop 201*f8caae30SBrad Bishop ret = self.process_submodules(ud, ud.destdir, unpack_submodules, d) 202*f8caae30SBrad Bishop 203*f8caae30SBrad Bishop if not ud.bareclone and ret: 2041a4b7ee2SBrad Bishop # Run submodule update, this sets up the directories -- without touching the config 2051a4b7ee2SBrad Bishop runfetchcmd("%s submodule update --recursive --no-fetch" % (ud.basecmd), d, quiet=True, workdir=ud.destdir) 206