1# ex:ts=4:sw=4:sts=4:et 2# -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- 3""" 4BitBake 'Fetch' git submodules implementation 5 6Inherits from and extends the Git fetcher to retrieve submodules of a git repository 7after cloning. 8 9SRC_URI = "gitsm://<see Git fetcher for syntax>" 10 11See the Git fetcher, git://, for usage documentation. 12 13NOTE: Switching a SRC_URI from "git://" to "gitsm://" requires a clean of your recipe. 14 15""" 16 17# Copyright (C) 2013 Richard Purdie 18# 19# This program is free software; you can redistribute it and/or modify 20# it under the terms of the GNU General Public License version 2 as 21# published by the Free Software Foundation. 22# 23# This program is distributed in the hope that it will be useful, 24# but WITHOUT ANY WARRANTY; without even the implied warranty of 25# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 26# GNU General Public License for more details. 27# 28# You should have received a copy of the GNU General Public License along 29# with this program; if not, write to the Free Software Foundation, Inc., 30# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. 31 32import os 33import bb 34import copy 35from bb.fetch2.git import Git 36from bb.fetch2 import runfetchcmd 37from bb.fetch2 import logger 38from bb.fetch2 import Fetch 39from bb.fetch2 import BBFetchException 40 41class GitSM(Git): 42 def supports(self, ud, d): 43 """ 44 Check to see if a given url can be fetched with git. 45 """ 46 return ud.type in ['gitsm'] 47 48 def process_submodules(self, ud, workdir, function, d): 49 """ 50 Iterate over all of the submodules in this repository and execute 51 the 'function' for each of them. 52 """ 53 54 submodules = [] 55 paths = {} 56 revision = {} 57 uris = {} 58 subrevision = {} 59 60 def parse_gitmodules(gitmodules): 61 modules = {} 62 module = "" 63 for line in gitmodules.splitlines(): 64 if line.startswith('[submodule'): 65 module = line.split('"')[1] 66 modules[module] = {} 67 elif module and line.strip().startswith('path'): 68 path = line.split('=')[1].strip() 69 modules[module]['path'] = path 70 elif module and line.strip().startswith('url'): 71 url = line.split('=')[1].strip() 72 modules[module]['url'] = url 73 return modules 74 75 # Collect the defined submodules, and their attributes 76 for name in ud.names: 77 try: 78 gitmodules = runfetchcmd("%s show %s:.gitmodules" % (ud.basecmd, ud.revisions[name]), d, quiet=True, workdir=workdir) 79 except: 80 # No submodules to update 81 continue 82 83 for m, md in parse_gitmodules(gitmodules).items(): 84 try: 85 module_hash = runfetchcmd("%s ls-tree -z -d %s %s" % (ud.basecmd, ud.revisions[name], md['path']), d, quiet=True, workdir=workdir) 86 except: 87 # If the command fails, we don't have a valid file to check. If it doesn't 88 # fail -- it still might be a failure, see next check... 89 module_hash = "" 90 91 if not module_hash: 92 logger.debug(1, "submodule %s is defined, but is not initialized in the repository. Skipping", m) 93 continue 94 95 submodules.append(m) 96 paths[m] = md['path'] 97 revision[m] = ud.revisions[name] 98 uris[m] = md['url'] 99 subrevision[m] = module_hash.split()[2] 100 101 # Convert relative to absolute uri based on parent uri 102 if uris[m].startswith('..'): 103 newud = copy.copy(ud) 104 newud.path = os.path.realpath(os.path.join(newud.path, uris[m])) 105 uris[m] = Git._get_repo_url(self, newud) 106 107 for module in submodules: 108 # Translate the module url into a SRC_URI 109 110 if "://" in uris[module]: 111 # Properly formated URL already 112 proto = uris[module].split(':', 1)[0] 113 url = uris[module].replace('%s:' % proto, 'gitsm:', 1) 114 else: 115 if ":" in uris[module]: 116 # Most likely an SSH style reference 117 proto = "ssh" 118 if ":/" in uris[module]: 119 # Absolute reference, easy to convert.. 120 url = "gitsm://" + uris[module].replace(':/', '/', 1) 121 else: 122 # Relative reference, no way to know if this is right! 123 logger.warning("Submodule included by %s refers to relative ssh reference %s. References may fail if not absolute." % (ud.url, uris[module])) 124 url = "gitsm://" + uris[module].replace(':', '/', 1) 125 else: 126 # This has to be a file reference 127 proto = "file" 128 url = "gitsm://" + uris[module] 129 130 url += ';protocol=%s' % proto 131 url += ";name=%s" % module 132 url += ";subpath=%s" % paths[module] 133 134 ld = d.createCopy() 135 # Not necessary to set SRC_URI, since we're passing the URI to 136 # Fetch. 137 #ld.setVar('SRC_URI', url) 138 ld.setVar('SRCREV_%s' % module, subrevision[module]) 139 140 # Workaround for issues with SRCPV/SRCREV_FORMAT errors 141 # error refer to 'multiple' repositories. Only the repository 142 # in the original SRC_URI actually matters... 143 ld.setVar('SRCPV', d.getVar('SRCPV')) 144 ld.setVar('SRCREV_FORMAT', module) 145 146 function(ud, url, module, paths[module], ld) 147 148 return submodules != [] 149 150 def download(self, ud, d): 151 def download_submodule(ud, url, module, modpath, d): 152 url += ";bareclone=1;nobranch=1" 153 154 # Is the following still needed? 155 #url += ";nocheckout=1" 156 157 try: 158 newfetch = Fetch([url], d, cache=False) 159 newfetch.download() 160 except Exception as e: 161 logger.error('gitsm: submodule download failed: %s %s' % (type(e).__name__, str(e))) 162 raise 163 164 Git.download(self, ud, d) 165 self.process_submodules(ud, ud.clonedir, download_submodule, d) 166 167 def unpack(self, ud, destdir, d): 168 def unpack_submodules(ud, url, module, modpath, d): 169 url += ";bareclone=1;nobranch=1" 170 171 # Figure out where we clone over the bare submodules... 172 if ud.bareclone: 173 repo_conf = ud.destdir 174 else: 175 repo_conf = os.path.join(ud.destdir, '.git') 176 177 try: 178 newfetch = Fetch([url], d, cache=False) 179 newfetch.unpack(root=os.path.dirname(os.path.join(repo_conf, 'modules', modpath))) 180 except Exception as e: 181 logger.error('gitsm: submodule unpack failed: %s %s' % (type(e).__name__, str(e))) 182 raise 183 184 local_path = newfetch.localpath(url) 185 186 # Correct the submodule references to the local download version... 187 runfetchcmd("%(basecmd)s config submodule.%(module)s.url %(url)s" % {'basecmd': ud.basecmd, 'module': module, 'url' : local_path}, d, workdir=ud.destdir) 188 189 if ud.shallow: 190 runfetchcmd("%(basecmd)s config submodule.%(module)s.shallow true" % {'basecmd': ud.basecmd, 'module': module}, d, workdir=ud.destdir) 191 192 # Ensure the submodule repository is NOT set to bare, since we're checking it out... 193 try: 194 runfetchcmd("%s config core.bare false" % (ud.basecmd), d, quiet=True, workdir=os.path.join(repo_conf, 'modules', modpath)) 195 except: 196 logger.error("Unable to set git config core.bare to false for %s" % os.path.join(repo_conf, 'modules', modpath)) 197 raise 198 199 Git.unpack(self, ud, destdir, d) 200 201 ret = self.process_submodules(ud, ud.destdir, unpack_submodules, d) 202 203 if not ud.bareclone and ret: 204 # Run submodule update, this sets up the directories -- without touching the config 205 runfetchcmd("%s submodule update --recursive --no-fetch" % (ud.basecmd), d, quiet=True, workdir=ud.destdir) 206