1# ex:ts=4:sw=4:sts=4:et 2# -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- 3""" 4BitBake 'Fetch' git submodules implementation 5 6Inherits from and extends the Git fetcher to retrieve submodules of a git repository 7after cloning. 8 9SRC_URI = "gitsm://<see Git fetcher for syntax>" 10 11See the Git fetcher, git://, for usage documentation. 12 13NOTE: Switching a SRC_URI from "git://" to "gitsm://" requires a clean of your recipe. 14 15""" 16 17# Copyright (C) 2013 Richard Purdie 18# 19# This program is free software; you can redistribute it and/or modify 20# it under the terms of the GNU General Public License version 2 as 21# published by the Free Software Foundation. 22# 23# This program is distributed in the hope that it will be useful, 24# but WITHOUT ANY WARRANTY; without even the implied warranty of 25# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 26# GNU General Public License for more details. 27# 28# You should have received a copy of the GNU General Public License along 29# with this program; if not, write to the Free Software Foundation, Inc., 30# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. 31 32import os 33import bb 34import copy 35from bb.fetch2.git import Git 36from bb.fetch2 import runfetchcmd 37from bb.fetch2 import logger 38from bb.fetch2 import Fetch 39from bb.fetch2 import BBFetchException 40 41class GitSM(Git): 42 def supports(self, ud, d): 43 """ 44 Check to see if a given url can be fetched with git. 45 """ 46 return ud.type in ['gitsm'] 47 48 def process_submodules(self, ud, workdir, function, d): 49 """ 50 Iterate over all of the submodules in this repository and execute 51 the 'function' for each of them. 52 """ 53 54 submodules = [] 55 paths = {} 56 revision = {} 57 uris = {} 58 subrevision = {} 59 60 def parse_gitmodules(gitmodules): 61 modules = {} 62 module = "" 63 for line in gitmodules.splitlines(): 64 if line.startswith('[submodule'): 65 module = line.split('"')[1] 66 modules[module] = {} 67 elif module and line.strip().startswith('path'): 68 path = line.split('=')[1].strip() 69 modules[module]['path'] = path 70 elif module and line.strip().startswith('url'): 71 url = line.split('=')[1].strip() 72 modules[module]['url'] = url 73 return modules 74 75 # Collect the defined submodules, and their attributes 76 for name in ud.names: 77 try: 78 gitmodules = runfetchcmd("%s show %s:.gitmodules" % (ud.basecmd, ud.revisions[name]), d, quiet=True, workdir=workdir) 79 except: 80 # No submodules to update 81 continue 82 83 for m, md in parse_gitmodules(gitmodules).items(): 84 try: 85 module_hash = runfetchcmd("%s ls-tree -z -d %s %s" % (ud.basecmd, ud.revisions[name], md['path']), d, quiet=True, workdir=workdir) 86 except: 87 # If the command fails, we don't have a valid file to check. If it doesn't 88 # fail -- it still might be a failure, see next check... 89 module_hash = "" 90 91 if not module_hash: 92 logger.debug(1, "submodule %s is defined, but is not initialized in the repository. Skipping", m) 93 continue 94 95 submodules.append(m) 96 paths[m] = md['path'] 97 revision[m] = ud.revisions[name] 98 uris[m] = md['url'] 99 subrevision[m] = module_hash.split()[2] 100 101 # Convert relative to absolute uri based on parent uri 102 if uris[m].startswith('..'): 103 newud = copy.copy(ud) 104 newud.path = os.path.realpath(os.path.join(newud.path, uris[m])) 105 uris[m] = Git._get_repo_url(self, newud) 106 107 for module in submodules: 108 # Translate the module url into a SRC_URI 109 110 if "://" in uris[module]: 111 # Properly formated URL already 112 proto = uris[module].split(':', 1)[0] 113 url = uris[module].replace('%s:' % proto, 'gitsm:', 1) 114 else: 115 if ":" in uris[module]: 116 # Most likely an SSH style reference 117 proto = "ssh" 118 if ":/" in uris[module]: 119 # Absolute reference, easy to convert.. 120 url = "gitsm://" + uris[module].replace(':/', '/', 1) 121 else: 122 # Relative reference, no way to know if this is right! 123 logger.warning("Submodule included by %s refers to relative ssh reference %s. References may fail if not absolute." % (ud.url, uris[module])) 124 url = "gitsm://" + uris[module].replace(':', '/', 1) 125 else: 126 # This has to be a file reference 127 proto = "file" 128 url = "gitsm://" + uris[module] 129 130 url += ';protocol=%s' % proto 131 url += ";name=%s" % module 132 url += ";subpath=%s" % paths[module] 133 134 ld = d.createCopy() 135 # Not necessary to set SRC_URI, since we're passing the URI to 136 # Fetch. 137 #ld.setVar('SRC_URI', url) 138 ld.setVar('SRCREV_%s' % module, subrevision[module]) 139 140 # Workaround for issues with SRCPV/SRCREV_FORMAT errors 141 # error refer to 'multiple' repositories. Only the repository 142 # in the original SRC_URI actually matters... 143 ld.setVar('SRCPV', d.getVar('SRCPV')) 144 ld.setVar('SRCREV_FORMAT', module) 145 146 function(ud, url, module, paths[module], ld) 147 148 return submodules != [] 149 150 def need_update(self, ud, d): 151 if Git.need_update(self, ud, d): 152 return True 153 154 try: 155 # Check for the nugget dropped by the download operation 156 known_srcrevs = runfetchcmd("%s config --get-all bitbake.srcrev" % \ 157 (ud.basecmd), d, workdir=ud.clonedir) 158 159 if ud.revisions[ud.names[0]] not in known_srcrevs.split(): 160 return True 161 except bb.fetch2.FetchError: 162 # No srcrev nuggets, so this is new and needs to be updated 163 return True 164 165 return False 166 167 def download(self, ud, d): 168 def download_submodule(ud, url, module, modpath, d): 169 url += ";bareclone=1;nobranch=1" 170 171 # Is the following still needed? 172 #url += ";nocheckout=1" 173 174 try: 175 newfetch = Fetch([url], d, cache=False) 176 newfetch.download() 177 # Drop a nugget to add each of the srcrevs we've fetched (used by need_update) 178 runfetchcmd("%s config --add bitbake.srcrev %s" % \ 179 (ud.basecmd, ud.revisions[ud.names[0]]), d, workdir=ud.clonedir) 180 except Exception as e: 181 logger.error('gitsm: submodule download failed: %s %s' % (type(e).__name__, str(e))) 182 raise 183 184 Git.download(self, ud, d) 185 self.process_submodules(ud, ud.clonedir, download_submodule, d) 186 187 def unpack(self, ud, destdir, d): 188 def unpack_submodules(ud, url, module, modpath, d): 189 url += ";bareclone=1;nobranch=1" 190 191 # Figure out where we clone over the bare submodules... 192 if ud.bareclone: 193 repo_conf = ud.destdir 194 else: 195 repo_conf = os.path.join(ud.destdir, '.git') 196 197 try: 198 newfetch = Fetch([url], d, cache=False) 199 newfetch.unpack(root=os.path.dirname(os.path.join(repo_conf, 'modules', modpath))) 200 except Exception as e: 201 logger.error('gitsm: submodule unpack failed: %s %s' % (type(e).__name__, str(e))) 202 raise 203 204 local_path = newfetch.localpath(url) 205 206 # Correct the submodule references to the local download version... 207 runfetchcmd("%(basecmd)s config submodule.%(module)s.url %(url)s" % {'basecmd': ud.basecmd, 'module': module, 'url' : local_path}, d, workdir=ud.destdir) 208 209 if ud.shallow: 210 runfetchcmd("%(basecmd)s config submodule.%(module)s.shallow true" % {'basecmd': ud.basecmd, 'module': module}, d, workdir=ud.destdir) 211 212 # Ensure the submodule repository is NOT set to bare, since we're checking it out... 213 try: 214 runfetchcmd("%s config core.bare false" % (ud.basecmd), d, quiet=True, workdir=os.path.join(repo_conf, 'modules', modpath)) 215 except: 216 logger.error("Unable to set git config core.bare to false for %s" % os.path.join(repo_conf, 'modules', modpath)) 217 raise 218 219 Git.unpack(self, ud, destdir, d) 220 221 ret = self.process_submodules(ud, ud.destdir, unpack_submodules, d) 222 223 if not ud.bareclone and ret: 224 # Run submodule update, this sets up the directories -- without touching the config 225 runfetchcmd("%s submodule update --recursive --no-fetch" % (ud.basecmd), d, quiet=True, workdir=ud.destdir) 226