1""" 2BitBake 'Fetch' git submodules implementation 3 4Inherits from and extends the Git fetcher to retrieve submodules of a git repository 5after cloning. 6 7SRC_URI = "gitsm://<see Git fetcher for syntax>" 8 9See the Git fetcher, git://, for usage documentation. 10 11NOTE: Switching a SRC_URI from "git://" to "gitsm://" requires a clean of your recipe. 12 13""" 14 15# Copyright (C) 2013 Richard Purdie 16# 17# SPDX-License-Identifier: GPL-2.0-only 18# 19 20import os 21import bb 22import copy 23import shutil 24import tempfile 25from bb.fetch2.git import Git 26from bb.fetch2 import runfetchcmd 27from bb.fetch2 import logger 28from bb.fetch2 import Fetch 29 30class GitSM(Git): 31 def supports(self, ud, d): 32 """ 33 Check to see if a given url can be fetched with git. 34 """ 35 return ud.type in ['gitsm'] 36 37 def process_submodules(self, ud, workdir, function, d): 38 """ 39 Iterate over all of the submodules in this repository and execute 40 the 'function' for each of them. 41 """ 42 43 submodules = [] 44 paths = {} 45 revision = {} 46 uris = {} 47 subrevision = {} 48 49 def parse_gitmodules(gitmodules): 50 modules = {} 51 module = "" 52 for line in gitmodules.splitlines(): 53 if line.startswith('[submodule'): 54 module = line.split('"')[1] 55 modules[module] = {} 56 elif module and line.strip().startswith('path'): 57 path = line.split('=')[1].strip() 58 modules[module]['path'] = path 59 elif module and line.strip().startswith('url'): 60 url = line.split('=')[1].strip() 61 modules[module]['url'] = url 62 return modules 63 64 # Collect the defined submodules, and their attributes 65 for name in ud.names: 66 try: 67 gitmodules = runfetchcmd("%s show %s:.gitmodules" % (ud.basecmd, ud.revisions[name]), d, quiet=True, workdir=workdir) 68 except: 69 # No submodules to update 70 continue 71 72 for m, md in parse_gitmodules(gitmodules).items(): 73 try: 74 module_hash = runfetchcmd("%s ls-tree -z -d %s %s" % (ud.basecmd, ud.revisions[name], md['path']), d, quiet=True, workdir=workdir) 75 except: 76 # If the command fails, we don't have a valid file to check. If it doesn't 77 # fail -- it still might be a failure, see next check... 78 module_hash = "" 79 80 if not module_hash: 81 logger.debug("submodule %s is defined, but is not initialized in the repository. Skipping", m) 82 continue 83 84 submodules.append(m) 85 paths[m] = md['path'] 86 revision[m] = ud.revisions[name] 87 uris[m] = md['url'] 88 subrevision[m] = module_hash.split()[2] 89 90 # Convert relative to absolute uri based on parent uri 91 if uris[m].startswith('..') or uris[m].startswith('./'): 92 newud = copy.copy(ud) 93 newud.path = os.path.normpath(os.path.join(newud.path, uris[m])) 94 uris[m] = Git._get_repo_url(self, newud) 95 96 for module in submodules: 97 # Translate the module url into a SRC_URI 98 99 if "://" in uris[module]: 100 # Properly formated URL already 101 proto = uris[module].split(':', 1)[0] 102 url = uris[module].replace('%s:' % proto, 'gitsm:', 1) 103 else: 104 if ":" in uris[module]: 105 # Most likely an SSH style reference 106 proto = "ssh" 107 if ":/" in uris[module]: 108 # Absolute reference, easy to convert.. 109 url = "gitsm://" + uris[module].replace(':/', '/', 1) 110 else: 111 # Relative reference, no way to know if this is right! 112 logger.warning("Submodule included by %s refers to relative ssh reference %s. References may fail if not absolute." % (ud.url, uris[module])) 113 url = "gitsm://" + uris[module].replace(':', '/', 1) 114 else: 115 # This has to be a file reference 116 proto = "file" 117 url = "gitsm://" + uris[module] 118 if url.endswith("{}{}".format(ud.host, ud.path)): 119 raise bb.fetch2.FetchError("Submodule refers to the parent repository. This will cause deadlock situation in current version of Bitbake." \ 120 "Consider using git fetcher instead.") 121 122 url += ';protocol=%s' % proto 123 url += ";name=%s" % module 124 url += ";subpath=%s" % module 125 url += ";nobranch=1" 126 url += ";lfs=%s" % self._need_lfs(ud) 127 # Note that adding "user=" here to give credentials to the 128 # submodule is not supported. Since using SRC_URI to give git:// 129 # URL a password is not supported, one have to use one of the 130 # recommended way (eg. ~/.netrc or SSH config) which does specify 131 # the user (See comment in git.py). 132 # So, we will not take patches adding "user=" support here. 133 134 ld = d.createCopy() 135 # Not necessary to set SRC_URI, since we're passing the URI to 136 # Fetch. 137 #ld.setVar('SRC_URI', url) 138 ld.setVar('SRCREV_%s' % module, subrevision[module]) 139 140 # Workaround for issues with SRCPV/SRCREV_FORMAT errors 141 # error refer to 'multiple' repositories. Only the repository 142 # in the original SRC_URI actually matters... 143 ld.setVar('SRCPV', d.getVar('SRCPV')) 144 ld.setVar('SRCREV_FORMAT', module) 145 146 function(ud, url, module, paths[module], workdir, ld) 147 148 return submodules != [] 149 150 def need_update(self, ud, d): 151 if Git.need_update(self, ud, d): 152 return True 153 154 need_update_list = [] 155 def need_update_submodule(ud, url, module, modpath, workdir, d): 156 url += ";bareclone=1;nobranch=1" 157 158 try: 159 newfetch = Fetch([url], d, cache=False) 160 new_ud = newfetch.ud[url] 161 if new_ud.method.need_update(new_ud, d): 162 need_update_list.append(modpath) 163 except Exception as e: 164 logger.error('gitsm: submodule update check failed: %s %s' % (type(e).__name__, str(e))) 165 need_update_result = True 166 167 # If we're using a shallow mirror tarball it needs to be unpacked 168 # temporarily so that we can examine the .gitmodules file 169 if ud.shallow and os.path.exists(ud.fullshallow) and not os.path.exists(ud.clonedir): 170 tmpdir = tempfile.mkdtemp(dir=d.getVar("DL_DIR")) 171 runfetchcmd("tar -xzf %s" % ud.fullshallow, d, workdir=tmpdir) 172 self.process_submodules(ud, tmpdir, need_update_submodule, d) 173 shutil.rmtree(tmpdir) 174 else: 175 self.process_submodules(ud, ud.clonedir, need_update_submodule, d) 176 177 if need_update_list: 178 logger.debug('gitsm: Submodules requiring update: %s' % (' '.join(need_update_list))) 179 return True 180 181 return False 182 183 def download(self, ud, d): 184 def download_submodule(ud, url, module, modpath, workdir, d): 185 url += ";bareclone=1;nobranch=1" 186 187 # Is the following still needed? 188 #url += ";nocheckout=1" 189 190 try: 191 newfetch = Fetch([url], d, cache=False) 192 newfetch.download() 193 except Exception as e: 194 logger.error('gitsm: submodule download failed: %s %s' % (type(e).__name__, str(e))) 195 raise 196 197 Git.download(self, ud, d) 198 199 # If we're using a shallow mirror tarball it needs to be unpacked 200 # temporarily so that we can examine the .gitmodules file 201 if ud.shallow and os.path.exists(ud.fullshallow) and self.need_update(ud, d): 202 tmpdir = tempfile.mkdtemp(dir=d.getVar("DL_DIR")) 203 runfetchcmd("tar -xzf %s" % ud.fullshallow, d, workdir=tmpdir) 204 self.process_submodules(ud, tmpdir, download_submodule, d) 205 shutil.rmtree(tmpdir) 206 else: 207 self.process_submodules(ud, ud.clonedir, download_submodule, d) 208 209 def unpack(self, ud, destdir, d): 210 def unpack_submodules(ud, url, module, modpath, workdir, d): 211 url += ";bareclone=1;nobranch=1" 212 213 # Figure out where we clone over the bare submodules... 214 if ud.bareclone: 215 repo_conf = ud.destdir 216 else: 217 repo_conf = os.path.join(ud.destdir, '.git') 218 219 try: 220 newfetch = Fetch([url], d, cache=False) 221 # modpath is needed by unpack tracer to calculate submodule 222 # checkout dir 223 new_ud = newfetch.ud[url] 224 new_ud.modpath = modpath 225 newfetch.unpack(root=os.path.dirname(os.path.join(repo_conf, 'modules', module))) 226 except Exception as e: 227 logger.error('gitsm: submodule unpack failed: %s %s' % (type(e).__name__, str(e))) 228 raise 229 230 local_path = newfetch.localpath(url) 231 232 # Correct the submodule references to the local download version... 233 runfetchcmd("%(basecmd)s config submodule.%(module)s.url %(url)s" % {'basecmd': ud.basecmd, 'module': module, 'url' : local_path}, d, workdir=ud.destdir) 234 235 if ud.shallow: 236 runfetchcmd("%(basecmd)s config submodule.%(module)s.shallow true" % {'basecmd': ud.basecmd, 'module': module}, d, workdir=ud.destdir) 237 238 # Ensure the submodule repository is NOT set to bare, since we're checking it out... 239 try: 240 runfetchcmd("%s config core.bare false" % (ud.basecmd), d, quiet=True, workdir=os.path.join(repo_conf, 'modules', module)) 241 except: 242 logger.error("Unable to set git config core.bare to false for %s" % os.path.join(repo_conf, 'modules', module)) 243 raise 244 245 Git.unpack(self, ud, destdir, d) 246 247 ret = self.process_submodules(ud, ud.destdir, unpack_submodules, d) 248 249 if not ud.bareclone and ret: 250 # All submodules should already be downloaded and configured in the tree. This simply 251 # sets up the configuration and checks out the files. The main project config should 252 # remain unmodified, and no download from the internet should occur. As such, lfs smudge 253 # should also be skipped as these files were already smudged in the fetch stage if lfs 254 # was enabled. 255 runfetchcmd("GIT_LFS_SKIP_SMUDGE=1 %s submodule update --recursive --no-fetch" % (ud.basecmd), d, quiet=True, workdir=ud.destdir) 256 257 def implicit_urldata(self, ud, d): 258 import shutil, subprocess, tempfile 259 260 urldata = [] 261 def add_submodule(ud, url, module, modpath, workdir, d): 262 url += ";bareclone=1;nobranch=1" 263 newfetch = Fetch([url], d, cache=False) 264 urldata.extend(newfetch.expanded_urldata()) 265 266 # If we're using a shallow mirror tarball it needs to be unpacked 267 # temporarily so that we can examine the .gitmodules file 268 if ud.shallow and os.path.exists(ud.fullshallow) and ud.method.need_update(ud, d): 269 tmpdir = tempfile.mkdtemp(dir=d.getVar("DL_DIR")) 270 subprocess.check_call("tar -xzf %s" % ud.fullshallow, cwd=tmpdir, shell=True) 271 self.process_submodules(ud, tmpdir, add_submodule, d) 272 shutil.rmtree(tmpdir) 273 else: 274 self.process_submodules(ud, ud.clonedir, add_submodule, d) 275 276 return urldata 277