1""" 2BitBake 'Fetch' git submodules implementation 3 4Inherits from and extends the Git fetcher to retrieve submodules of a git repository 5after cloning. 6 7SRC_URI = "gitsm://<see Git fetcher for syntax>" 8 9See the Git fetcher, git://, for usage documentation. 10 11NOTE: Switching a SRC_URI from "git://" to "gitsm://" requires a clean of your recipe. 12 13""" 14 15# Copyright (C) 2013 Richard Purdie 16# 17# SPDX-License-Identifier: GPL-2.0-only 18# 19 20import os 21import bb 22import copy 23import shutil 24import tempfile 25from bb.fetch2.git import Git 26from bb.fetch2 import runfetchcmd 27from bb.fetch2 import logger 28from bb.fetch2 import Fetch 29 30class GitSM(Git): 31 def supports(self, ud, d): 32 """ 33 Check to see if a given url can be fetched with git. 34 """ 35 return ud.type in ['gitsm'] 36 37 def process_submodules(self, ud, workdir, function, d): 38 """ 39 Iterate over all of the submodules in this repository and execute 40 the 'function' for each of them. 41 """ 42 43 submodules = [] 44 paths = {} 45 revision = {} 46 uris = {} 47 subrevision = {} 48 49 def parse_gitmodules(gitmodules): 50 modules = {} 51 module = "" 52 for line in gitmodules.splitlines(): 53 if line.startswith('[submodule'): 54 module = line.split('"')[1] 55 modules[module] = {} 56 elif module and line.strip().startswith('path'): 57 path = line.split('=')[1].strip() 58 modules[module]['path'] = path 59 elif module and line.strip().startswith('url'): 60 url = line.split('=')[1].strip() 61 modules[module]['url'] = url 62 return modules 63 64 # Collect the defined submodules, and their attributes 65 try: 66 gitmodules = runfetchcmd("%s show %s:.gitmodules" % (ud.basecmd, ud.revision), d, quiet=True, workdir=workdir) 67 except: 68 # No submodules to update 69 gitmodules = "" 70 71 for m, md in parse_gitmodules(gitmodules).items(): 72 try: 73 module_hash = runfetchcmd("%s ls-tree -z -d %s %s" % (ud.basecmd, ud.revision, md['path']), d, quiet=True, workdir=workdir) 74 except: 75 # If the command fails, we don't have a valid file to check. If it doesn't 76 # fail -- it still might be a failure, see next check... 77 module_hash = "" 78 79 if not module_hash: 80 logger.debug("submodule %s is defined, but is not initialized in the repository. Skipping", m) 81 continue 82 83 submodules.append(m) 84 paths[m] = md['path'] 85 revision[m] = ud.revision 86 uris[m] = md['url'] 87 subrevision[m] = module_hash.split()[2] 88 89 # Convert relative to absolute uri based on parent uri 90 if uris[m].startswith('..') or uris[m].startswith('./'): 91 newud = copy.copy(ud) 92 newud.path = os.path.normpath(os.path.join(newud.path, uris[m])) 93 uris[m] = Git._get_repo_url(self, newud) 94 95 for module in submodules: 96 # Translate the module url into a SRC_URI 97 98 if "://" in uris[module]: 99 # Properly formated URL already 100 proto = uris[module].split(':', 1)[0] 101 url = uris[module].replace('%s:' % proto, 'gitsm:', 1) 102 else: 103 if ":" in uris[module]: 104 # Most likely an SSH style reference 105 proto = "ssh" 106 if ":/" in uris[module]: 107 # Absolute reference, easy to convert.. 108 url = "gitsm://" + uris[module].replace(':/', '/', 1) 109 else: 110 # Relative reference, no way to know if this is right! 111 logger.warning("Submodule included by %s refers to relative ssh reference %s. References may fail if not absolute." % (ud.url, uris[module])) 112 url = "gitsm://" + uris[module].replace(':', '/', 1) 113 else: 114 # This has to be a file reference 115 proto = "file" 116 url = "gitsm://" + uris[module] 117 if url.endswith("{}{}".format(ud.host, ud.path)): 118 raise bb.fetch2.FetchError("Submodule refers to the parent repository. This will cause deadlock situation in current version of Bitbake." \ 119 "Consider using git fetcher instead.") 120 121 url += ';protocol=%s' % proto 122 url += ";name=%s" % module 123 url += ";subpath=%s" % module 124 url += ";nobranch=1" 125 url += ";lfs=%s" % ("1" if self._need_lfs(ud) else "0") 126 # Note that adding "user=" here to give credentials to the 127 # submodule is not supported. Since using SRC_URI to give git:// 128 # URL a password is not supported, one have to use one of the 129 # recommended way (eg. ~/.netrc or SSH config) which does specify 130 # the user (See comment in git.py). 131 # So, we will not take patches adding "user=" support here. 132 133 ld = d.createCopy() 134 # Not necessary to set SRC_URI, since we're passing the URI to 135 # Fetch. 136 #ld.setVar('SRC_URI', url) 137 ld.setVar('SRCREV_%s' % module, subrevision[module]) 138 139 # Workaround for issues with SRCPV/SRCREV_FORMAT errors 140 # error refer to 'multiple' repositories. Only the repository 141 # in the original SRC_URI actually matters... 142 ld.setVar('SRCPV', d.getVar('SRCPV')) 143 ld.setVar('SRCREV_FORMAT', module) 144 145 function(ud, url, module, paths[module], workdir, ld) 146 147 return submodules != [] 148 149 def call_process_submodules(self, ud, d, extra_check, subfunc): 150 # If we're using a shallow mirror tarball it needs to be 151 # unpacked temporarily so that we can examine the .gitmodules file 152 # Unpack even when ud.clonedir is not available, 153 # which may occur during a fast shallow clone 154 unpack = extra_check or not os.path.exists(ud.clonedir) 155 if ud.shallow and os.path.exists(ud.fullshallow) and unpack: 156 tmpdir = tempfile.mkdtemp(dir=d.getVar("DL_DIR")) 157 try: 158 runfetchcmd("tar -xzf %s" % ud.fullshallow, d, workdir=tmpdir) 159 self.process_submodules(ud, tmpdir, subfunc, d) 160 finally: 161 shutil.rmtree(tmpdir) 162 else: 163 self.process_submodules(ud, ud.clonedir, subfunc, d) 164 165 def need_update(self, ud, d): 166 if Git.need_update(self, ud, d): 167 return True 168 169 need_update_list = [] 170 def need_update_submodule(ud, url, module, modpath, workdir, d): 171 url += ";bareclone=1;nobranch=1" 172 173 try: 174 newfetch = Fetch([url], d, cache=False) 175 new_ud = newfetch.ud[url] 176 if new_ud.method.need_update(new_ud, d): 177 need_update_list.append(modpath) 178 except Exception as e: 179 logger.error('gitsm: submodule update check failed: %s %s' % (type(e).__name__, str(e))) 180 need_update_result = True 181 182 self.call_process_submodules(ud, d, not os.path.exists(ud.clonedir), need_update_submodule) 183 184 if need_update_list: 185 logger.debug('gitsm: Submodules requiring update: %s' % (' '.join(need_update_list))) 186 return True 187 188 return False 189 190 def download(self, ud, d): 191 def download_submodule(ud, url, module, modpath, workdir, d): 192 url += ";bareclone=1;nobranch=1" 193 194 # Is the following still needed? 195 #url += ";nocheckout=1" 196 197 try: 198 newfetch = Fetch([url], d, cache=False) 199 newfetch.download() 200 except Exception as e: 201 logger.error('gitsm: submodule download failed: %s %s' % (type(e).__name__, str(e))) 202 raise 203 204 Git.download(self, ud, d) 205 self.call_process_submodules(ud, d, self.need_update(ud, d), download_submodule) 206 207 def unpack(self, ud, destdir, d): 208 def unpack_submodules(ud, url, module, modpath, workdir, d): 209 url += ";bareclone=1;nobranch=1" 210 211 # Figure out where we clone over the bare submodules... 212 if ud.bareclone: 213 repo_conf = ud.destdir 214 else: 215 repo_conf = os.path.join(ud.destdir, '.git') 216 217 try: 218 newfetch = Fetch([url], d, cache=False) 219 # modpath is needed by unpack tracer to calculate submodule 220 # checkout dir 221 new_ud = newfetch.ud[url] 222 new_ud.modpath = modpath 223 newfetch.unpack(root=os.path.dirname(os.path.join(repo_conf, 'modules', module))) 224 except Exception as e: 225 logger.error('gitsm: submodule unpack failed: %s %s' % (type(e).__name__, str(e))) 226 raise 227 228 local_path = newfetch.localpath(url) 229 230 # Correct the submodule references to the local download version... 231 runfetchcmd("%(basecmd)s config submodule.%(module)s.url %(url)s" % {'basecmd': ud.basecmd, 'module': module, 'url' : local_path}, d, workdir=ud.destdir) 232 233 if ud.shallow: 234 runfetchcmd("%(basecmd)s config submodule.%(module)s.shallow true" % {'basecmd': ud.basecmd, 'module': module}, d, workdir=ud.destdir) 235 236 # Ensure the submodule repository is NOT set to bare, since we're checking it out... 237 try: 238 runfetchcmd("%s config core.bare false" % (ud.basecmd), d, quiet=True, workdir=os.path.join(repo_conf, 'modules', module)) 239 except: 240 logger.error("Unable to set git config core.bare to false for %s" % os.path.join(repo_conf, 'modules', module)) 241 raise 242 243 Git.unpack(self, ud, destdir, d) 244 245 ret = self.process_submodules(ud, ud.destdir, unpack_submodules, d) 246 247 if not ud.bareclone and ret: 248 cmdprefix = "" 249 # Avoid LFS smudging (replacing the LFS pointers with the actual content) when LFS shouldn't be used but git-lfs is installed. 250 if not self._need_lfs(ud): 251 cmdprefix = "GIT_LFS_SKIP_SMUDGE=1 " 252 runfetchcmd("%s%s submodule update --recursive --no-fetch" % (cmdprefix, ud.basecmd), d, quiet=True, workdir=ud.destdir) 253 def clean(self, ud, d): 254 def clean_submodule(ud, url, module, modpath, workdir, d): 255 url += ";bareclone=1;nobranch=1" 256 try: 257 newfetch = Fetch([url], d, cache=False) 258 newfetch.clean() 259 except Exception as e: 260 logger.warning('gitsm: submodule clean failed: %s %s' % (type(e).__name__, str(e))) 261 262 self.call_process_submodules(ud, d, True, clean_submodule) 263 264 # Clean top git dir 265 Git.clean(self, ud, d) 266 267 def implicit_urldata(self, ud, d): 268 import subprocess 269 270 urldata = [] 271 def add_submodule(ud, url, module, modpath, workdir, d): 272 url += ";bareclone=1;nobranch=1" 273 newfetch = Fetch([url], d, cache=False) 274 urldata.extend(newfetch.expanded_urldata()) 275 276 self.call_process_submodules(ud, d, ud.method.need_update(ud, d), add_submodule) 277 278 return urldata 279