1""" 2BitBake 'Fetch' git submodules implementation 3 4Inherits from and extends the Git fetcher to retrieve submodules of a git repository 5after cloning. 6 7SRC_URI = "gitsm://<see Git fetcher for syntax>" 8 9See the Git fetcher, git://, for usage documentation. 10 11NOTE: Switching a SRC_URI from "git://" to "gitsm://" requires a clean of your recipe. 12 13""" 14 15# Copyright (C) 2013 Richard Purdie 16# 17# SPDX-License-Identifier: GPL-2.0-only 18# 19 20import os 21import bb 22import copy 23from bb.fetch2.git import Git 24from bb.fetch2 import runfetchcmd 25from bb.fetch2 import logger 26from bb.fetch2 import Fetch 27from bb.fetch2 import BBFetchException 28 29class GitSM(Git): 30 def supports(self, ud, d): 31 """ 32 Check to see if a given url can be fetched with git. 33 """ 34 return ud.type in ['gitsm'] 35 36 def process_submodules(self, ud, workdir, function, d): 37 """ 38 Iterate over all of the submodules in this repository and execute 39 the 'function' for each of them. 40 """ 41 42 submodules = [] 43 paths = {} 44 revision = {} 45 uris = {} 46 subrevision = {} 47 48 def parse_gitmodules(gitmodules): 49 modules = {} 50 module = "" 51 for line in gitmodules.splitlines(): 52 if line.startswith('[submodule'): 53 module = line.split('"')[1] 54 modules[module] = {} 55 elif module and line.strip().startswith('path'): 56 path = line.split('=')[1].strip() 57 modules[module]['path'] = path 58 elif module and line.strip().startswith('url'): 59 url = line.split('=')[1].strip() 60 modules[module]['url'] = url 61 return modules 62 63 # Collect the defined submodules, and their attributes 64 for name in ud.names: 65 try: 66 gitmodules = runfetchcmd("%s show %s:.gitmodules" % (ud.basecmd, ud.revisions[name]), d, quiet=True, workdir=workdir) 67 except: 68 # No submodules to update 69 continue 70 71 for m, md in parse_gitmodules(gitmodules).items(): 72 try: 73 module_hash = runfetchcmd("%s ls-tree -z -d %s %s" % (ud.basecmd, ud.revisions[name], md['path']), d, quiet=True, workdir=workdir) 74 except: 75 # If the command fails, we don't have a valid file to check. If it doesn't 76 # fail -- it still might be a failure, see next check... 77 module_hash = "" 78 79 if not module_hash: 80 logger.debug(1, "submodule %s is defined, but is not initialized in the repository. Skipping", m) 81 continue 82 83 submodules.append(m) 84 paths[m] = md['path'] 85 revision[m] = ud.revisions[name] 86 uris[m] = md['url'] 87 subrevision[m] = module_hash.split()[2] 88 89 # Convert relative to absolute uri based on parent uri 90 if uris[m].startswith('..'): 91 newud = copy.copy(ud) 92 newud.path = os.path.realpath(os.path.join(newud.path, uris[m])) 93 uris[m] = Git._get_repo_url(self, newud) 94 95 for module in submodules: 96 # Translate the module url into a SRC_URI 97 98 if "://" in uris[module]: 99 # Properly formated URL already 100 proto = uris[module].split(':', 1)[0] 101 url = uris[module].replace('%s:' % proto, 'gitsm:', 1) 102 else: 103 if ":" in uris[module]: 104 # Most likely an SSH style reference 105 proto = "ssh" 106 if ":/" in uris[module]: 107 # Absolute reference, easy to convert.. 108 url = "gitsm://" + uris[module].replace(':/', '/', 1) 109 else: 110 # Relative reference, no way to know if this is right! 111 logger.warning("Submodule included by %s refers to relative ssh reference %s. References may fail if not absolute." % (ud.url, uris[module])) 112 url = "gitsm://" + uris[module].replace(':', '/', 1) 113 else: 114 # This has to be a file reference 115 proto = "file" 116 url = "gitsm://" + uris[module] 117 118 url += ';protocol=%s' % proto 119 url += ";name=%s" % module 120 url += ";subpath=%s" % module 121 122 ld = d.createCopy() 123 # Not necessary to set SRC_URI, since we're passing the URI to 124 # Fetch. 125 #ld.setVar('SRC_URI', url) 126 ld.setVar('SRCREV_%s' % module, subrevision[module]) 127 128 # Workaround for issues with SRCPV/SRCREV_FORMAT errors 129 # error refer to 'multiple' repositories. Only the repository 130 # in the original SRC_URI actually matters... 131 ld.setVar('SRCPV', d.getVar('SRCPV')) 132 ld.setVar('SRCREV_FORMAT', module) 133 134 function(ud, url, module, paths[module], ld) 135 136 return submodules != [] 137 138 def need_update(self, ud, d): 139 if Git.need_update(self, ud, d): 140 return True 141 142 try: 143 # Check for the nugget dropped by the download operation 144 known_srcrevs = runfetchcmd("%s config --get-all bitbake.srcrev" % \ 145 (ud.basecmd), d, workdir=ud.clonedir) 146 147 if ud.revisions[ud.names[0]] not in known_srcrevs.split(): 148 return True 149 except bb.fetch2.FetchError: 150 # No srcrev nuggets, so this is new and needs to be updated 151 return True 152 153 return False 154 155 def download(self, ud, d): 156 def download_submodule(ud, url, module, modpath, d): 157 url += ";bareclone=1;nobranch=1" 158 159 # Is the following still needed? 160 #url += ";nocheckout=1" 161 162 try: 163 newfetch = Fetch([url], d, cache=False) 164 newfetch.download() 165 # Drop a nugget to add each of the srcrevs we've fetched (used by need_update) 166 runfetchcmd("%s config --add bitbake.srcrev %s" % \ 167 (ud.basecmd, ud.revisions[ud.names[0]]), d, workdir=ud.clonedir) 168 except Exception as e: 169 logger.error('gitsm: submodule download failed: %s %s' % (type(e).__name__, str(e))) 170 raise 171 172 Git.download(self, ud, d) 173 self.process_submodules(ud, ud.clonedir, download_submodule, d) 174 175 def unpack(self, ud, destdir, d): 176 def unpack_submodules(ud, url, module, modpath, d): 177 url += ";bareclone=1;nobranch=1" 178 179 # Figure out where we clone over the bare submodules... 180 if ud.bareclone: 181 repo_conf = ud.destdir 182 else: 183 repo_conf = os.path.join(ud.destdir, '.git') 184 185 try: 186 newfetch = Fetch([url], d, cache=False) 187 newfetch.unpack(root=os.path.dirname(os.path.join(repo_conf, 'modules', module))) 188 except Exception as e: 189 logger.error('gitsm: submodule unpack failed: %s %s' % (type(e).__name__, str(e))) 190 raise 191 192 local_path = newfetch.localpath(url) 193 194 # Correct the submodule references to the local download version... 195 runfetchcmd("%(basecmd)s config submodule.%(module)s.url %(url)s" % {'basecmd': ud.basecmd, 'module': module, 'url' : local_path}, d, workdir=ud.destdir) 196 197 if ud.shallow: 198 runfetchcmd("%(basecmd)s config submodule.%(module)s.shallow true" % {'basecmd': ud.basecmd, 'module': module}, d, workdir=ud.destdir) 199 200 # Ensure the submodule repository is NOT set to bare, since we're checking it out... 201 try: 202 runfetchcmd("%s config core.bare false" % (ud.basecmd), d, quiet=True, workdir=os.path.join(repo_conf, 'modules', module)) 203 except: 204 logger.error("Unable to set git config core.bare to false for %s" % os.path.join(repo_conf, 'modules', module)) 205 raise 206 207 Git.unpack(self, ud, destdir, d) 208 209 ret = self.process_submodules(ud, ud.destdir, unpack_submodules, d) 210 211 if not ud.bareclone and ret: 212 # All submodules should already be downloaded and configured in the tree. This simply sets 213 # up the configuration and checks out the files. The main project config should remain 214 # unmodified, and no download from the internet should occur. 215 runfetchcmd("%s submodule update --recursive --no-fetch" % (ud.basecmd), d, quiet=True, workdir=ud.destdir) 216