1""" 2BitBake 'Fetch' implementation for Go modules 3 4The gomod/gomodgit fetchers are used to download Go modules to the module cache 5from a module proxy or directly from a version control repository. 6 7Example SRC_URI: 8 9SRC_URI += "gomod://golang.org/x/net;version=v0.9.0;sha256sum=..." 10SRC_URI += "gomodgit://golang.org/x/net;version=v0.9.0;repo=go.googlesource.com/net;srcrev=..." 11 12Required SRC_URI parameters: 13 14- version 15 The version of the module. 16 17Optional SRC_URI parameters: 18 19- mod 20 Fetch and unpack the go.mod file only instead of the complete module. 21 The go command may need to download go.mod files for many different modules 22 when computing the build list, and go.mod files are much smaller than 23 module zip files. 24 The default is "0", set mod=1 for the go.mod file only. 25 26- sha256sum 27 The checksum of the module zip file, or the go.mod file in case of fetching 28 only the go.mod file. Alternatively, set the SRC_URI varible flag for 29 "module@version.sha256sum". 30 31- protocol 32 The method used when fetching directly from a version control repository. 33 The default is "https" for git. 34 35- repo 36 The URL when fetching directly from a version control repository. Required 37 when the URL is different from the module path. 38 39- srcrev 40 The revision identifier used when fetching directly from a version control 41 repository. Alternatively, set the SRCREV varible for "module@version". 42 43- subdir 44 The module subdirectory when fetching directly from a version control 45 repository. Required when the module is not located in the root of the 46 repository. 47 48Related variables: 49 50- GO_MOD_PROXY 51 The module proxy used by the fetcher. 52 53- GO_MOD_CACHE_DIR 54 The directory where the module cache is located. 55 This must match the exported GOMODCACHE variable for the go command to find 56 the downloaded modules. 57 58See the Go modules reference, https://go.dev/ref/mod, for more information 59about the module cache, module proxies and version control systems. 60""" 61 62import hashlib 63import os 64import re 65import shutil 66import subprocess 67import zipfile 68 69import bb 70from bb.fetch2 import FetchError 71from bb.fetch2 import MissingParameterError 72from bb.fetch2 import runfetchcmd 73from bb.fetch2 import subprocess_setup 74from bb.fetch2.git import Git 75from bb.fetch2.wget import Wget 76 77 78def escape(path): 79 """Escape capital letters using exclamation points.""" 80 return re.sub(r'([A-Z])', lambda m: '!' + m.group(1).lower(), path) 81 82 83class GoMod(Wget): 84 """Class to fetch Go modules from a Go module proxy via wget""" 85 86 def supports(self, ud, d): 87 """Check to see if a given URL is for this fetcher.""" 88 return ud.type == 'gomod' 89 90 def urldata_init(self, ud, d): 91 """Set up to download the module from the module proxy. 92 93 Set up to download the module zip file to the module cache directory 94 and unpack the go.mod file (unless downloading only the go.mod file): 95 96 cache/download/<module>/@v/<version>.zip: The module zip file. 97 cache/download/<module>/@v/<version>.mod: The go.mod file. 98 """ 99 100 proxy = d.getVar('GO_MOD_PROXY') or 'proxy.golang.org' 101 moddir = d.getVar('GO_MOD_CACHE_DIR') or 'pkg/mod' 102 103 if 'version' not in ud.parm: 104 raise MissingParameterError('version', ud.url) 105 106 module = ud.host 107 if ud.path != '/': 108 module += ud.path 109 ud.parm['module'] = module 110 version = ud.parm['version'] 111 112 # Set URL and filename for wget download 113 if ud.parm.get('mod', '0') == '1': 114 ext = '.mod' 115 else: 116 ext = '.zip' 117 path = escape(f"{module}/@v/{version}{ext}") 118 ud.url = bb.fetch2.encodeurl( 119 ('https', proxy, '/' + path, None, None, None)) 120 ud.parm['downloadfilename'] = f"{module.replace('/', '.')}@{version}{ext}" 121 122 # Set name for checksum verification 123 ud.parm['name'] = f"{module}@{version}" 124 125 # Set path for unpack 126 ud.parm['unpackpath'] = os.path.join(moddir, 'cache/download', path) 127 128 super().urldata_init(ud, d) 129 130 def unpack(self, ud, rootdir, d): 131 """Unpack the module in the module cache.""" 132 133 # Unpack the module zip file or go.mod file 134 unpackpath = os.path.join(rootdir, ud.parm['unpackpath']) 135 unpackdir = os.path.dirname(unpackpath) 136 bb.utils.mkdirhier(unpackdir) 137 ud.unpack_tracer.unpack("file-copy", unpackdir) 138 cmd = f"cp {ud.localpath} {unpackpath}" 139 path = d.getVar('PATH') 140 if path: 141 cmd = f"PATH={path} {cmd}" 142 name = os.path.basename(unpackpath) 143 bb.note(f"Unpacking {name} to {unpackdir}/") 144 subprocess.check_call(cmd, shell=True, preexec_fn=subprocess_setup) 145 146 if name.endswith('.zip'): 147 # Unpack the go.mod file from the zip file 148 module = ud.parm['module'] 149 name = name.rsplit('.', 1)[0] + '.mod' 150 bb.note(f"Unpacking {name} to {unpackdir}/") 151 with zipfile.ZipFile(ud.localpath) as zf: 152 with open(os.path.join(unpackdir, name), mode='wb') as mf: 153 try: 154 f = module + '@' + ud.parm['version'] + '/go.mod' 155 shutil.copyfileobj(zf.open(f), mf) 156 except KeyError: 157 # If the module does not have a go.mod file, synthesize 158 # one containing only a module statement. 159 mf.write(f'module {module}\n'.encode()) 160 161 162class GoModGit(Git): 163 """Class to fetch Go modules directly from a git repository""" 164 165 def supports(self, ud, d): 166 """Check to see if a given URL is for this fetcher.""" 167 return ud.type == 'gomodgit' 168 169 def urldata_init(self, ud, d): 170 """Set up to download the module from the git repository. 171 172 Set up to download the git repository to the module cache directory and 173 unpack the module zip file and the go.mod file: 174 175 cache/vcs/<hash>: The bare git repository. 176 cache/download/<module>/@v/<version>.zip: The module zip file. 177 cache/download/<module>/@v/<version>.mod: The go.mod file. 178 """ 179 180 moddir = d.getVar('GO_MOD_CACHE_DIR') or 'pkg/mod' 181 182 if 'version' not in ud.parm: 183 raise MissingParameterError('version', ud.url) 184 185 module = ud.host 186 if ud.path != '/': 187 module += ud.path 188 ud.parm['module'] = module 189 190 # Set host, path and srcrev for git download 191 if 'repo' in ud.parm: 192 repo = ud.parm['repo'] 193 idx = repo.find('/') 194 if idx != -1: 195 ud.host = repo[:idx] 196 ud.path = repo[idx:] 197 else: 198 ud.host = repo 199 ud.path = '' 200 if 'protocol' not in ud.parm: 201 ud.parm['protocol'] = 'https' 202 ud.name = f"{module}@{ud.parm['version']}" 203 srcrev = d.getVar('SRCREV_' + ud.name) 204 if srcrev: 205 if 'srcrev' not in ud.parm: 206 ud.parm['srcrev'] = srcrev 207 else: 208 if 'srcrev' in ud.parm: 209 d.setVar('SRCREV_' + ud.name, ud.parm['srcrev']) 210 if 'branch' not in ud.parm: 211 ud.parm['nobranch'] = '1' 212 213 # Set subpath, subdir and bareclone for git unpack 214 if 'subdir' in ud.parm: 215 ud.parm['subpath'] = ud.parm['subdir'] 216 key = f"git3:{ud.parm['protocol']}://{ud.host}{ud.path}".encode() 217 ud.parm['key'] = key 218 ud.parm['subdir'] = os.path.join(moddir, 'cache/vcs', 219 hashlib.sha256(key).hexdigest()) 220 ud.parm['bareclone'] = '1' 221 222 super().urldata_init(ud, d) 223 224 def unpack(self, ud, rootdir, d): 225 """Unpack the module in the module cache.""" 226 227 # Unpack the bare git repository 228 super().unpack(ud, rootdir, d) 229 230 moddir = d.getVar('GO_MOD_CACHE_DIR') or 'pkg/mod' 231 232 # Create the info file 233 module = ud.parm['module'] 234 repodir = os.path.join(rootdir, ud.parm['subdir']) 235 with open(repodir + '.info', 'wb') as f: 236 f.write(ud.parm['key']) 237 238 # Unpack the go.mod file from the repository 239 unpackdir = os.path.join(rootdir, moddir, 'cache/download', 240 escape(module), '@v') 241 bb.utils.mkdirhier(unpackdir) 242 srcrev = ud.parm['srcrev'] 243 version = ud.parm['version'] 244 escaped_version = escape(version) 245 cmd = f"git ls-tree -r --name-only '{srcrev}'" 246 if 'subpath' in ud.parm: 247 cmd += f" '{ud.parm['subpath']}'" 248 files = runfetchcmd(cmd, d, workdir=repodir).split() 249 name = escaped_version + '.mod' 250 bb.note(f"Unpacking {name} to {unpackdir}/") 251 with open(os.path.join(unpackdir, name), mode='wb') as mf: 252 f = 'go.mod' 253 if 'subpath' in ud.parm: 254 f = os.path.join(ud.parm['subpath'], f) 255 if f in files: 256 cmd = ['git', 'cat-file', 'blob', srcrev + ':' + f] 257 subprocess.check_call(cmd, stdout=mf, cwd=repodir, 258 preexec_fn=subprocess_setup) 259 else: 260 # If the module does not have a go.mod file, synthesize one 261 # containing only a module statement. 262 mf.write(f'module {module}\n'.encode()) 263 264 # Synthesize the module zip file from the repository 265 name = escaped_version + '.zip' 266 bb.note(f"Unpacking {name} to {unpackdir}/") 267 with zipfile.ZipFile(os.path.join(unpackdir, name), mode='w') as zf: 268 prefix = module + '@' + version + '/' 269 for f in files: 270 cmd = ['git', 'cat-file', 'blob', srcrev + ':' + f] 271 data = subprocess.check_output(cmd, cwd=repodir, 272 preexec_fn=subprocess_setup) 273 zf.writestr(prefix + f, data) 274