1# 2# SPDX-License-Identifier: GPL-2.0-only 3# 4""" 5BitBake 'Fetch' NPM implementation 6 7The NPM fetcher is used to retrieve files from the npmjs repository 8 9Usage in the recipe: 10 11 SRC_URI = "npm://registry.npmjs.org/;name=${PN};version=${PV}" 12 Suported SRC_URI options are: 13 14 - name 15 - version 16 17 npm://registry.npmjs.org/${PN}/-/${PN}-${PV}.tgz would become npm://registry.npmjs.org;name=${PN};version=${PV} 18 The fetcher all triggers off the existence of ud.localpath. If that exists and has the ".done" stamp, its assumed the fetch is good/done 19 20""" 21 22import os 23import sys 24import urllib.request, urllib.parse, urllib.error 25import json 26import subprocess 27import signal 28import bb 29from bb.fetch2 import FetchMethod 30from bb.fetch2 import FetchError 31from bb.fetch2 import ChecksumError 32from bb.fetch2 import runfetchcmd 33from bb.fetch2 import logger 34from bb.fetch2 import UnpackError 35from bb.fetch2 import ParameterError 36 37def subprocess_setup(): 38 # Python installs a SIGPIPE handler by default. This is usually not what 39 # non-Python subprocesses expect. 40 # SIGPIPE errors are known issues with gzip/bash 41 signal.signal(signal.SIGPIPE, signal.SIG_DFL) 42 43class Npm(FetchMethod): 44 45 """Class to fetch urls via 'npm'""" 46 def init(self, d): 47 pass 48 49 def supports(self, ud, d): 50 """ 51 Check to see if a given url can be fetched with npm 52 """ 53 return ud.type in ['npm'] 54 55 def debug(self, msg): 56 logger.debug(1, "NpmFetch: %s", msg) 57 58 def clean(self, ud, d): 59 logger.debug(2, "Calling cleanup %s" % ud.pkgname) 60 bb.utils.remove(ud.localpath, False) 61 bb.utils.remove(ud.pkgdatadir, True) 62 bb.utils.remove(ud.fullmirror, False) 63 64 def urldata_init(self, ud, d): 65 """ 66 init NPM specific variable within url data 67 """ 68 if 'downloadfilename' in ud.parm: 69 ud.basename = ud.parm['downloadfilename'] 70 else: 71 ud.basename = os.path.basename(ud.path) 72 73 # can't call it ud.name otherwise fetcher base class will start doing sha1stuff 74 # TODO: find a way to get an sha1/sha256 manifest of pkg & all deps 75 ud.pkgname = ud.parm.get("name", None) 76 if not ud.pkgname: 77 raise ParameterError("NPM fetcher requires a name parameter", ud.url) 78 ud.version = ud.parm.get("version", None) 79 if not ud.version: 80 raise ParameterError("NPM fetcher requires a version parameter", ud.url) 81 ud.bbnpmmanifest = "%s-%s.deps.json" % (ud.pkgname, ud.version) 82 ud.bbnpmmanifest = ud.bbnpmmanifest.replace('/', '-') 83 ud.registry = "http://%s" % (ud.url.replace('npm://', '', 1).split(';'))[0] 84 prefixdir = "npm/%s" % ud.pkgname 85 ud.pkgdatadir = d.expand("${DL_DIR}/%s" % prefixdir) 86 if not os.path.exists(ud.pkgdatadir): 87 bb.utils.mkdirhier(ud.pkgdatadir) 88 ud.localpath = d.expand("${DL_DIR}/npm/%s" % ud.bbnpmmanifest) 89 90 self.basecmd = d.getVar("FETCHCMD_wget") or "/usr/bin/env wget -O -t 2 -T 30 -nv --passive-ftp --no-check-certificate " 91 ud.prefixdir = prefixdir 92 93 ud.write_tarballs = ((d.getVar("BB_GENERATE_MIRROR_TARBALLS") or "0") != "0") 94 mirrortarball = 'npm_%s-%s.tar.xz' % (ud.pkgname, ud.version) 95 mirrortarball = mirrortarball.replace('/', '-') 96 ud.fullmirror = os.path.join(d.getVar("DL_DIR"), mirrortarball) 97 ud.mirrortarballs = [mirrortarball] 98 99 def need_update(self, ud, d): 100 if os.path.exists(ud.localpath): 101 return False 102 return True 103 104 def _runpack(self, ud, d, pkgfullname: str, quiet=False) -> str: 105 """ 106 Runs npm pack on a full package name. 107 Returns the filename of the downloaded package 108 """ 109 bb.fetch2.check_network_access(d, pkgfullname, ud.registry) 110 dldir = d.getVar("DL_DIR") 111 dldir = os.path.join(dldir, ud.prefixdir) 112 113 command = "npm pack {} --registry {}".format(pkgfullname, ud.registry) 114 logger.debug(2, "Fetching {} using command '{}' in {}".format(pkgfullname, command, dldir)) 115 filename = runfetchcmd(command, d, quiet, workdir=dldir) 116 return filename.rstrip() 117 118 def _unpackdep(self, ud, pkg, data, destdir, dldir, d): 119 file = data[pkg]['tgz'] 120 logger.debug(2, "file to extract is %s" % file) 121 if file.endswith('.tgz') or file.endswith('.tar.gz') or file.endswith('.tar.Z'): 122 cmd = 'tar xz --strip 1 --no-same-owner --warning=no-unknown-keyword -f %s/%s' % (dldir, file) 123 else: 124 bb.fatal("NPM package %s downloaded not a tarball!" % file) 125 126 # Change to subdir before executing command 127 if not os.path.exists(destdir): 128 os.makedirs(destdir) 129 path = d.getVar('PATH') 130 if path: 131 cmd = "PATH=\"%s\" %s" % (path, cmd) 132 bb.note("Unpacking %s to %s/" % (file, destdir)) 133 ret = subprocess.call(cmd, preexec_fn=subprocess_setup, shell=True, cwd=destdir) 134 135 if ret != 0: 136 raise UnpackError("Unpack command %s failed with return value %s" % (cmd, ret), ud.url) 137 138 if 'deps' not in data[pkg]: 139 return 140 for dep in data[pkg]['deps']: 141 self._unpackdep(ud, dep, data[pkg]['deps'], "%s/node_modules/%s" % (destdir, dep), dldir, d) 142 143 144 def unpack(self, ud, destdir, d): 145 dldir = d.getVar("DL_DIR") 146 with open("%s/npm/%s" % (dldir, ud.bbnpmmanifest)) as datafile: 147 workobj = json.load(datafile) 148 dldir = "%s/%s" % (os.path.dirname(ud.localpath), ud.pkgname) 149 150 if 'subdir' in ud.parm: 151 unpackdir = '%s/%s' % (destdir, ud.parm.get('subdir')) 152 else: 153 unpackdir = '%s/npmpkg' % destdir 154 155 self._unpackdep(ud, ud.pkgname, workobj, unpackdir, dldir, d) 156 157 def _parse_view(self, output): 158 ''' 159 Parse the output of npm view --json; the last JSON result 160 is assumed to be the one that we're interested in. 161 ''' 162 pdata = json.loads(output); 163 try: 164 return pdata[-1] 165 except: 166 return pdata 167 168 def _getdependencies(self, pkg, data, version, d, ud, optional=False, fetchedlist=None): 169 if fetchedlist is None: 170 fetchedlist = [] 171 pkgfullname = pkg 172 if version != '*' and not '/' in version: 173 pkgfullname += "@'%s'" % version 174 if pkgfullname in fetchedlist: 175 return 176 177 logger.debug(2, "Calling getdeps on %s" % pkg) 178 fetchcmd = "npm view %s --json --registry %s" % (pkgfullname, ud.registry) 179 output = runfetchcmd(fetchcmd, d, True) 180 pdata = self._parse_view(output) 181 if not pdata: 182 raise FetchError("The command '%s' returned no output" % fetchcmd) 183 if optional: 184 pkg_os = pdata.get('os', None) 185 if pkg_os: 186 if not isinstance(pkg_os, list): 187 pkg_os = [pkg_os] 188 blacklist = False 189 for item in pkg_os: 190 if item.startswith('!'): 191 blacklist = True 192 break 193 if (not blacklist and 'linux' not in pkg_os) or '!linux' in pkg_os: 194 logger.debug(2, "Skipping %s since it's incompatible with Linux" % pkg) 195 return 196 filename = self._runpack(ud, d, pkgfullname) 197 data[pkg] = {} 198 data[pkg]['tgz'] = filename 199 fetchedlist.append(pkgfullname) 200 201 dependencies = pdata.get('dependencies', {}) 202 optionalDependencies = pdata.get('optionalDependencies', {}) 203 dependencies.update(optionalDependencies) 204 depsfound = {} 205 optdepsfound = {} 206 data[pkg]['deps'] = {} 207 for dep in dependencies: 208 if dep in optionalDependencies: 209 optdepsfound[dep] = dependencies[dep] 210 else: 211 depsfound[dep] = dependencies[dep] 212 for dep, version in optdepsfound.items(): 213 self._getdependencies(dep, data[pkg]['deps'], version, d, ud, optional=True, fetchedlist=fetchedlist) 214 for dep, version in depsfound.items(): 215 self._getdependencies(dep, data[pkg]['deps'], version, d, ud, fetchedlist=fetchedlist) 216 217 def _getshrinkeddependencies(self, pkg, data, version, d, ud, lockdown, manifest, toplevel=True): 218 logger.debug(2, "NPM shrinkwrap file is %s" % data) 219 if toplevel: 220 name = data.get('name', None) 221 if name and name != pkg: 222 for obj in data.get('dependencies', []): 223 if obj == pkg: 224 self._getshrinkeddependencies(obj, data['dependencies'][obj], data['dependencies'][obj]['version'], d, ud, lockdown, manifest, False) 225 return 226 227 pkgnameWithVersion = "{}@{}".format(pkg, version) 228 logger.debug(2, "Get dependencies for {}".format(pkgnameWithVersion)) 229 filename = self._runpack(ud, d, pkgnameWithVersion) 230 manifest[pkg] = {} 231 manifest[pkg]['tgz'] = filename 232 manifest[pkg]['deps'] = {} 233 234 if pkg in lockdown: 235 sha1_expected = lockdown[pkg][version] 236 sha1_data = bb.utils.sha1_file("npm/%s/%s" % (ud.pkgname, manifest[pkg]['tgz'])) 237 if sha1_expected != sha1_data: 238 msg = "\nFile: '%s' has %s checksum %s when %s was expected" % (manifest[pkg]['tgz'], 'sha1', sha1_data, sha1_expected) 239 raise ChecksumError('Checksum mismatch!%s' % msg) 240 else: 241 logger.debug(2, "No lockdown data for %s@%s" % (pkg, version)) 242 243 if 'dependencies' in data: 244 for obj in data['dependencies']: 245 logger.debug(2, "Found dep is %s" % str(obj)) 246 self._getshrinkeddependencies(obj, data['dependencies'][obj], data['dependencies'][obj]['version'], d, ud, lockdown, manifest[pkg]['deps'], False) 247 248 def download(self, ud, d): 249 """Fetch url""" 250 jsondepobj = {} 251 shrinkobj = {} 252 lockdown = {} 253 254 if not os.listdir(ud.pkgdatadir) and os.path.exists(ud.fullmirror): 255 dest = d.getVar("DL_DIR") 256 bb.utils.mkdirhier(dest) 257 runfetchcmd("tar -xJf %s" % (ud.fullmirror), d, workdir=dest) 258 return 259 260 if ud.parm.get("noverify", None) != '1': 261 shwrf = d.getVar('NPM_SHRINKWRAP') 262 logger.debug(2, "NPM shrinkwrap file is %s" % shwrf) 263 if shwrf: 264 try: 265 with open(shwrf) as datafile: 266 shrinkobj = json.load(datafile) 267 except Exception as e: 268 raise FetchError('Error loading NPM_SHRINKWRAP file "%s" for %s: %s' % (shwrf, ud.pkgname, str(e))) 269 elif not ud.ignore_checksums: 270 logger.warning('Missing shrinkwrap file in NPM_SHRINKWRAP for %s, this will lead to unreliable builds!' % ud.pkgname) 271 lckdf = d.getVar('NPM_LOCKDOWN') 272 logger.debug(2, "NPM lockdown file is %s" % lckdf) 273 if lckdf: 274 try: 275 with open(lckdf) as datafile: 276 lockdown = json.load(datafile) 277 except Exception as e: 278 raise FetchError('Error loading NPM_LOCKDOWN file "%s" for %s: %s' % (lckdf, ud.pkgname, str(e))) 279 elif not ud.ignore_checksums: 280 logger.warning('Missing lockdown file in NPM_LOCKDOWN for %s, this will lead to unreproducible builds!' % ud.pkgname) 281 282 if ('name' not in shrinkobj): 283 self._getdependencies(ud.pkgname, jsondepobj, ud.version, d, ud) 284 else: 285 self._getshrinkeddependencies(ud.pkgname, shrinkobj, ud.version, d, ud, lockdown, jsondepobj) 286 287 with open(ud.localpath, 'w') as outfile: 288 json.dump(jsondepobj, outfile) 289 290 def build_mirror_data(self, ud, d): 291 # Generate a mirror tarball if needed 292 if ud.write_tarballs and not os.path.exists(ud.fullmirror): 293 # it's possible that this symlink points to read-only filesystem with PREMIRROR 294 if os.path.islink(ud.fullmirror): 295 os.unlink(ud.fullmirror) 296 297 dldir = d.getVar("DL_DIR") 298 logger.info("Creating tarball of npm data") 299 runfetchcmd("tar -cJf %s npm/%s npm/%s" % (ud.fullmirror, ud.bbnpmmanifest, ud.pkgname), d, 300 workdir=dldir) 301 runfetchcmd("touch %s.done" % (ud.fullmirror), d, workdir=dldir) 302