1# ex:ts=4:sw=4:sts=4:et 2# -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- 3""" 4BitBake 'Fetch' NPM implementation 5 6The NPM fetcher is used to retrieve files from the npmjs repository 7 8Usage in the recipe: 9 10 SRC_URI = "npm://registry.npmjs.org/;name=${PN};version=${PV}" 11 Suported SRC_URI options are: 12 13 - name 14 - version 15 16 npm://registry.npmjs.org/${PN}/-/${PN}-${PV}.tgz would become npm://registry.npmjs.org;name=${PN};version=${PV} 17 The fetcher all triggers off the existence of ud.localpath. If that exists and has the ".done" stamp, its assumed the fetch is good/done 18 19""" 20 21import os 22import sys 23import urllib.request, urllib.parse, urllib.error 24import json 25import subprocess 26import signal 27import bb 28from bb.fetch2 import FetchMethod 29from bb.fetch2 import FetchError 30from bb.fetch2 import ChecksumError 31from bb.fetch2 import runfetchcmd 32from bb.fetch2 import logger 33from bb.fetch2 import UnpackError 34from bb.fetch2 import ParameterError 35from distutils import spawn 36 37def subprocess_setup(): 38 # Python installs a SIGPIPE handler by default. This is usually not what 39 # non-Python subprocesses expect. 40 # SIGPIPE errors are known issues with gzip/bash 41 signal.signal(signal.SIGPIPE, signal.SIG_DFL) 42 43class Npm(FetchMethod): 44 45 """Class to fetch urls via 'npm'""" 46 def init(self, d): 47 pass 48 49 def supports(self, ud, d): 50 """ 51 Check to see if a given url can be fetched with npm 52 """ 53 return ud.type in ['npm'] 54 55 def debug(self, msg): 56 logger.debug(1, "NpmFetch: %s", msg) 57 58 def clean(self, ud, d): 59 logger.debug(2, "Calling cleanup %s" % ud.pkgname) 60 bb.utils.remove(ud.localpath, False) 61 bb.utils.remove(ud.pkgdatadir, True) 62 bb.utils.remove(ud.fullmirror, False) 63 64 def urldata_init(self, ud, d): 65 """ 66 init NPM specific variable within url data 67 """ 68 if 'downloadfilename' in ud.parm: 69 ud.basename = ud.parm['downloadfilename'] 70 else: 71 ud.basename = os.path.basename(ud.path) 72 73 # can't call it ud.name otherwise fetcher base class will start doing sha1stuff 74 # TODO: find a way to get an sha1/sha256 manifest of pkg & all deps 75 ud.pkgname = ud.parm.get("name", None) 76 if not ud.pkgname: 77 raise ParameterError("NPM fetcher requires a name parameter", ud.url) 78 ud.version = ud.parm.get("version", None) 79 if not ud.version: 80 raise ParameterError("NPM fetcher requires a version parameter", ud.url) 81 ud.bbnpmmanifest = "%s-%s.deps.json" % (ud.pkgname, ud.version) 82 ud.bbnpmmanifest = ud.bbnpmmanifest.replace('/', '-') 83 ud.registry = "http://%s" % (ud.url.replace('npm://', '', 1).split(';'))[0] 84 prefixdir = "npm/%s" % ud.pkgname 85 ud.pkgdatadir = d.expand("${DL_DIR}/%s" % prefixdir) 86 if not os.path.exists(ud.pkgdatadir): 87 bb.utils.mkdirhier(ud.pkgdatadir) 88 ud.localpath = d.expand("${DL_DIR}/npm/%s" % ud.bbnpmmanifest) 89 90 self.basecmd = d.getVar("FETCHCMD_wget") or "/usr/bin/env wget -O -t 2 -T 30 -nv --passive-ftp --no-check-certificate " 91 ud.prefixdir = prefixdir 92 93 ud.write_tarballs = ((d.getVar("BB_GENERATE_MIRROR_TARBALLS") or "0") != "0") 94 mirrortarball = 'npm_%s-%s.tar.xz' % (ud.pkgname, ud.version) 95 mirrortarball = mirrortarball.replace('/', '-') 96 ud.fullmirror = os.path.join(d.getVar("DL_DIR"), mirrortarball) 97 ud.mirrortarballs = [mirrortarball] 98 99 def need_update(self, ud, d): 100 if os.path.exists(ud.localpath): 101 return False 102 return True 103 104 def _runwget(self, ud, d, command, quiet): 105 logger.debug(2, "Fetching %s using command '%s'" % (ud.url, command)) 106 bb.fetch2.check_network_access(d, command, ud.url) 107 dldir = d.getVar("DL_DIR") 108 runfetchcmd(command, d, quiet, workdir=dldir) 109 110 def _unpackdep(self, ud, pkg, data, destdir, dldir, d): 111 file = data[pkg]['tgz'] 112 logger.debug(2, "file to extract is %s" % file) 113 if file.endswith('.tgz') or file.endswith('.tar.gz') or file.endswith('.tar.Z'): 114 cmd = 'tar xz --strip 1 --no-same-owner --warning=no-unknown-keyword -f %s/%s' % (dldir, file) 115 else: 116 bb.fatal("NPM package %s downloaded not a tarball!" % file) 117 118 # Change to subdir before executing command 119 if not os.path.exists(destdir): 120 os.makedirs(destdir) 121 path = d.getVar('PATH') 122 if path: 123 cmd = "PATH=\"%s\" %s" % (path, cmd) 124 bb.note("Unpacking %s to %s/" % (file, destdir)) 125 ret = subprocess.call(cmd, preexec_fn=subprocess_setup, shell=True, cwd=destdir) 126 127 if ret != 0: 128 raise UnpackError("Unpack command %s failed with return value %s" % (cmd, ret), ud.url) 129 130 if 'deps' not in data[pkg]: 131 return 132 for dep in data[pkg]['deps']: 133 self._unpackdep(ud, dep, data[pkg]['deps'], "%s/node_modules/%s" % (destdir, dep), dldir, d) 134 135 136 def unpack(self, ud, destdir, d): 137 dldir = d.getVar("DL_DIR") 138 with open("%s/npm/%s" % (dldir, ud.bbnpmmanifest)) as datafile: 139 workobj = json.load(datafile) 140 dldir = "%s/%s" % (os.path.dirname(ud.localpath), ud.pkgname) 141 142 if 'subdir' in ud.parm: 143 unpackdir = '%s/%s' % (destdir, ud.parm.get('subdir')) 144 else: 145 unpackdir = '%s/npmpkg' % destdir 146 147 self._unpackdep(ud, ud.pkgname, workobj, unpackdir, dldir, d) 148 149 def _parse_view(self, output): 150 ''' 151 Parse the output of npm view --json; the last JSON result 152 is assumed to be the one that we're interested in. 153 ''' 154 pdata = None 155 outdeps = {} 156 datalines = [] 157 bracelevel = 0 158 for line in output.splitlines(): 159 if bracelevel: 160 datalines.append(line) 161 elif '{' in line: 162 datalines = [] 163 datalines.append(line) 164 bracelevel = bracelevel + line.count('{') - line.count('}') 165 if datalines: 166 pdata = json.loads('\n'.join(datalines)) 167 return pdata 168 169 def _getdependencies(self, pkg, data, version, d, ud, optional=False, fetchedlist=None): 170 if fetchedlist is None: 171 fetchedlist = [] 172 pkgfullname = pkg 173 if version != '*' and not '/' in version: 174 pkgfullname += "@'%s'" % version 175 logger.debug(2, "Calling getdeps on %s" % pkg) 176 fetchcmd = "npm view %s --json --registry %s" % (pkgfullname, ud.registry) 177 output = runfetchcmd(fetchcmd, d, True) 178 pdata = self._parse_view(output) 179 if not pdata: 180 raise FetchError("The command '%s' returned no output" % fetchcmd) 181 if optional: 182 pkg_os = pdata.get('os', None) 183 if pkg_os: 184 if not isinstance(pkg_os, list): 185 pkg_os = [pkg_os] 186 blacklist = False 187 for item in pkg_os: 188 if item.startswith('!'): 189 blacklist = True 190 break 191 if (not blacklist and 'linux' not in pkg_os) or '!linux' in pkg_os: 192 logger.debug(2, "Skipping %s since it's incompatible with Linux" % pkg) 193 return 194 #logger.debug(2, "Output URL is %s - %s - %s" % (ud.basepath, ud.basename, ud.localfile)) 195 outputurl = pdata['dist']['tarball'] 196 data[pkg] = {} 197 data[pkg]['tgz'] = os.path.basename(outputurl) 198 if outputurl in fetchedlist: 199 return 200 201 self._runwget(ud, d, "%s --directory-prefix=%s %s" % (self.basecmd, ud.prefixdir, outputurl), False) 202 fetchedlist.append(outputurl) 203 204 dependencies = pdata.get('dependencies', {}) 205 optionalDependencies = pdata.get('optionalDependencies', {}) 206 dependencies.update(optionalDependencies) 207 depsfound = {} 208 optdepsfound = {} 209 data[pkg]['deps'] = {} 210 for dep in dependencies: 211 if dep in optionalDependencies: 212 optdepsfound[dep] = dependencies[dep] 213 else: 214 depsfound[dep] = dependencies[dep] 215 for dep, version in optdepsfound.items(): 216 self._getdependencies(dep, data[pkg]['deps'], version, d, ud, optional=True, fetchedlist=fetchedlist) 217 for dep, version in depsfound.items(): 218 self._getdependencies(dep, data[pkg]['deps'], version, d, ud, fetchedlist=fetchedlist) 219 220 def _getshrinkeddependencies(self, pkg, data, version, d, ud, lockdown, manifest, toplevel=True): 221 logger.debug(2, "NPM shrinkwrap file is %s" % data) 222 if toplevel: 223 name = data.get('name', None) 224 if name and name != pkg: 225 for obj in data.get('dependencies', []): 226 if obj == pkg: 227 self._getshrinkeddependencies(obj, data['dependencies'][obj], data['dependencies'][obj]['version'], d, ud, lockdown, manifest, False) 228 return 229 outputurl = "invalid" 230 if ('resolved' not in data) or (not data['resolved'].startswith('http')): 231 # will be the case for ${PN} 232 fetchcmd = "npm view %s@%s dist.tarball --registry %s" % (pkg, version, ud.registry) 233 logger.debug(2, "Found this matching URL: %s" % str(fetchcmd)) 234 outputurl = runfetchcmd(fetchcmd, d, True) 235 else: 236 outputurl = data['resolved'] 237 self._runwget(ud, d, "%s --directory-prefix=%s %s" % (self.basecmd, ud.prefixdir, outputurl), False) 238 manifest[pkg] = {} 239 manifest[pkg]['tgz'] = os.path.basename(outputurl).rstrip() 240 manifest[pkg]['deps'] = {} 241 242 if pkg in lockdown: 243 sha1_expected = lockdown[pkg][version] 244 sha1_data = bb.utils.sha1_file("npm/%s/%s" % (ud.pkgname, manifest[pkg]['tgz'])) 245 if sha1_expected != sha1_data: 246 msg = "\nFile: '%s' has %s checksum %s when %s was expected" % (manifest[pkg]['tgz'], 'sha1', sha1_data, sha1_expected) 247 raise ChecksumError('Checksum mismatch!%s' % msg) 248 else: 249 logger.debug(2, "No lockdown data for %s@%s" % (pkg, version)) 250 251 if 'dependencies' in data: 252 for obj in data['dependencies']: 253 logger.debug(2, "Found dep is %s" % str(obj)) 254 self._getshrinkeddependencies(obj, data['dependencies'][obj], data['dependencies'][obj]['version'], d, ud, lockdown, manifest[pkg]['deps'], False) 255 256 def download(self, ud, d): 257 """Fetch url""" 258 jsondepobj = {} 259 shrinkobj = {} 260 lockdown = {} 261 262 if not os.listdir(ud.pkgdatadir) and os.path.exists(ud.fullmirror): 263 dest = d.getVar("DL_DIR") 264 bb.utils.mkdirhier(dest) 265 runfetchcmd("tar -xJf %s" % (ud.fullmirror), d, workdir=dest) 266 return 267 268 if ud.parm.get("noverify", None) != '1': 269 shwrf = d.getVar('NPM_SHRINKWRAP') 270 logger.debug(2, "NPM shrinkwrap file is %s" % shwrf) 271 if shwrf: 272 try: 273 with open(shwrf) as datafile: 274 shrinkobj = json.load(datafile) 275 except Exception as e: 276 raise FetchError('Error loading NPM_SHRINKWRAP file "%s" for %s: %s' % (shwrf, ud.pkgname, str(e))) 277 elif not ud.ignore_checksums: 278 logger.warning('Missing shrinkwrap file in NPM_SHRINKWRAP for %s, this will lead to unreliable builds!' % ud.pkgname) 279 lckdf = d.getVar('NPM_LOCKDOWN') 280 logger.debug(2, "NPM lockdown file is %s" % lckdf) 281 if lckdf: 282 try: 283 with open(lckdf) as datafile: 284 lockdown = json.load(datafile) 285 except Exception as e: 286 raise FetchError('Error loading NPM_LOCKDOWN file "%s" for %s: %s' % (lckdf, ud.pkgname, str(e))) 287 elif not ud.ignore_checksums: 288 logger.warning('Missing lockdown file in NPM_LOCKDOWN for %s, this will lead to unreproducible builds!' % ud.pkgname) 289 290 if ('name' not in shrinkobj): 291 self._getdependencies(ud.pkgname, jsondepobj, ud.version, d, ud) 292 else: 293 self._getshrinkeddependencies(ud.pkgname, shrinkobj, ud.version, d, ud, lockdown, jsondepobj) 294 295 with open(ud.localpath, 'w') as outfile: 296 json.dump(jsondepobj, outfile) 297 298 def build_mirror_data(self, ud, d): 299 # Generate a mirror tarball if needed 300 if ud.write_tarballs and not os.path.exists(ud.fullmirror): 301 # it's possible that this symlink points to read-only filesystem with PREMIRROR 302 if os.path.islink(ud.fullmirror): 303 os.unlink(ud.fullmirror) 304 305 dldir = d.getVar("DL_DIR") 306 logger.info("Creating tarball of npm data") 307 runfetchcmd("tar -cJf %s npm/%s npm/%s" % (ud.fullmirror, ud.bbnpmmanifest, ud.pkgname), d, 308 workdir=dldir) 309 runfetchcmd("touch %s.done" % (ud.fullmirror), d, workdir=dldir) 310