1# ex:ts=4:sw=4:sts=4:et 2# -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- 3""" 4BitBake 'Fetch' NPM implementation 5 6The NPM fetcher is used to retrieve files from the npmjs repository 7 8Usage in the recipe: 9 10 SRC_URI = "npm://registry.npmjs.org/;name=${PN};version=${PV}" 11 Suported SRC_URI options are: 12 13 - name 14 - version 15 16 npm://registry.npmjs.org/${PN}/-/${PN}-${PV}.tgz would become npm://registry.npmjs.org;name=${PN};version=${PV} 17 The fetcher all triggers off the existence of ud.localpath. If that exists and has the ".done" stamp, its assumed the fetch is good/done 18 19""" 20 21import os 22import sys 23import urllib.request, urllib.parse, urllib.error 24import json 25import subprocess 26import signal 27import bb 28from bb.fetch2 import FetchMethod 29from bb.fetch2 import FetchError 30from bb.fetch2 import ChecksumError 31from bb.fetch2 import runfetchcmd 32from bb.fetch2 import logger 33from bb.fetch2 import UnpackError 34from bb.fetch2 import ParameterError 35 36def subprocess_setup(): 37 # Python installs a SIGPIPE handler by default. This is usually not what 38 # non-Python subprocesses expect. 39 # SIGPIPE errors are known issues with gzip/bash 40 signal.signal(signal.SIGPIPE, signal.SIG_DFL) 41 42class Npm(FetchMethod): 43 44 """Class to fetch urls via 'npm'""" 45 def init(self, d): 46 pass 47 48 def supports(self, ud, d): 49 """ 50 Check to see if a given url can be fetched with npm 51 """ 52 return ud.type in ['npm'] 53 54 def debug(self, msg): 55 logger.debug(1, "NpmFetch: %s", msg) 56 57 def clean(self, ud, d): 58 logger.debug(2, "Calling cleanup %s" % ud.pkgname) 59 bb.utils.remove(ud.localpath, False) 60 bb.utils.remove(ud.pkgdatadir, True) 61 bb.utils.remove(ud.fullmirror, False) 62 63 def urldata_init(self, ud, d): 64 """ 65 init NPM specific variable within url data 66 """ 67 if 'downloadfilename' in ud.parm: 68 ud.basename = ud.parm['downloadfilename'] 69 else: 70 ud.basename = os.path.basename(ud.path) 71 72 # can't call it ud.name otherwise fetcher base class will start doing sha1stuff 73 # TODO: find a way to get an sha1/sha256 manifest of pkg & all deps 74 ud.pkgname = ud.parm.get("name", None) 75 if not ud.pkgname: 76 raise ParameterError("NPM fetcher requires a name parameter", ud.url) 77 ud.version = ud.parm.get("version", None) 78 if not ud.version: 79 raise ParameterError("NPM fetcher requires a version parameter", ud.url) 80 ud.bbnpmmanifest = "%s-%s.deps.json" % (ud.pkgname, ud.version) 81 ud.bbnpmmanifest = ud.bbnpmmanifest.replace('/', '-') 82 ud.registry = "http://%s" % (ud.url.replace('npm://', '', 1).split(';'))[0] 83 prefixdir = "npm/%s" % ud.pkgname 84 ud.pkgdatadir = d.expand("${DL_DIR}/%s" % prefixdir) 85 if not os.path.exists(ud.pkgdatadir): 86 bb.utils.mkdirhier(ud.pkgdatadir) 87 ud.localpath = d.expand("${DL_DIR}/npm/%s" % ud.bbnpmmanifest) 88 89 self.basecmd = d.getVar("FETCHCMD_wget") or "/usr/bin/env wget -O -t 2 -T 30 -nv --passive-ftp --no-check-certificate " 90 ud.prefixdir = prefixdir 91 92 ud.write_tarballs = ((d.getVar("BB_GENERATE_MIRROR_TARBALLS") or "0") != "0") 93 mirrortarball = 'npm_%s-%s.tar.xz' % (ud.pkgname, ud.version) 94 mirrortarball = mirrortarball.replace('/', '-') 95 ud.fullmirror = os.path.join(d.getVar("DL_DIR"), mirrortarball) 96 ud.mirrortarballs = [mirrortarball] 97 98 def need_update(self, ud, d): 99 if os.path.exists(ud.localpath): 100 return False 101 return True 102 103 def _runwget(self, ud, d, command, quiet): 104 logger.debug(2, "Fetching %s using command '%s'" % (ud.url, command)) 105 bb.fetch2.check_network_access(d, command, ud.url) 106 dldir = d.getVar("DL_DIR") 107 runfetchcmd(command, d, quiet, workdir=dldir) 108 109 def _unpackdep(self, ud, pkg, data, destdir, dldir, d): 110 file = data[pkg]['tgz'] 111 logger.debug(2, "file to extract is %s" % file) 112 if file.endswith('.tgz') or file.endswith('.tar.gz') or file.endswith('.tar.Z'): 113 cmd = 'tar xz --strip 1 --no-same-owner --warning=no-unknown-keyword -f %s/%s' % (dldir, file) 114 else: 115 bb.fatal("NPM package %s downloaded not a tarball!" % file) 116 117 # Change to subdir before executing command 118 if not os.path.exists(destdir): 119 os.makedirs(destdir) 120 path = d.getVar('PATH') 121 if path: 122 cmd = "PATH=\"%s\" %s" % (path, cmd) 123 bb.note("Unpacking %s to %s/" % (file, destdir)) 124 ret = subprocess.call(cmd, preexec_fn=subprocess_setup, shell=True, cwd=destdir) 125 126 if ret != 0: 127 raise UnpackError("Unpack command %s failed with return value %s" % (cmd, ret), ud.url) 128 129 if 'deps' not in data[pkg]: 130 return 131 for dep in data[pkg]['deps']: 132 self._unpackdep(ud, dep, data[pkg]['deps'], "%s/node_modules/%s" % (destdir, dep), dldir, d) 133 134 135 def unpack(self, ud, destdir, d): 136 dldir = d.getVar("DL_DIR") 137 with open("%s/npm/%s" % (dldir, ud.bbnpmmanifest)) as datafile: 138 workobj = json.load(datafile) 139 dldir = "%s/%s" % (os.path.dirname(ud.localpath), ud.pkgname) 140 141 if 'subdir' in ud.parm: 142 unpackdir = '%s/%s' % (destdir, ud.parm.get('subdir')) 143 else: 144 unpackdir = '%s/npmpkg' % destdir 145 146 self._unpackdep(ud, ud.pkgname, workobj, unpackdir, dldir, d) 147 148 def _parse_view(self, output): 149 ''' 150 Parse the output of npm view --json; the last JSON result 151 is assumed to be the one that we're interested in. 152 ''' 153 pdata = None 154 outdeps = {} 155 datalines = [] 156 bracelevel = 0 157 for line in output.splitlines(): 158 if bracelevel: 159 datalines.append(line) 160 elif '{' in line: 161 datalines = [] 162 datalines.append(line) 163 bracelevel = bracelevel + line.count('{') - line.count('}') 164 if datalines: 165 pdata = json.loads('\n'.join(datalines)) 166 return pdata 167 168 def _getdependencies(self, pkg, data, version, d, ud, optional=False, fetchedlist=None): 169 if fetchedlist is None: 170 fetchedlist = [] 171 pkgfullname = pkg 172 if version != '*' and not '/' in version: 173 pkgfullname += "@'%s'" % version 174 logger.debug(2, "Calling getdeps on %s" % pkg) 175 fetchcmd = "npm view %s --json --registry %s" % (pkgfullname, ud.registry) 176 output = runfetchcmd(fetchcmd, d, True) 177 pdata = self._parse_view(output) 178 if not pdata: 179 raise FetchError("The command '%s' returned no output" % fetchcmd) 180 if optional: 181 pkg_os = pdata.get('os', None) 182 if pkg_os: 183 if not isinstance(pkg_os, list): 184 pkg_os = [pkg_os] 185 blacklist = False 186 for item in pkg_os: 187 if item.startswith('!'): 188 blacklist = True 189 break 190 if (not blacklist and 'linux' not in pkg_os) or '!linux' in pkg_os: 191 logger.debug(2, "Skipping %s since it's incompatible with Linux" % pkg) 192 return 193 #logger.debug(2, "Output URL is %s - %s - %s" % (ud.basepath, ud.basename, ud.localfile)) 194 outputurl = pdata['dist']['tarball'] 195 data[pkg] = {} 196 data[pkg]['tgz'] = os.path.basename(outputurl) 197 if outputurl in fetchedlist: 198 return 199 200 self._runwget(ud, d, "%s --directory-prefix=%s %s" % (self.basecmd, ud.prefixdir, outputurl), False) 201 fetchedlist.append(outputurl) 202 203 dependencies = pdata.get('dependencies', {}) 204 optionalDependencies = pdata.get('optionalDependencies', {}) 205 dependencies.update(optionalDependencies) 206 depsfound = {} 207 optdepsfound = {} 208 data[pkg]['deps'] = {} 209 for dep in dependencies: 210 if dep in optionalDependencies: 211 optdepsfound[dep] = dependencies[dep] 212 else: 213 depsfound[dep] = dependencies[dep] 214 for dep, version in optdepsfound.items(): 215 self._getdependencies(dep, data[pkg]['deps'], version, d, ud, optional=True, fetchedlist=fetchedlist) 216 for dep, version in depsfound.items(): 217 self._getdependencies(dep, data[pkg]['deps'], version, d, ud, fetchedlist=fetchedlist) 218 219 def _getshrinkeddependencies(self, pkg, data, version, d, ud, lockdown, manifest, toplevel=True): 220 logger.debug(2, "NPM shrinkwrap file is %s" % data) 221 if toplevel: 222 name = data.get('name', None) 223 if name and name != pkg: 224 for obj in data.get('dependencies', []): 225 if obj == pkg: 226 self._getshrinkeddependencies(obj, data['dependencies'][obj], data['dependencies'][obj]['version'], d, ud, lockdown, manifest, False) 227 return 228 outputurl = "invalid" 229 if ('resolved' not in data) or (not data['resolved'].startswith('http')): 230 # will be the case for ${PN} 231 fetchcmd = "npm view %s@%s dist.tarball --registry %s" % (pkg, version, ud.registry) 232 logger.debug(2, "Found this matching URL: %s" % str(fetchcmd)) 233 outputurl = runfetchcmd(fetchcmd, d, True) 234 else: 235 outputurl = data['resolved'] 236 self._runwget(ud, d, "%s --directory-prefix=%s %s" % (self.basecmd, ud.prefixdir, outputurl), False) 237 manifest[pkg] = {} 238 manifest[pkg]['tgz'] = os.path.basename(outputurl).rstrip() 239 manifest[pkg]['deps'] = {} 240 241 if pkg in lockdown: 242 sha1_expected = lockdown[pkg][version] 243 sha1_data = bb.utils.sha1_file("npm/%s/%s" % (ud.pkgname, manifest[pkg]['tgz'])) 244 if sha1_expected != sha1_data: 245 msg = "\nFile: '%s' has %s checksum %s when %s was expected" % (manifest[pkg]['tgz'], 'sha1', sha1_data, sha1_expected) 246 raise ChecksumError('Checksum mismatch!%s' % msg) 247 else: 248 logger.debug(2, "No lockdown data for %s@%s" % (pkg, version)) 249 250 if 'dependencies' in data: 251 for obj in data['dependencies']: 252 logger.debug(2, "Found dep is %s" % str(obj)) 253 self._getshrinkeddependencies(obj, data['dependencies'][obj], data['dependencies'][obj]['version'], d, ud, lockdown, manifest[pkg]['deps'], False) 254 255 def download(self, ud, d): 256 """Fetch url""" 257 jsondepobj = {} 258 shrinkobj = {} 259 lockdown = {} 260 261 if not os.listdir(ud.pkgdatadir) and os.path.exists(ud.fullmirror): 262 dest = d.getVar("DL_DIR") 263 bb.utils.mkdirhier(dest) 264 runfetchcmd("tar -xJf %s" % (ud.fullmirror), d, workdir=dest) 265 return 266 267 if ud.parm.get("noverify", None) != '1': 268 shwrf = d.getVar('NPM_SHRINKWRAP') 269 logger.debug(2, "NPM shrinkwrap file is %s" % shwrf) 270 if shwrf: 271 try: 272 with open(shwrf) as datafile: 273 shrinkobj = json.load(datafile) 274 except Exception as e: 275 raise FetchError('Error loading NPM_SHRINKWRAP file "%s" for %s: %s' % (shwrf, ud.pkgname, str(e))) 276 elif not ud.ignore_checksums: 277 logger.warning('Missing shrinkwrap file in NPM_SHRINKWRAP for %s, this will lead to unreliable builds!' % ud.pkgname) 278 lckdf = d.getVar('NPM_LOCKDOWN') 279 logger.debug(2, "NPM lockdown file is %s" % lckdf) 280 if lckdf: 281 try: 282 with open(lckdf) as datafile: 283 lockdown = json.load(datafile) 284 except Exception as e: 285 raise FetchError('Error loading NPM_LOCKDOWN file "%s" for %s: %s' % (lckdf, ud.pkgname, str(e))) 286 elif not ud.ignore_checksums: 287 logger.warning('Missing lockdown file in NPM_LOCKDOWN for %s, this will lead to unreproducible builds!' % ud.pkgname) 288 289 if ('name' not in shrinkobj): 290 self._getdependencies(ud.pkgname, jsondepobj, ud.version, d, ud) 291 else: 292 self._getshrinkeddependencies(ud.pkgname, shrinkobj, ud.version, d, ud, lockdown, jsondepobj) 293 294 with open(ud.localpath, 'w') as outfile: 295 json.dump(jsondepobj, outfile) 296 297 def build_mirror_data(self, ud, d): 298 # Generate a mirror tarball if needed 299 if ud.write_tarballs and not os.path.exists(ud.fullmirror): 300 # it's possible that this symlink points to read-only filesystem with PREMIRROR 301 if os.path.islink(ud.fullmirror): 302 os.unlink(ud.fullmirror) 303 304 dldir = d.getVar("DL_DIR") 305 logger.info("Creating tarball of npm data") 306 runfetchcmd("tar -cJf %s npm/%s npm/%s" % (ud.fullmirror, ud.bbnpmmanifest, ud.pkgname), d, 307 workdir=dldir) 308 runfetchcmd("touch %s.done" % (ud.fullmirror), d, workdir=dldir) 309