1*c342db35SBrad Bishop# 2*c342db35SBrad Bishop# SPDX-License-Identifier: GPL-2.0-only 3*c342db35SBrad Bishop# 4eb8dc403SDave Cobbley""" 5eb8dc403SDave CobbleyBitBake 'Fetch' NPM implementation 6eb8dc403SDave Cobbley 7eb8dc403SDave CobbleyThe NPM fetcher is used to retrieve files from the npmjs repository 8eb8dc403SDave Cobbley 9eb8dc403SDave CobbleyUsage in the recipe: 10eb8dc403SDave Cobbley 11eb8dc403SDave Cobbley SRC_URI = "npm://registry.npmjs.org/;name=${PN};version=${PV}" 12eb8dc403SDave Cobbley Suported SRC_URI options are: 13eb8dc403SDave Cobbley 14eb8dc403SDave Cobbley - name 15eb8dc403SDave Cobbley - version 16eb8dc403SDave Cobbley 17eb8dc403SDave Cobbley npm://registry.npmjs.org/${PN}/-/${PN}-${PV}.tgz would become npm://registry.npmjs.org;name=${PN};version=${PV} 18eb8dc403SDave Cobbley The fetcher all triggers off the existence of ud.localpath. If that exists and has the ".done" stamp, its assumed the fetch is good/done 19eb8dc403SDave Cobbley 20eb8dc403SDave Cobbley""" 21eb8dc403SDave Cobbley 22eb8dc403SDave Cobbleyimport os 23eb8dc403SDave Cobbleyimport sys 24eb8dc403SDave Cobbleyimport urllib.request, urllib.parse, urllib.error 25eb8dc403SDave Cobbleyimport json 26eb8dc403SDave Cobbleyimport subprocess 27eb8dc403SDave Cobbleyimport signal 28eb8dc403SDave Cobbleyimport bb 29eb8dc403SDave Cobbleyfrom bb.fetch2 import FetchMethod 30eb8dc403SDave Cobbleyfrom bb.fetch2 import FetchError 31eb8dc403SDave Cobbleyfrom bb.fetch2 import ChecksumError 32eb8dc403SDave Cobbleyfrom bb.fetch2 import runfetchcmd 33eb8dc403SDave Cobbleyfrom bb.fetch2 import logger 34eb8dc403SDave Cobbleyfrom bb.fetch2 import UnpackError 35eb8dc403SDave Cobbleyfrom bb.fetch2 import ParameterError 36eb8dc403SDave Cobbley 37eb8dc403SDave Cobbleydef subprocess_setup(): 38eb8dc403SDave Cobbley # Python installs a SIGPIPE handler by default. This is usually not what 39eb8dc403SDave Cobbley # non-Python subprocesses expect. 40eb8dc403SDave Cobbley # SIGPIPE errors are known issues with gzip/bash 41eb8dc403SDave Cobbley signal.signal(signal.SIGPIPE, signal.SIG_DFL) 42eb8dc403SDave Cobbley 43eb8dc403SDave Cobbleyclass Npm(FetchMethod): 44eb8dc403SDave Cobbley 45eb8dc403SDave Cobbley """Class to fetch urls via 'npm'""" 46eb8dc403SDave Cobbley def init(self, d): 47eb8dc403SDave Cobbley pass 48eb8dc403SDave Cobbley 49eb8dc403SDave Cobbley def supports(self, ud, d): 50eb8dc403SDave Cobbley """ 51eb8dc403SDave Cobbley Check to see if a given url can be fetched with npm 52eb8dc403SDave Cobbley """ 53eb8dc403SDave Cobbley return ud.type in ['npm'] 54eb8dc403SDave Cobbley 55eb8dc403SDave Cobbley def debug(self, msg): 56eb8dc403SDave Cobbley logger.debug(1, "NpmFetch: %s", msg) 57eb8dc403SDave Cobbley 58eb8dc403SDave Cobbley def clean(self, ud, d): 59eb8dc403SDave Cobbley logger.debug(2, "Calling cleanup %s" % ud.pkgname) 60eb8dc403SDave Cobbley bb.utils.remove(ud.localpath, False) 61eb8dc403SDave Cobbley bb.utils.remove(ud.pkgdatadir, True) 62eb8dc403SDave Cobbley bb.utils.remove(ud.fullmirror, False) 63eb8dc403SDave Cobbley 64eb8dc403SDave Cobbley def urldata_init(self, ud, d): 65eb8dc403SDave Cobbley """ 66eb8dc403SDave Cobbley init NPM specific variable within url data 67eb8dc403SDave Cobbley """ 68eb8dc403SDave Cobbley if 'downloadfilename' in ud.parm: 69eb8dc403SDave Cobbley ud.basename = ud.parm['downloadfilename'] 70eb8dc403SDave Cobbley else: 71eb8dc403SDave Cobbley ud.basename = os.path.basename(ud.path) 72eb8dc403SDave Cobbley 73eb8dc403SDave Cobbley # can't call it ud.name otherwise fetcher base class will start doing sha1stuff 74eb8dc403SDave Cobbley # TODO: find a way to get an sha1/sha256 manifest of pkg & all deps 75eb8dc403SDave Cobbley ud.pkgname = ud.parm.get("name", None) 76eb8dc403SDave Cobbley if not ud.pkgname: 77eb8dc403SDave Cobbley raise ParameterError("NPM fetcher requires a name parameter", ud.url) 78eb8dc403SDave Cobbley ud.version = ud.parm.get("version", None) 79eb8dc403SDave Cobbley if not ud.version: 80eb8dc403SDave Cobbley raise ParameterError("NPM fetcher requires a version parameter", ud.url) 81eb8dc403SDave Cobbley ud.bbnpmmanifest = "%s-%s.deps.json" % (ud.pkgname, ud.version) 82eb8dc403SDave Cobbley ud.bbnpmmanifest = ud.bbnpmmanifest.replace('/', '-') 83eb8dc403SDave Cobbley ud.registry = "http://%s" % (ud.url.replace('npm://', '', 1).split(';'))[0] 84eb8dc403SDave Cobbley prefixdir = "npm/%s" % ud.pkgname 85eb8dc403SDave Cobbley ud.pkgdatadir = d.expand("${DL_DIR}/%s" % prefixdir) 86eb8dc403SDave Cobbley if not os.path.exists(ud.pkgdatadir): 87eb8dc403SDave Cobbley bb.utils.mkdirhier(ud.pkgdatadir) 88eb8dc403SDave Cobbley ud.localpath = d.expand("${DL_DIR}/npm/%s" % ud.bbnpmmanifest) 89eb8dc403SDave Cobbley 90eb8dc403SDave Cobbley self.basecmd = d.getVar("FETCHCMD_wget") or "/usr/bin/env wget -O -t 2 -T 30 -nv --passive-ftp --no-check-certificate " 91eb8dc403SDave Cobbley ud.prefixdir = prefixdir 92eb8dc403SDave Cobbley 93eb8dc403SDave Cobbley ud.write_tarballs = ((d.getVar("BB_GENERATE_MIRROR_TARBALLS") or "0") != "0") 94eb8dc403SDave Cobbley mirrortarball = 'npm_%s-%s.tar.xz' % (ud.pkgname, ud.version) 95eb8dc403SDave Cobbley mirrortarball = mirrortarball.replace('/', '-') 96eb8dc403SDave Cobbley ud.fullmirror = os.path.join(d.getVar("DL_DIR"), mirrortarball) 97eb8dc403SDave Cobbley ud.mirrortarballs = [mirrortarball] 98eb8dc403SDave Cobbley 99eb8dc403SDave Cobbley def need_update(self, ud, d): 100eb8dc403SDave Cobbley if os.path.exists(ud.localpath): 101eb8dc403SDave Cobbley return False 102eb8dc403SDave Cobbley return True 103eb8dc403SDave Cobbley 104eb8dc403SDave Cobbley def _runwget(self, ud, d, command, quiet): 105eb8dc403SDave Cobbley logger.debug(2, "Fetching %s using command '%s'" % (ud.url, command)) 106eb8dc403SDave Cobbley bb.fetch2.check_network_access(d, command, ud.url) 107eb8dc403SDave Cobbley dldir = d.getVar("DL_DIR") 108eb8dc403SDave Cobbley runfetchcmd(command, d, quiet, workdir=dldir) 109eb8dc403SDave Cobbley 110eb8dc403SDave Cobbley def _unpackdep(self, ud, pkg, data, destdir, dldir, d): 111eb8dc403SDave Cobbley file = data[pkg]['tgz'] 112eb8dc403SDave Cobbley logger.debug(2, "file to extract is %s" % file) 113eb8dc403SDave Cobbley if file.endswith('.tgz') or file.endswith('.tar.gz') or file.endswith('.tar.Z'): 114eb8dc403SDave Cobbley cmd = 'tar xz --strip 1 --no-same-owner --warning=no-unknown-keyword -f %s/%s' % (dldir, file) 115eb8dc403SDave Cobbley else: 116eb8dc403SDave Cobbley bb.fatal("NPM package %s downloaded not a tarball!" % file) 117eb8dc403SDave Cobbley 118eb8dc403SDave Cobbley # Change to subdir before executing command 119eb8dc403SDave Cobbley if not os.path.exists(destdir): 120eb8dc403SDave Cobbley os.makedirs(destdir) 121eb8dc403SDave Cobbley path = d.getVar('PATH') 122eb8dc403SDave Cobbley if path: 123eb8dc403SDave Cobbley cmd = "PATH=\"%s\" %s" % (path, cmd) 124eb8dc403SDave Cobbley bb.note("Unpacking %s to %s/" % (file, destdir)) 125eb8dc403SDave Cobbley ret = subprocess.call(cmd, preexec_fn=subprocess_setup, shell=True, cwd=destdir) 126eb8dc403SDave Cobbley 127eb8dc403SDave Cobbley if ret != 0: 128eb8dc403SDave Cobbley raise UnpackError("Unpack command %s failed with return value %s" % (cmd, ret), ud.url) 129eb8dc403SDave Cobbley 130eb8dc403SDave Cobbley if 'deps' not in data[pkg]: 131eb8dc403SDave Cobbley return 132eb8dc403SDave Cobbley for dep in data[pkg]['deps']: 133eb8dc403SDave Cobbley self._unpackdep(ud, dep, data[pkg]['deps'], "%s/node_modules/%s" % (destdir, dep), dldir, d) 134eb8dc403SDave Cobbley 135eb8dc403SDave Cobbley 136eb8dc403SDave Cobbley def unpack(self, ud, destdir, d): 137eb8dc403SDave Cobbley dldir = d.getVar("DL_DIR") 138eb8dc403SDave Cobbley with open("%s/npm/%s" % (dldir, ud.bbnpmmanifest)) as datafile: 139eb8dc403SDave Cobbley workobj = json.load(datafile) 140eb8dc403SDave Cobbley dldir = "%s/%s" % (os.path.dirname(ud.localpath), ud.pkgname) 141eb8dc403SDave Cobbley 142eb8dc403SDave Cobbley if 'subdir' in ud.parm: 143eb8dc403SDave Cobbley unpackdir = '%s/%s' % (destdir, ud.parm.get('subdir')) 144eb8dc403SDave Cobbley else: 145eb8dc403SDave Cobbley unpackdir = '%s/npmpkg' % destdir 146eb8dc403SDave Cobbley 147eb8dc403SDave Cobbley self._unpackdep(ud, ud.pkgname, workobj, unpackdir, dldir, d) 148eb8dc403SDave Cobbley 149eb8dc403SDave Cobbley def _parse_view(self, output): 150eb8dc403SDave Cobbley ''' 151eb8dc403SDave Cobbley Parse the output of npm view --json; the last JSON result 152eb8dc403SDave Cobbley is assumed to be the one that we're interested in. 153eb8dc403SDave Cobbley ''' 154eb8dc403SDave Cobbley pdata = None 155eb8dc403SDave Cobbley outdeps = {} 156eb8dc403SDave Cobbley datalines = [] 157eb8dc403SDave Cobbley bracelevel = 0 158eb8dc403SDave Cobbley for line in output.splitlines(): 159eb8dc403SDave Cobbley if bracelevel: 160eb8dc403SDave Cobbley datalines.append(line) 161eb8dc403SDave Cobbley elif '{' in line: 162eb8dc403SDave Cobbley datalines = [] 163eb8dc403SDave Cobbley datalines.append(line) 164eb8dc403SDave Cobbley bracelevel = bracelevel + line.count('{') - line.count('}') 165eb8dc403SDave Cobbley if datalines: 166eb8dc403SDave Cobbley pdata = json.loads('\n'.join(datalines)) 167eb8dc403SDave Cobbley return pdata 168eb8dc403SDave Cobbley 169eb8dc403SDave Cobbley def _getdependencies(self, pkg, data, version, d, ud, optional=False, fetchedlist=None): 170eb8dc403SDave Cobbley if fetchedlist is None: 171eb8dc403SDave Cobbley fetchedlist = [] 172eb8dc403SDave Cobbley pkgfullname = pkg 173eb8dc403SDave Cobbley if version != '*' and not '/' in version: 174eb8dc403SDave Cobbley pkgfullname += "@'%s'" % version 175eb8dc403SDave Cobbley logger.debug(2, "Calling getdeps on %s" % pkg) 176eb8dc403SDave Cobbley fetchcmd = "npm view %s --json --registry %s" % (pkgfullname, ud.registry) 177eb8dc403SDave Cobbley output = runfetchcmd(fetchcmd, d, True) 178eb8dc403SDave Cobbley pdata = self._parse_view(output) 179eb8dc403SDave Cobbley if not pdata: 180eb8dc403SDave Cobbley raise FetchError("The command '%s' returned no output" % fetchcmd) 181eb8dc403SDave Cobbley if optional: 182eb8dc403SDave Cobbley pkg_os = pdata.get('os', None) 183eb8dc403SDave Cobbley if pkg_os: 184eb8dc403SDave Cobbley if not isinstance(pkg_os, list): 185eb8dc403SDave Cobbley pkg_os = [pkg_os] 186eb8dc403SDave Cobbley blacklist = False 187eb8dc403SDave Cobbley for item in pkg_os: 188eb8dc403SDave Cobbley if item.startswith('!'): 189eb8dc403SDave Cobbley blacklist = True 190eb8dc403SDave Cobbley break 191eb8dc403SDave Cobbley if (not blacklist and 'linux' not in pkg_os) or '!linux' in pkg_os: 192eb8dc403SDave Cobbley logger.debug(2, "Skipping %s since it's incompatible with Linux" % pkg) 193eb8dc403SDave Cobbley return 194eb8dc403SDave Cobbley #logger.debug(2, "Output URL is %s - %s - %s" % (ud.basepath, ud.basename, ud.localfile)) 195eb8dc403SDave Cobbley outputurl = pdata['dist']['tarball'] 196eb8dc403SDave Cobbley data[pkg] = {} 197eb8dc403SDave Cobbley data[pkg]['tgz'] = os.path.basename(outputurl) 198eb8dc403SDave Cobbley if outputurl in fetchedlist: 199eb8dc403SDave Cobbley return 200eb8dc403SDave Cobbley 201eb8dc403SDave Cobbley self._runwget(ud, d, "%s --directory-prefix=%s %s" % (self.basecmd, ud.prefixdir, outputurl), False) 202eb8dc403SDave Cobbley fetchedlist.append(outputurl) 203eb8dc403SDave Cobbley 204eb8dc403SDave Cobbley dependencies = pdata.get('dependencies', {}) 205eb8dc403SDave Cobbley optionalDependencies = pdata.get('optionalDependencies', {}) 206eb8dc403SDave Cobbley dependencies.update(optionalDependencies) 207eb8dc403SDave Cobbley depsfound = {} 208eb8dc403SDave Cobbley optdepsfound = {} 209eb8dc403SDave Cobbley data[pkg]['deps'] = {} 210eb8dc403SDave Cobbley for dep in dependencies: 211eb8dc403SDave Cobbley if dep in optionalDependencies: 212eb8dc403SDave Cobbley optdepsfound[dep] = dependencies[dep] 213eb8dc403SDave Cobbley else: 214eb8dc403SDave Cobbley depsfound[dep] = dependencies[dep] 215eb8dc403SDave Cobbley for dep, version in optdepsfound.items(): 216eb8dc403SDave Cobbley self._getdependencies(dep, data[pkg]['deps'], version, d, ud, optional=True, fetchedlist=fetchedlist) 217eb8dc403SDave Cobbley for dep, version in depsfound.items(): 218eb8dc403SDave Cobbley self._getdependencies(dep, data[pkg]['deps'], version, d, ud, fetchedlist=fetchedlist) 219eb8dc403SDave Cobbley 220eb8dc403SDave Cobbley def _getshrinkeddependencies(self, pkg, data, version, d, ud, lockdown, manifest, toplevel=True): 221eb8dc403SDave Cobbley logger.debug(2, "NPM shrinkwrap file is %s" % data) 222eb8dc403SDave Cobbley if toplevel: 223eb8dc403SDave Cobbley name = data.get('name', None) 224eb8dc403SDave Cobbley if name and name != pkg: 225eb8dc403SDave Cobbley for obj in data.get('dependencies', []): 226eb8dc403SDave Cobbley if obj == pkg: 227eb8dc403SDave Cobbley self._getshrinkeddependencies(obj, data['dependencies'][obj], data['dependencies'][obj]['version'], d, ud, lockdown, manifest, False) 228eb8dc403SDave Cobbley return 229eb8dc403SDave Cobbley outputurl = "invalid" 23019323693SBrad Bishop if ('resolved' not in data) or (not data['resolved'].startswith('http://') and not data['resolved'].startswith('https://')): 231eb8dc403SDave Cobbley # will be the case for ${PN} 232eb8dc403SDave Cobbley fetchcmd = "npm view %s@%s dist.tarball --registry %s" % (pkg, version, ud.registry) 233eb8dc403SDave Cobbley logger.debug(2, "Found this matching URL: %s" % str(fetchcmd)) 234eb8dc403SDave Cobbley outputurl = runfetchcmd(fetchcmd, d, True) 235eb8dc403SDave Cobbley else: 236eb8dc403SDave Cobbley outputurl = data['resolved'] 237eb8dc403SDave Cobbley self._runwget(ud, d, "%s --directory-prefix=%s %s" % (self.basecmd, ud.prefixdir, outputurl), False) 238eb8dc403SDave Cobbley manifest[pkg] = {} 239eb8dc403SDave Cobbley manifest[pkg]['tgz'] = os.path.basename(outputurl).rstrip() 240eb8dc403SDave Cobbley manifest[pkg]['deps'] = {} 241eb8dc403SDave Cobbley 242eb8dc403SDave Cobbley if pkg in lockdown: 243eb8dc403SDave Cobbley sha1_expected = lockdown[pkg][version] 244eb8dc403SDave Cobbley sha1_data = bb.utils.sha1_file("npm/%s/%s" % (ud.pkgname, manifest[pkg]['tgz'])) 245eb8dc403SDave Cobbley if sha1_expected != sha1_data: 246eb8dc403SDave Cobbley msg = "\nFile: '%s' has %s checksum %s when %s was expected" % (manifest[pkg]['tgz'], 'sha1', sha1_data, sha1_expected) 247eb8dc403SDave Cobbley raise ChecksumError('Checksum mismatch!%s' % msg) 248eb8dc403SDave Cobbley else: 249eb8dc403SDave Cobbley logger.debug(2, "No lockdown data for %s@%s" % (pkg, version)) 250eb8dc403SDave Cobbley 251eb8dc403SDave Cobbley if 'dependencies' in data: 252eb8dc403SDave Cobbley for obj in data['dependencies']: 253eb8dc403SDave Cobbley logger.debug(2, "Found dep is %s" % str(obj)) 254eb8dc403SDave Cobbley self._getshrinkeddependencies(obj, data['dependencies'][obj], data['dependencies'][obj]['version'], d, ud, lockdown, manifest[pkg]['deps'], False) 255eb8dc403SDave Cobbley 256eb8dc403SDave Cobbley def download(self, ud, d): 257eb8dc403SDave Cobbley """Fetch url""" 258eb8dc403SDave Cobbley jsondepobj = {} 259eb8dc403SDave Cobbley shrinkobj = {} 260eb8dc403SDave Cobbley lockdown = {} 261eb8dc403SDave Cobbley 262eb8dc403SDave Cobbley if not os.listdir(ud.pkgdatadir) and os.path.exists(ud.fullmirror): 263eb8dc403SDave Cobbley dest = d.getVar("DL_DIR") 264eb8dc403SDave Cobbley bb.utils.mkdirhier(dest) 265eb8dc403SDave Cobbley runfetchcmd("tar -xJf %s" % (ud.fullmirror), d, workdir=dest) 266eb8dc403SDave Cobbley return 267eb8dc403SDave Cobbley 268eb8dc403SDave Cobbley if ud.parm.get("noverify", None) != '1': 269eb8dc403SDave Cobbley shwrf = d.getVar('NPM_SHRINKWRAP') 270eb8dc403SDave Cobbley logger.debug(2, "NPM shrinkwrap file is %s" % shwrf) 271eb8dc403SDave Cobbley if shwrf: 272eb8dc403SDave Cobbley try: 273eb8dc403SDave Cobbley with open(shwrf) as datafile: 274eb8dc403SDave Cobbley shrinkobj = json.load(datafile) 275eb8dc403SDave Cobbley except Exception as e: 276eb8dc403SDave Cobbley raise FetchError('Error loading NPM_SHRINKWRAP file "%s" for %s: %s' % (shwrf, ud.pkgname, str(e))) 277eb8dc403SDave Cobbley elif not ud.ignore_checksums: 278eb8dc403SDave Cobbley logger.warning('Missing shrinkwrap file in NPM_SHRINKWRAP for %s, this will lead to unreliable builds!' % ud.pkgname) 279eb8dc403SDave Cobbley lckdf = d.getVar('NPM_LOCKDOWN') 280eb8dc403SDave Cobbley logger.debug(2, "NPM lockdown file is %s" % lckdf) 281eb8dc403SDave Cobbley if lckdf: 282eb8dc403SDave Cobbley try: 283eb8dc403SDave Cobbley with open(lckdf) as datafile: 284eb8dc403SDave Cobbley lockdown = json.load(datafile) 285eb8dc403SDave Cobbley except Exception as e: 286eb8dc403SDave Cobbley raise FetchError('Error loading NPM_LOCKDOWN file "%s" for %s: %s' % (lckdf, ud.pkgname, str(e))) 287eb8dc403SDave Cobbley elif not ud.ignore_checksums: 288eb8dc403SDave Cobbley logger.warning('Missing lockdown file in NPM_LOCKDOWN for %s, this will lead to unreproducible builds!' % ud.pkgname) 289eb8dc403SDave Cobbley 290eb8dc403SDave Cobbley if ('name' not in shrinkobj): 291eb8dc403SDave Cobbley self._getdependencies(ud.pkgname, jsondepobj, ud.version, d, ud) 292eb8dc403SDave Cobbley else: 293eb8dc403SDave Cobbley self._getshrinkeddependencies(ud.pkgname, shrinkobj, ud.version, d, ud, lockdown, jsondepobj) 294eb8dc403SDave Cobbley 295eb8dc403SDave Cobbley with open(ud.localpath, 'w') as outfile: 296eb8dc403SDave Cobbley json.dump(jsondepobj, outfile) 297eb8dc403SDave Cobbley 298eb8dc403SDave Cobbley def build_mirror_data(self, ud, d): 299eb8dc403SDave Cobbley # Generate a mirror tarball if needed 300eb8dc403SDave Cobbley if ud.write_tarballs and not os.path.exists(ud.fullmirror): 301eb8dc403SDave Cobbley # it's possible that this symlink points to read-only filesystem with PREMIRROR 302eb8dc403SDave Cobbley if os.path.islink(ud.fullmirror): 303eb8dc403SDave Cobbley os.unlink(ud.fullmirror) 304eb8dc403SDave Cobbley 305eb8dc403SDave Cobbley dldir = d.getVar("DL_DIR") 306eb8dc403SDave Cobbley logger.info("Creating tarball of npm data") 307eb8dc403SDave Cobbley runfetchcmd("tar -cJf %s npm/%s npm/%s" % (ud.fullmirror, ud.bbnpmmanifest, ud.pkgname), d, 308eb8dc403SDave Cobbley workdir=dldir) 309eb8dc403SDave Cobbley runfetchcmd("touch %s.done" % (ud.fullmirror), d, workdir=dldir) 310