182c905dcSAndrew Geissler# Copyright (C) 2020 Savoir-Faire Linux 2c342db35SBrad Bishop# 3c342db35SBrad Bishop# SPDX-License-Identifier: GPL-2.0-only 4c342db35SBrad Bishop# 5eb8dc403SDave Cobbley""" 682c905dcSAndrew GeisslerBitBake 'Fetch' npm implementation 7eb8dc403SDave Cobbley 882c905dcSAndrew Geisslernpm fetcher support the SRC_URI with format of: 982c905dcSAndrew GeisslerSRC_URI = "npm://some.registry.url;OptionA=xxx;OptionB=xxx;..." 10eb8dc403SDave Cobbley 1182c905dcSAndrew GeisslerSupported SRC_URI options are: 12eb8dc403SDave Cobbley 1382c905dcSAndrew Geissler- package 1482c905dcSAndrew Geissler The npm package name. This is a mandatory parameter. 15eb8dc403SDave Cobbley 16eb8dc403SDave Cobbley- version 1782c905dcSAndrew Geissler The npm package version. This is a mandatory parameter. 18eb8dc403SDave Cobbley 1982c905dcSAndrew Geissler- downloadfilename 2082c905dcSAndrew Geissler Specifies the filename used when storing the downloaded file. 21eb8dc403SDave Cobbley 2282c905dcSAndrew Geissler- destsuffix 2382c905dcSAndrew Geissler Specifies the directory to use to unpack the package (default: npm). 24eb8dc403SDave Cobbley""" 25eb8dc403SDave Cobbley 2682c905dcSAndrew Geisslerimport base64 27eb8dc403SDave Cobbleyimport json 2882c905dcSAndrew Geisslerimport os 2982c905dcSAndrew Geisslerimport re 3082c905dcSAndrew Geisslerimport shlex 3182c905dcSAndrew Geisslerimport tempfile 32eb8dc403SDave Cobbleyimport bb 3382c905dcSAndrew Geisslerfrom bb.fetch2 import Fetch 34eb8dc403SDave Cobbleyfrom bb.fetch2 import FetchError 3582c905dcSAndrew Geisslerfrom bb.fetch2 import FetchMethod 3682c905dcSAndrew Geisslerfrom bb.fetch2 import MissingParameterError 37eb8dc403SDave Cobbleyfrom bb.fetch2 import ParameterError 3882c905dcSAndrew Geisslerfrom bb.fetch2 import URI 3982c905dcSAndrew Geisslerfrom bb.fetch2 import check_network_access 4082c905dcSAndrew Geisslerfrom bb.fetch2 import runfetchcmd 4182c905dcSAndrew Geisslerfrom bb.utils import is_semver 42eb8dc403SDave Cobbley 4382c905dcSAndrew Geisslerdef npm_package(package): 4482c905dcSAndrew Geissler """Convert the npm package name to remove unsupported character""" 4582c905dcSAndrew Geissler # Scoped package names (with the @) use the same naming convention 4682c905dcSAndrew Geissler # as the 'npm pack' command. 4782c905dcSAndrew Geissler if package.startswith("@"): 4882c905dcSAndrew Geissler return re.sub("/", "-", package[1:]) 4982c905dcSAndrew Geissler return package 5082c905dcSAndrew Geissler 5182c905dcSAndrew Geisslerdef npm_filename(package, version): 5282c905dcSAndrew Geissler """Get the filename of a npm package""" 5382c905dcSAndrew Geissler return npm_package(package) + "-" + version + ".tgz" 5482c905dcSAndrew Geissler 557e0e3c0cSAndrew Geisslerdef npm_localfile(package, version=None): 5682c905dcSAndrew Geissler """Get the local filename of a npm package""" 577e0e3c0cSAndrew Geissler if version is not None: 587e0e3c0cSAndrew Geissler filename = npm_filename(package, version) 597e0e3c0cSAndrew Geissler else: 607e0e3c0cSAndrew Geissler filename = package 617e0e3c0cSAndrew Geissler return os.path.join("npm2", filename) 6282c905dcSAndrew Geissler 6382c905dcSAndrew Geisslerdef npm_integrity(integrity): 6482c905dcSAndrew Geissler """ 6582c905dcSAndrew Geissler Get the checksum name and expected value from the subresource integrity 6682c905dcSAndrew Geissler https://www.w3.org/TR/SRI/ 6782c905dcSAndrew Geissler """ 6882c905dcSAndrew Geissler algo, value = integrity.split("-", maxsplit=1) 6982c905dcSAndrew Geissler return "%ssum" % algo, base64.b64decode(value).hex() 7082c905dcSAndrew Geissler 7182c905dcSAndrew Geisslerdef npm_unpack(tarball, destdir, d): 7282c905dcSAndrew Geissler """Unpack a npm tarball""" 7382c905dcSAndrew Geissler bb.utils.mkdirhier(destdir) 7482c905dcSAndrew Geissler cmd = "tar --extract --gzip --file=%s" % shlex.quote(tarball) 7582c905dcSAndrew Geissler cmd += " --no-same-owner" 76eff27476SAndrew Geissler cmd += " --delay-directory-restore" 7782c905dcSAndrew Geissler cmd += " --strip-components=1" 7882c905dcSAndrew Geissler runfetchcmd(cmd, d, workdir=destdir) 79595f6308SAndrew Geissler runfetchcmd("chmod -R +X '%s'" % (destdir), d, quiet=True, workdir=destdir) 8082c905dcSAndrew Geissler 8182c905dcSAndrew Geisslerclass NpmEnvironment(object): 8282c905dcSAndrew Geissler """ 8382c905dcSAndrew Geissler Using a npm config file seems more reliable than using cli arguments. 8482c905dcSAndrew Geissler This class allows to create a controlled environment for npm commands. 8582c905dcSAndrew Geissler """ 863d983ec7SStefan Herbrechtsmeier def __init__(self, d, configs=[], npmrc=None): 8782c905dcSAndrew Geissler self.d = d 88eff27476SAndrew Geissler 89eff27476SAndrew Geissler self.user_config = tempfile.NamedTemporaryFile(mode="w", buffering=1) 90eff27476SAndrew Geissler for key, value in configs: 91eff27476SAndrew Geissler self.user_config.write("%s=%s\n" % (key, value)) 92eff27476SAndrew Geissler 93eff27476SAndrew Geissler if npmrc: 94eff27476SAndrew Geissler self.global_config_name = npmrc 95eff27476SAndrew Geissler else: 96eff27476SAndrew Geissler self.global_config_name = "/dev/null" 97eff27476SAndrew Geissler 98eff27476SAndrew Geissler def __del__(self): 99eff27476SAndrew Geissler if self.user_config: 100eff27476SAndrew Geissler self.user_config.close() 10182c905dcSAndrew Geissler 10282c905dcSAndrew Geissler def run(self, cmd, args=None, configs=None, workdir=None): 10382c905dcSAndrew Geissler """Run npm command in a controlled environment""" 10482c905dcSAndrew Geissler with tempfile.TemporaryDirectory() as tmpdir: 10582c905dcSAndrew Geissler d = bb.data.createCopy(self.d) 106*e760df85SPatrick Williams d.setVar("PATH", d.getVar("PATH")) # PATH might contain $HOME - evaluate it before patching 10782c905dcSAndrew Geissler d.setVar("HOME", tmpdir) 10882c905dcSAndrew Geissler 10982c905dcSAndrew Geissler if not workdir: 11082c905dcSAndrew Geissler workdir = tmpdir 11182c905dcSAndrew Geissler 11282c905dcSAndrew Geissler def _run(cmd): 1133d983ec7SStefan Herbrechtsmeier cmd = "NPM_CONFIG_USERCONFIG=%s " % (self.user_config.name) + cmd 114eff27476SAndrew Geissler cmd = "NPM_CONFIG_GLOBALCONFIG=%s " % (self.global_config_name) + cmd 11582c905dcSAndrew Geissler return runfetchcmd(cmd, d, workdir=workdir) 11682c905dcSAndrew Geissler 11782c905dcSAndrew Geissler if configs: 118eff27476SAndrew Geissler bb.warn("Use of configs argument of NpmEnvironment.run() function" 119eff27476SAndrew Geissler " is deprecated. Please use args argument instead.") 12082c905dcSAndrew Geissler for key, value in configs: 121eff27476SAndrew Geissler cmd += " --%s=%s" % (key, shlex.quote(value)) 12282c905dcSAndrew Geissler 12382c905dcSAndrew Geissler if args: 12482c905dcSAndrew Geissler for key, value in args: 12582c905dcSAndrew Geissler cmd += " --%s=%s" % (key, shlex.quote(value)) 12682c905dcSAndrew Geissler 12782c905dcSAndrew Geissler return _run(cmd) 128eb8dc403SDave Cobbley 129eb8dc403SDave Cobbleyclass Npm(FetchMethod): 13082c905dcSAndrew Geissler """Class to fetch a package from a npm registry""" 131eb8dc403SDave Cobbley 132eb8dc403SDave Cobbley def supports(self, ud, d): 13382c905dcSAndrew Geissler """Check if a given url can be fetched with npm""" 13482c905dcSAndrew Geissler return ud.type in ["npm"] 135eb8dc403SDave Cobbley 136eb8dc403SDave Cobbley def urldata_init(self, ud, d): 13782c905dcSAndrew Geissler """Init npm specific variables within url data""" 13882c905dcSAndrew Geissler ud.package = None 13982c905dcSAndrew Geissler ud.version = None 14082c905dcSAndrew Geissler ud.registry = None 141eb8dc403SDave Cobbley 14282c905dcSAndrew Geissler # Get the 'package' parameter 14382c905dcSAndrew Geissler if "package" in ud.parm: 14482c905dcSAndrew Geissler ud.package = ud.parm.get("package") 14582c905dcSAndrew Geissler 14682c905dcSAndrew Geissler if not ud.package: 14782c905dcSAndrew Geissler raise MissingParameterError("Parameter 'package' required", ud.url) 14882c905dcSAndrew Geissler 14982c905dcSAndrew Geissler # Get the 'version' parameter 15082c905dcSAndrew Geissler if "version" in ud.parm: 15182c905dcSAndrew Geissler ud.version = ud.parm.get("version") 15282c905dcSAndrew Geissler 153eb8dc403SDave Cobbley if not ud.version: 15482c905dcSAndrew Geissler raise MissingParameterError("Parameter 'version' required", ud.url) 155eb8dc403SDave Cobbley 15682c905dcSAndrew Geissler if not is_semver(ud.version) and not ud.version == "latest": 15782c905dcSAndrew Geissler raise ParameterError("Invalid 'version' parameter", ud.url) 158eb8dc403SDave Cobbley 15982c905dcSAndrew Geissler # Extract the 'registry' part of the url 16092b42cb3SPatrick Williams ud.registry = re.sub(r"^npm://", "https://", ud.url.split(";")[0]) 16182c905dcSAndrew Geissler 16282c905dcSAndrew Geissler # Using the 'downloadfilename' parameter as local filename 16382c905dcSAndrew Geissler # or the npm package name. 16482c905dcSAndrew Geissler if "downloadfilename" in ud.parm: 1657e0e3c0cSAndrew Geissler ud.localfile = npm_localfile(d.expand(ud.parm["downloadfilename"])) 16682c905dcSAndrew Geissler else: 16782c905dcSAndrew Geissler ud.localfile = npm_localfile(ud.package, ud.version) 16882c905dcSAndrew Geissler 16982c905dcSAndrew Geissler # Get the base 'npm' command 17082c905dcSAndrew Geissler ud.basecmd = d.getVar("FETCHCMD_npm") or "npm" 17182c905dcSAndrew Geissler 17282c905dcSAndrew Geissler # This fetcher resolves a URI from a npm package name and version and 17382c905dcSAndrew Geissler # then forwards it to a proxy fetcher. A resolve file containing the 17482c905dcSAndrew Geissler # resolved URI is created to avoid unwanted network access (if the file 17582c905dcSAndrew Geissler # already exists). The management of the donestamp file, the lockfile 17682c905dcSAndrew Geissler # and the checksums are forwarded to the proxy fetcher. 17782c905dcSAndrew Geissler ud.proxy = None 17882c905dcSAndrew Geissler ud.needdonestamp = False 17982c905dcSAndrew Geissler ud.resolvefile = self.localpath(ud, d) + ".resolved" 18082c905dcSAndrew Geissler 18182c905dcSAndrew Geissler def _resolve_proxy_url(self, ud, d): 18282c905dcSAndrew Geissler def _npm_view(): 183eff27476SAndrew Geissler args = [] 184eff27476SAndrew Geissler args.append(("json", "true")) 185eff27476SAndrew Geissler args.append(("registry", ud.registry)) 18682c905dcSAndrew Geissler pkgver = shlex.quote(ud.package + "@" + ud.version) 18782c905dcSAndrew Geissler cmd = ud.basecmd + " view %s" % pkgver 18882c905dcSAndrew Geissler env = NpmEnvironment(d) 18982c905dcSAndrew Geissler check_network_access(d, cmd, ud.registry) 190eff27476SAndrew Geissler view_string = env.run(cmd, args=args) 19182c905dcSAndrew Geissler 19282c905dcSAndrew Geissler if not view_string: 19382c905dcSAndrew Geissler raise FetchError("Unavailable package %s" % pkgver, ud.url) 19482c905dcSAndrew Geissler 19582c905dcSAndrew Geissler try: 19682c905dcSAndrew Geissler view = json.loads(view_string) 19782c905dcSAndrew Geissler 19882c905dcSAndrew Geissler error = view.get("error") 19982c905dcSAndrew Geissler if error is not None: 20082c905dcSAndrew Geissler raise FetchError(error.get("summary"), ud.url) 20182c905dcSAndrew Geissler 20282c905dcSAndrew Geissler if ud.version == "latest": 20382c905dcSAndrew Geissler bb.warn("The npm package %s is using the latest " \ 20482c905dcSAndrew Geissler "version available. This could lead to " \ 20582c905dcSAndrew Geissler "non-reproducible builds." % pkgver) 20682c905dcSAndrew Geissler elif ud.version != view.get("version"): 20782c905dcSAndrew Geissler raise ParameterError("Invalid 'version' parameter", ud.url) 20882c905dcSAndrew Geissler 20982c905dcSAndrew Geissler return view 21082c905dcSAndrew Geissler 21182c905dcSAndrew Geissler except Exception as e: 21282c905dcSAndrew Geissler raise FetchError("Invalid view from npm: %s" % str(e), ud.url) 21382c905dcSAndrew Geissler 21482c905dcSAndrew Geissler def _get_url(view): 21582c905dcSAndrew Geissler tarball_url = view.get("dist", {}).get("tarball") 21682c905dcSAndrew Geissler 21782c905dcSAndrew Geissler if tarball_url is None: 21882c905dcSAndrew Geissler raise FetchError("Invalid 'dist.tarball' in view", ud.url) 21982c905dcSAndrew Geissler 22082c905dcSAndrew Geissler uri = URI(tarball_url) 22182c905dcSAndrew Geissler uri.params["downloadfilename"] = ud.localfile 22282c905dcSAndrew Geissler 22382c905dcSAndrew Geissler integrity = view.get("dist", {}).get("integrity") 22482c905dcSAndrew Geissler shasum = view.get("dist", {}).get("shasum") 22582c905dcSAndrew Geissler 22682c905dcSAndrew Geissler if integrity is not None: 22782c905dcSAndrew Geissler checksum_name, checksum_expected = npm_integrity(integrity) 22882c905dcSAndrew Geissler uri.params[checksum_name] = checksum_expected 22982c905dcSAndrew Geissler elif shasum is not None: 23082c905dcSAndrew Geissler uri.params["sha1sum"] = shasum 23182c905dcSAndrew Geissler else: 23282c905dcSAndrew Geissler raise FetchError("Invalid 'dist.integrity' in view", ud.url) 23382c905dcSAndrew Geissler 23482c905dcSAndrew Geissler return str(uri) 23582c905dcSAndrew Geissler 23682c905dcSAndrew Geissler url = _get_url(_npm_view()) 23782c905dcSAndrew Geissler 23882c905dcSAndrew Geissler bb.utils.mkdirhier(os.path.dirname(ud.resolvefile)) 23982c905dcSAndrew Geissler with open(ud.resolvefile, "w") as f: 24082c905dcSAndrew Geissler f.write(url) 24182c905dcSAndrew Geissler 24282c905dcSAndrew Geissler def _setup_proxy(self, ud, d): 24382c905dcSAndrew Geissler if ud.proxy is None: 24482c905dcSAndrew Geissler if not os.path.exists(ud.resolvefile): 24582c905dcSAndrew Geissler self._resolve_proxy_url(ud, d) 24682c905dcSAndrew Geissler 24782c905dcSAndrew Geissler with open(ud.resolvefile, "r") as f: 24882c905dcSAndrew Geissler url = f.read() 24982c905dcSAndrew Geissler 25082c905dcSAndrew Geissler # Avoid conflicts between the environment data and: 25182c905dcSAndrew Geissler # - the proxy url checksum 25282c905dcSAndrew Geissler data = bb.data.createCopy(d) 25382c905dcSAndrew Geissler data.delVarFlags("SRC_URI") 25482c905dcSAndrew Geissler ud.proxy = Fetch([url], data) 25582c905dcSAndrew Geissler 25682c905dcSAndrew Geissler def _get_proxy_method(self, ud, d): 25782c905dcSAndrew Geissler self._setup_proxy(ud, d) 25882c905dcSAndrew Geissler proxy_url = ud.proxy.urls[0] 25982c905dcSAndrew Geissler proxy_ud = ud.proxy.ud[proxy_url] 26082c905dcSAndrew Geissler proxy_d = ud.proxy.d 26182c905dcSAndrew Geissler proxy_ud.setup_localpath(proxy_d) 26282c905dcSAndrew Geissler return proxy_ud.method, proxy_ud, proxy_d 26382c905dcSAndrew Geissler 26482c905dcSAndrew Geissler def verify_donestamp(self, ud, d): 26582c905dcSAndrew Geissler """Verify the donestamp file""" 26682c905dcSAndrew Geissler proxy_m, proxy_ud, proxy_d = self._get_proxy_method(ud, d) 26782c905dcSAndrew Geissler return proxy_m.verify_donestamp(proxy_ud, proxy_d) 26882c905dcSAndrew Geissler 26982c905dcSAndrew Geissler def update_donestamp(self, ud, d): 27082c905dcSAndrew Geissler """Update the donestamp file""" 27182c905dcSAndrew Geissler proxy_m, proxy_ud, proxy_d = self._get_proxy_method(ud, d) 27282c905dcSAndrew Geissler proxy_m.update_donestamp(proxy_ud, proxy_d) 273eb8dc403SDave Cobbley 274eb8dc403SDave Cobbley def need_update(self, ud, d): 27582c905dcSAndrew Geissler """Force a fetch, even if localpath exists ?""" 27682c905dcSAndrew Geissler if not os.path.exists(ud.resolvefile): 277eb8dc403SDave Cobbley return True 27882c905dcSAndrew Geissler if ud.version == "latest": 27982c905dcSAndrew Geissler return True 28082c905dcSAndrew Geissler proxy_m, proxy_ud, proxy_d = self._get_proxy_method(ud, d) 28182c905dcSAndrew Geissler return proxy_m.need_update(proxy_ud, proxy_d) 282eb8dc403SDave Cobbley 28382c905dcSAndrew Geissler def try_mirrors(self, fetch, ud, d, mirrors): 28482c905dcSAndrew Geissler """Try to use a mirror""" 28582c905dcSAndrew Geissler proxy_m, proxy_ud, proxy_d = self._get_proxy_method(ud, d) 28682c905dcSAndrew Geissler return proxy_m.try_mirrors(fetch, proxy_ud, proxy_d, mirrors) 287eb8dc403SDave Cobbley 288eb8dc403SDave Cobbley def download(self, ud, d): 289eb8dc403SDave Cobbley """Fetch url""" 29082c905dcSAndrew Geissler self._setup_proxy(ud, d) 29182c905dcSAndrew Geissler ud.proxy.download() 292eb8dc403SDave Cobbley 29382c905dcSAndrew Geissler def unpack(self, ud, rootdir, d): 29482c905dcSAndrew Geissler """Unpack the downloaded archive""" 29582c905dcSAndrew Geissler destsuffix = ud.parm.get("destsuffix", "npm") 29682c905dcSAndrew Geissler destdir = os.path.join(rootdir, destsuffix) 29782c905dcSAndrew Geissler npm_unpack(ud.localpath, destdir, d) 298eb8dc403SDave Cobbley 29982c905dcSAndrew Geissler def clean(self, ud, d): 30082c905dcSAndrew Geissler """Clean any existing full or partial download""" 30182c905dcSAndrew Geissler if os.path.exists(ud.resolvefile): 30282c905dcSAndrew Geissler self._setup_proxy(ud, d) 30382c905dcSAndrew Geissler ud.proxy.clean() 30482c905dcSAndrew Geissler bb.utils.remove(ud.resolvefile) 305eb8dc403SDave Cobbley 30682c905dcSAndrew Geissler def done(self, ud, d): 30782c905dcSAndrew Geissler """Is the download done ?""" 30882c905dcSAndrew Geissler if not os.path.exists(ud.resolvefile): 30982c905dcSAndrew Geissler return False 31082c905dcSAndrew Geissler proxy_m, proxy_ud, proxy_d = self._get_proxy_method(ud, d) 31182c905dcSAndrew Geissler return proxy_m.done(proxy_ud, proxy_d) 312