182c905dcSAndrew Geissler# Copyright (C) 2020 Savoir-Faire Linux 2c342db35SBrad Bishop# 3c342db35SBrad Bishop# SPDX-License-Identifier: GPL-2.0-only 4c342db35SBrad Bishop# 5eb8dc403SDave Cobbley""" 682c905dcSAndrew GeisslerBitBake 'Fetch' npm implementation 7eb8dc403SDave Cobbley 882c905dcSAndrew Geisslernpm fetcher support the SRC_URI with format of: 982c905dcSAndrew GeisslerSRC_URI = "npm://some.registry.url;OptionA=xxx;OptionB=xxx;..." 10eb8dc403SDave Cobbley 1182c905dcSAndrew GeisslerSupported SRC_URI options are: 12eb8dc403SDave Cobbley 1382c905dcSAndrew Geissler- package 1482c905dcSAndrew Geissler The npm package name. This is a mandatory parameter. 15eb8dc403SDave Cobbley 16eb8dc403SDave Cobbley- version 1782c905dcSAndrew Geissler The npm package version. This is a mandatory parameter. 18eb8dc403SDave Cobbley 1982c905dcSAndrew Geissler- downloadfilename 2082c905dcSAndrew Geissler Specifies the filename used when storing the downloaded file. 21eb8dc403SDave Cobbley 2282c905dcSAndrew Geissler- destsuffix 2382c905dcSAndrew Geissler Specifies the directory to use to unpack the package (default: npm). 24eb8dc403SDave Cobbley""" 25eb8dc403SDave Cobbley 2682c905dcSAndrew Geisslerimport base64 27eb8dc403SDave Cobbleyimport json 2882c905dcSAndrew Geisslerimport os 2982c905dcSAndrew Geisslerimport re 3082c905dcSAndrew Geisslerimport shlex 3182c905dcSAndrew Geisslerimport tempfile 32eb8dc403SDave Cobbleyimport bb 3382c905dcSAndrew Geisslerfrom bb.fetch2 import Fetch 34eb8dc403SDave Cobbleyfrom bb.fetch2 import FetchError 3582c905dcSAndrew Geisslerfrom bb.fetch2 import FetchMethod 3682c905dcSAndrew Geisslerfrom bb.fetch2 import MissingParameterError 37eb8dc403SDave Cobbleyfrom bb.fetch2 import ParameterError 3882c905dcSAndrew Geisslerfrom bb.fetch2 import URI 3982c905dcSAndrew Geisslerfrom bb.fetch2 import check_network_access 4082c905dcSAndrew Geisslerfrom bb.fetch2 import runfetchcmd 4182c905dcSAndrew Geisslerfrom bb.utils import is_semver 42eb8dc403SDave Cobbley 4382c905dcSAndrew Geisslerdef npm_package(package): 4482c905dcSAndrew Geissler """Convert the npm package name to remove unsupported character""" 4582c905dcSAndrew Geissler # Scoped package names (with the @) use the same naming convention 4682c905dcSAndrew Geissler # as the 'npm pack' command. 478f840685SAndrew Geissler name = re.sub("/", "-", package) 488f840685SAndrew Geissler name = name.lower() 498f840685SAndrew Geissler name = re.sub(r"[^\-a-z0-9]", "", name) 508f840685SAndrew Geissler name = name.strip("-") 518f840685SAndrew Geissler return name 528f840685SAndrew Geissler 5382c905dcSAndrew Geissler 5482c905dcSAndrew Geisslerdef npm_filename(package, version): 5582c905dcSAndrew Geissler """Get the filename of a npm package""" 5682c905dcSAndrew Geissler return npm_package(package) + "-" + version + ".tgz" 5782c905dcSAndrew Geissler 587e0e3c0cSAndrew Geisslerdef npm_localfile(package, version=None): 5982c905dcSAndrew Geissler """Get the local filename of a npm package""" 607e0e3c0cSAndrew Geissler if version is not None: 617e0e3c0cSAndrew Geissler filename = npm_filename(package, version) 627e0e3c0cSAndrew Geissler else: 637e0e3c0cSAndrew Geissler filename = package 647e0e3c0cSAndrew Geissler return os.path.join("npm2", filename) 6582c905dcSAndrew Geissler 6682c905dcSAndrew Geisslerdef npm_integrity(integrity): 6782c905dcSAndrew Geissler """ 6882c905dcSAndrew Geissler Get the checksum name and expected value from the subresource integrity 6982c905dcSAndrew Geissler https://www.w3.org/TR/SRI/ 7082c905dcSAndrew Geissler """ 7182c905dcSAndrew Geissler algo, value = integrity.split("-", maxsplit=1) 7282c905dcSAndrew Geissler return "%ssum" % algo, base64.b64decode(value).hex() 7382c905dcSAndrew Geissler 7482c905dcSAndrew Geisslerdef npm_unpack(tarball, destdir, d): 7582c905dcSAndrew Geissler """Unpack a npm tarball""" 7682c905dcSAndrew Geissler bb.utils.mkdirhier(destdir) 7782c905dcSAndrew Geissler cmd = "tar --extract --gzip --file=%s" % shlex.quote(tarball) 7882c905dcSAndrew Geissler cmd += " --no-same-owner" 79eff27476SAndrew Geissler cmd += " --delay-directory-restore" 8082c905dcSAndrew Geissler cmd += " --strip-components=1" 8182c905dcSAndrew Geissler runfetchcmd(cmd, d, workdir=destdir) 82595f6308SAndrew Geissler runfetchcmd("chmod -R +X '%s'" % (destdir), d, quiet=True, workdir=destdir) 8382c905dcSAndrew Geissler 8482c905dcSAndrew Geisslerclass NpmEnvironment(object): 8582c905dcSAndrew Geissler """ 8682c905dcSAndrew Geissler Using a npm config file seems more reliable than using cli arguments. 8782c905dcSAndrew Geissler This class allows to create a controlled environment for npm commands. 8882c905dcSAndrew Geissler """ 893d983ec7SStefan Herbrechtsmeier def __init__(self, d, configs=[], npmrc=None): 9082c905dcSAndrew Geissler self.d = d 91eff27476SAndrew Geissler 92eff27476SAndrew Geissler self.user_config = tempfile.NamedTemporaryFile(mode="w", buffering=1) 93eff27476SAndrew Geissler for key, value in configs: 94eff27476SAndrew Geissler self.user_config.write("%s=%s\n" % (key, value)) 95eff27476SAndrew Geissler 96eff27476SAndrew Geissler if npmrc: 97eff27476SAndrew Geissler self.global_config_name = npmrc 98eff27476SAndrew Geissler else: 99eff27476SAndrew Geissler self.global_config_name = "/dev/null" 100eff27476SAndrew Geissler 101eff27476SAndrew Geissler def __del__(self): 102eff27476SAndrew Geissler if self.user_config: 103eff27476SAndrew Geissler self.user_config.close() 10482c905dcSAndrew Geissler 10582c905dcSAndrew Geissler def run(self, cmd, args=None, configs=None, workdir=None): 10682c905dcSAndrew Geissler """Run npm command in a controlled environment""" 10782c905dcSAndrew Geissler with tempfile.TemporaryDirectory() as tmpdir: 10882c905dcSAndrew Geissler d = bb.data.createCopy(self.d) 109e760df85SPatrick Williams d.setVar("PATH", d.getVar("PATH")) # PATH might contain $HOME - evaluate it before patching 11082c905dcSAndrew Geissler d.setVar("HOME", tmpdir) 11182c905dcSAndrew Geissler 11282c905dcSAndrew Geissler if not workdir: 11382c905dcSAndrew Geissler workdir = tmpdir 11482c905dcSAndrew Geissler 11582c905dcSAndrew Geissler def _run(cmd): 1163d983ec7SStefan Herbrechtsmeier cmd = "NPM_CONFIG_USERCONFIG=%s " % (self.user_config.name) + cmd 117eff27476SAndrew Geissler cmd = "NPM_CONFIG_GLOBALCONFIG=%s " % (self.global_config_name) + cmd 11882c905dcSAndrew Geissler return runfetchcmd(cmd, d, workdir=workdir) 11982c905dcSAndrew Geissler 12082c905dcSAndrew Geissler if configs: 121eff27476SAndrew Geissler bb.warn("Use of configs argument of NpmEnvironment.run() function" 122eff27476SAndrew Geissler " is deprecated. Please use args argument instead.") 12382c905dcSAndrew Geissler for key, value in configs: 124eff27476SAndrew Geissler cmd += " --%s=%s" % (key, shlex.quote(value)) 12582c905dcSAndrew Geissler 12682c905dcSAndrew Geissler if args: 12782c905dcSAndrew Geissler for key, value in args: 12882c905dcSAndrew Geissler cmd += " --%s=%s" % (key, shlex.quote(value)) 12982c905dcSAndrew Geissler 13082c905dcSAndrew Geissler return _run(cmd) 131eb8dc403SDave Cobbley 132eb8dc403SDave Cobbleyclass Npm(FetchMethod): 13382c905dcSAndrew Geissler """Class to fetch a package from a npm registry""" 134eb8dc403SDave Cobbley 135eb8dc403SDave Cobbley def supports(self, ud, d): 13682c905dcSAndrew Geissler """Check if a given url can be fetched with npm""" 13782c905dcSAndrew Geissler return ud.type in ["npm"] 138eb8dc403SDave Cobbley 139eb8dc403SDave Cobbley def urldata_init(self, ud, d): 14082c905dcSAndrew Geissler """Init npm specific variables within url data""" 14182c905dcSAndrew Geissler ud.package = None 14282c905dcSAndrew Geissler ud.version = None 14382c905dcSAndrew Geissler ud.registry = None 144eb8dc403SDave Cobbley 14582c905dcSAndrew Geissler # Get the 'package' parameter 14682c905dcSAndrew Geissler if "package" in ud.parm: 14782c905dcSAndrew Geissler ud.package = ud.parm.get("package") 14882c905dcSAndrew Geissler 14982c905dcSAndrew Geissler if not ud.package: 15082c905dcSAndrew Geissler raise MissingParameterError("Parameter 'package' required", ud.url) 15182c905dcSAndrew Geissler 15282c905dcSAndrew Geissler # Get the 'version' parameter 15382c905dcSAndrew Geissler if "version" in ud.parm: 15482c905dcSAndrew Geissler ud.version = ud.parm.get("version") 15582c905dcSAndrew Geissler 156eb8dc403SDave Cobbley if not ud.version: 15782c905dcSAndrew Geissler raise MissingParameterError("Parameter 'version' required", ud.url) 158eb8dc403SDave Cobbley 15982c905dcSAndrew Geissler if not is_semver(ud.version) and not ud.version == "latest": 16082c905dcSAndrew Geissler raise ParameterError("Invalid 'version' parameter", ud.url) 161eb8dc403SDave Cobbley 16282c905dcSAndrew Geissler # Extract the 'registry' part of the url 16392b42cb3SPatrick Williams ud.registry = re.sub(r"^npm://", "https://", ud.url.split(";")[0]) 16482c905dcSAndrew Geissler 16582c905dcSAndrew Geissler # Using the 'downloadfilename' parameter as local filename 16682c905dcSAndrew Geissler # or the npm package name. 16782c905dcSAndrew Geissler if "downloadfilename" in ud.parm: 1687e0e3c0cSAndrew Geissler ud.localfile = npm_localfile(d.expand(ud.parm["downloadfilename"])) 16982c905dcSAndrew Geissler else: 17082c905dcSAndrew Geissler ud.localfile = npm_localfile(ud.package, ud.version) 17182c905dcSAndrew Geissler 17282c905dcSAndrew Geissler # Get the base 'npm' command 17382c905dcSAndrew Geissler ud.basecmd = d.getVar("FETCHCMD_npm") or "npm" 17482c905dcSAndrew Geissler 17582c905dcSAndrew Geissler # This fetcher resolves a URI from a npm package name and version and 17682c905dcSAndrew Geissler # then forwards it to a proxy fetcher. A resolve file containing the 17782c905dcSAndrew Geissler # resolved URI is created to avoid unwanted network access (if the file 17882c905dcSAndrew Geissler # already exists). The management of the donestamp file, the lockfile 17982c905dcSAndrew Geissler # and the checksums are forwarded to the proxy fetcher. 18082c905dcSAndrew Geissler ud.proxy = None 18182c905dcSAndrew Geissler ud.needdonestamp = False 18282c905dcSAndrew Geissler ud.resolvefile = self.localpath(ud, d) + ".resolved" 18382c905dcSAndrew Geissler 18482c905dcSAndrew Geissler def _resolve_proxy_url(self, ud, d): 18582c905dcSAndrew Geissler def _npm_view(): 186eff27476SAndrew Geissler args = [] 187eff27476SAndrew Geissler args.append(("json", "true")) 188eff27476SAndrew Geissler args.append(("registry", ud.registry)) 18982c905dcSAndrew Geissler pkgver = shlex.quote(ud.package + "@" + ud.version) 19082c905dcSAndrew Geissler cmd = ud.basecmd + " view %s" % pkgver 19182c905dcSAndrew Geissler env = NpmEnvironment(d) 19282c905dcSAndrew Geissler check_network_access(d, cmd, ud.registry) 193eff27476SAndrew Geissler view_string = env.run(cmd, args=args) 19482c905dcSAndrew Geissler 19582c905dcSAndrew Geissler if not view_string: 19682c905dcSAndrew Geissler raise FetchError("Unavailable package %s" % pkgver, ud.url) 19782c905dcSAndrew Geissler 19882c905dcSAndrew Geissler try: 19982c905dcSAndrew Geissler view = json.loads(view_string) 20082c905dcSAndrew Geissler 20182c905dcSAndrew Geissler error = view.get("error") 20282c905dcSAndrew Geissler if error is not None: 20382c905dcSAndrew Geissler raise FetchError(error.get("summary"), ud.url) 20482c905dcSAndrew Geissler 20582c905dcSAndrew Geissler if ud.version == "latest": 20682c905dcSAndrew Geissler bb.warn("The npm package %s is using the latest " \ 20782c905dcSAndrew Geissler "version available. This could lead to " \ 20882c905dcSAndrew Geissler "non-reproducible builds." % pkgver) 20982c905dcSAndrew Geissler elif ud.version != view.get("version"): 21082c905dcSAndrew Geissler raise ParameterError("Invalid 'version' parameter", ud.url) 21182c905dcSAndrew Geissler 21282c905dcSAndrew Geissler return view 21382c905dcSAndrew Geissler 21482c905dcSAndrew Geissler except Exception as e: 21582c905dcSAndrew Geissler raise FetchError("Invalid view from npm: %s" % str(e), ud.url) 21682c905dcSAndrew Geissler 21782c905dcSAndrew Geissler def _get_url(view): 21882c905dcSAndrew Geissler tarball_url = view.get("dist", {}).get("tarball") 21982c905dcSAndrew Geissler 22082c905dcSAndrew Geissler if tarball_url is None: 22182c905dcSAndrew Geissler raise FetchError("Invalid 'dist.tarball' in view", ud.url) 22282c905dcSAndrew Geissler 22382c905dcSAndrew Geissler uri = URI(tarball_url) 22482c905dcSAndrew Geissler uri.params["downloadfilename"] = ud.localfile 22582c905dcSAndrew Geissler 22682c905dcSAndrew Geissler integrity = view.get("dist", {}).get("integrity") 22782c905dcSAndrew Geissler shasum = view.get("dist", {}).get("shasum") 22882c905dcSAndrew Geissler 22982c905dcSAndrew Geissler if integrity is not None: 23082c905dcSAndrew Geissler checksum_name, checksum_expected = npm_integrity(integrity) 23182c905dcSAndrew Geissler uri.params[checksum_name] = checksum_expected 23282c905dcSAndrew Geissler elif shasum is not None: 23382c905dcSAndrew Geissler uri.params["sha1sum"] = shasum 23482c905dcSAndrew Geissler else: 23582c905dcSAndrew Geissler raise FetchError("Invalid 'dist.integrity' in view", ud.url) 23682c905dcSAndrew Geissler 23782c905dcSAndrew Geissler return str(uri) 23882c905dcSAndrew Geissler 23982c905dcSAndrew Geissler url = _get_url(_npm_view()) 24082c905dcSAndrew Geissler 24182c905dcSAndrew Geissler bb.utils.mkdirhier(os.path.dirname(ud.resolvefile)) 24282c905dcSAndrew Geissler with open(ud.resolvefile, "w") as f: 24382c905dcSAndrew Geissler f.write(url) 24482c905dcSAndrew Geissler 24582c905dcSAndrew Geissler def _setup_proxy(self, ud, d): 24682c905dcSAndrew Geissler if ud.proxy is None: 24782c905dcSAndrew Geissler if not os.path.exists(ud.resolvefile): 24882c905dcSAndrew Geissler self._resolve_proxy_url(ud, d) 24982c905dcSAndrew Geissler 25082c905dcSAndrew Geissler with open(ud.resolvefile, "r") as f: 25182c905dcSAndrew Geissler url = f.read() 25282c905dcSAndrew Geissler 25382c905dcSAndrew Geissler # Avoid conflicts between the environment data and: 25482c905dcSAndrew Geissler # - the proxy url checksum 25582c905dcSAndrew Geissler data = bb.data.createCopy(d) 25682c905dcSAndrew Geissler data.delVarFlags("SRC_URI") 25782c905dcSAndrew Geissler ud.proxy = Fetch([url], data) 25882c905dcSAndrew Geissler 25982c905dcSAndrew Geissler def _get_proxy_method(self, ud, d): 26082c905dcSAndrew Geissler self._setup_proxy(ud, d) 26182c905dcSAndrew Geissler proxy_url = ud.proxy.urls[0] 26282c905dcSAndrew Geissler proxy_ud = ud.proxy.ud[proxy_url] 26382c905dcSAndrew Geissler proxy_d = ud.proxy.d 26482c905dcSAndrew Geissler proxy_ud.setup_localpath(proxy_d) 26582c905dcSAndrew Geissler return proxy_ud.method, proxy_ud, proxy_d 26682c905dcSAndrew Geissler 26782c905dcSAndrew Geissler def verify_donestamp(self, ud, d): 26882c905dcSAndrew Geissler """Verify the donestamp file""" 26982c905dcSAndrew Geissler proxy_m, proxy_ud, proxy_d = self._get_proxy_method(ud, d) 27082c905dcSAndrew Geissler return proxy_m.verify_donestamp(proxy_ud, proxy_d) 27182c905dcSAndrew Geissler 27282c905dcSAndrew Geissler def update_donestamp(self, ud, d): 27382c905dcSAndrew Geissler """Update the donestamp file""" 27482c905dcSAndrew Geissler proxy_m, proxy_ud, proxy_d = self._get_proxy_method(ud, d) 27582c905dcSAndrew Geissler proxy_m.update_donestamp(proxy_ud, proxy_d) 276eb8dc403SDave Cobbley 277eb8dc403SDave Cobbley def need_update(self, ud, d): 27882c905dcSAndrew Geissler """Force a fetch, even if localpath exists ?""" 27982c905dcSAndrew Geissler if not os.path.exists(ud.resolvefile): 280eb8dc403SDave Cobbley return True 28182c905dcSAndrew Geissler if ud.version == "latest": 28282c905dcSAndrew Geissler return True 28382c905dcSAndrew Geissler proxy_m, proxy_ud, proxy_d = self._get_proxy_method(ud, d) 28482c905dcSAndrew Geissler return proxy_m.need_update(proxy_ud, proxy_d) 285eb8dc403SDave Cobbley 28682c905dcSAndrew Geissler def try_mirrors(self, fetch, ud, d, mirrors): 28782c905dcSAndrew Geissler """Try to use a mirror""" 28882c905dcSAndrew Geissler proxy_m, proxy_ud, proxy_d = self._get_proxy_method(ud, d) 28982c905dcSAndrew Geissler return proxy_m.try_mirrors(fetch, proxy_ud, proxy_d, mirrors) 290eb8dc403SDave Cobbley 291eb8dc403SDave Cobbley def download(self, ud, d): 292eb8dc403SDave Cobbley """Fetch url""" 29382c905dcSAndrew Geissler self._setup_proxy(ud, d) 29482c905dcSAndrew Geissler ud.proxy.download() 295eb8dc403SDave Cobbley 29682c905dcSAndrew Geissler def unpack(self, ud, rootdir, d): 29782c905dcSAndrew Geissler """Unpack the downloaded archive""" 29882c905dcSAndrew Geissler destsuffix = ud.parm.get("destsuffix", "npm") 29982c905dcSAndrew Geissler destdir = os.path.join(rootdir, destsuffix) 30082c905dcSAndrew Geissler npm_unpack(ud.localpath, destdir, d) 301*ac13d5f3SPatrick Williams ud.unpack_tracer.unpack("npm", destdir) 302eb8dc403SDave Cobbley 30382c905dcSAndrew Geissler def clean(self, ud, d): 30482c905dcSAndrew Geissler """Clean any existing full or partial download""" 30582c905dcSAndrew Geissler if os.path.exists(ud.resolvefile): 30682c905dcSAndrew Geissler self._setup_proxy(ud, d) 30782c905dcSAndrew Geissler ud.proxy.clean() 30882c905dcSAndrew Geissler bb.utils.remove(ud.resolvefile) 309eb8dc403SDave Cobbley 31082c905dcSAndrew Geissler def done(self, ud, d): 31182c905dcSAndrew Geissler """Is the download done ?""" 31282c905dcSAndrew Geissler if not os.path.exists(ud.resolvefile): 31382c905dcSAndrew Geissler return False 31482c905dcSAndrew Geissler proxy_m, proxy_ud, proxy_d = self._get_proxy_method(ud, d) 31582c905dcSAndrew Geissler return proxy_m.done(proxy_ud, proxy_d) 316