182c905dcSAndrew Geissler# Copyright (C) 2020 Savoir-Faire Linux 2c342db35SBrad Bishop# 3c342db35SBrad Bishop# SPDX-License-Identifier: GPL-2.0-only 4c342db35SBrad Bishop# 5eb8dc403SDave Cobbley""" 682c905dcSAndrew GeisslerBitBake 'Fetch' npm implementation 7eb8dc403SDave Cobbley 882c905dcSAndrew Geisslernpm fetcher support the SRC_URI with format of: 982c905dcSAndrew GeisslerSRC_URI = "npm://some.registry.url;OptionA=xxx;OptionB=xxx;..." 10eb8dc403SDave Cobbley 1182c905dcSAndrew GeisslerSupported SRC_URI options are: 12eb8dc403SDave Cobbley 1382c905dcSAndrew Geissler- package 1482c905dcSAndrew Geissler The npm package name. This is a mandatory parameter. 15eb8dc403SDave Cobbley 16eb8dc403SDave Cobbley- version 1782c905dcSAndrew Geissler The npm package version. This is a mandatory parameter. 18eb8dc403SDave Cobbley 1982c905dcSAndrew Geissler- downloadfilename 2082c905dcSAndrew Geissler Specifies the filename used when storing the downloaded file. 21eb8dc403SDave Cobbley 2282c905dcSAndrew Geissler- destsuffix 2382c905dcSAndrew Geissler Specifies the directory to use to unpack the package (default: npm). 24eb8dc403SDave Cobbley""" 25eb8dc403SDave Cobbley 2682c905dcSAndrew Geisslerimport base64 27eb8dc403SDave Cobbleyimport json 2882c905dcSAndrew Geisslerimport os 2982c905dcSAndrew Geisslerimport re 3082c905dcSAndrew Geisslerimport shlex 3182c905dcSAndrew Geisslerimport tempfile 32eb8dc403SDave Cobbleyimport bb 3382c905dcSAndrew Geisslerfrom bb.fetch2 import Fetch 34eb8dc403SDave Cobbleyfrom bb.fetch2 import FetchError 3582c905dcSAndrew Geisslerfrom bb.fetch2 import FetchMethod 3682c905dcSAndrew Geisslerfrom bb.fetch2 import MissingParameterError 37eb8dc403SDave Cobbleyfrom bb.fetch2 import ParameterError 3882c905dcSAndrew Geisslerfrom bb.fetch2 import URI 3982c905dcSAndrew Geisslerfrom bb.fetch2 import check_network_access 4082c905dcSAndrew Geisslerfrom bb.fetch2 import runfetchcmd 4182c905dcSAndrew Geisslerfrom bb.utils import is_semver 42eb8dc403SDave Cobbley 4382c905dcSAndrew Geisslerdef npm_package(package): 4482c905dcSAndrew Geissler """Convert the npm package name to remove unsupported character""" 4582c905dcSAndrew Geissler # Scoped package names (with the @) use the same naming convention 4682c905dcSAndrew Geissler # as the 'npm pack' command. 4782c905dcSAndrew Geissler if package.startswith("@"): 4882c905dcSAndrew Geissler return re.sub("/", "-", package[1:]) 4982c905dcSAndrew Geissler return package 5082c905dcSAndrew Geissler 5182c905dcSAndrew Geisslerdef npm_filename(package, version): 5282c905dcSAndrew Geissler """Get the filename of a npm package""" 5382c905dcSAndrew Geissler return npm_package(package) + "-" + version + ".tgz" 5482c905dcSAndrew Geissler 5582c905dcSAndrew Geisslerdef npm_localfile(package, version): 5682c905dcSAndrew Geissler """Get the local filename of a npm package""" 5782c905dcSAndrew Geissler return os.path.join("npm2", npm_filename(package, version)) 5882c905dcSAndrew Geissler 5982c905dcSAndrew Geisslerdef npm_integrity(integrity): 6082c905dcSAndrew Geissler """ 6182c905dcSAndrew Geissler Get the checksum name and expected value from the subresource integrity 6282c905dcSAndrew Geissler https://www.w3.org/TR/SRI/ 6382c905dcSAndrew Geissler """ 6482c905dcSAndrew Geissler algo, value = integrity.split("-", maxsplit=1) 6582c905dcSAndrew Geissler return "%ssum" % algo, base64.b64decode(value).hex() 6682c905dcSAndrew Geissler 6782c905dcSAndrew Geisslerdef npm_unpack(tarball, destdir, d): 6882c905dcSAndrew Geissler """Unpack a npm tarball""" 6982c905dcSAndrew Geissler bb.utils.mkdirhier(destdir) 7082c905dcSAndrew Geissler cmd = "tar --extract --gzip --file=%s" % shlex.quote(tarball) 7182c905dcSAndrew Geissler cmd += " --no-same-owner" 72*eff27476SAndrew Geissler cmd += " --delay-directory-restore" 7382c905dcSAndrew Geissler cmd += " --strip-components=1" 7482c905dcSAndrew Geissler runfetchcmd(cmd, d, workdir=destdir) 75*eff27476SAndrew Geissler runfetchcmd("chmod -R +X %s" % (destdir), d, quiet=True, workdir=destdir) 7682c905dcSAndrew Geissler 7782c905dcSAndrew Geisslerclass NpmEnvironment(object): 7882c905dcSAndrew Geissler """ 7982c905dcSAndrew Geissler Using a npm config file seems more reliable than using cli arguments. 8082c905dcSAndrew Geissler This class allows to create a controlled environment for npm commands. 8182c905dcSAndrew Geissler """ 82*eff27476SAndrew Geissler def __init__(self, d, configs=None, npmrc=None): 8382c905dcSAndrew Geissler self.d = d 84*eff27476SAndrew Geissler 85*eff27476SAndrew Geissler if configs: 86*eff27476SAndrew Geissler self.user_config = tempfile.NamedTemporaryFile(mode="w", buffering=1) 87*eff27476SAndrew Geissler self.user_config_name = self.user_config.name 88*eff27476SAndrew Geissler for key, value in configs: 89*eff27476SAndrew Geissler self.user_config.write("%s=%s\n" % (key, value)) 90*eff27476SAndrew Geissler else: 91*eff27476SAndrew Geissler self.user_config_name = "/dev/null" 92*eff27476SAndrew Geissler 93*eff27476SAndrew Geissler if npmrc: 94*eff27476SAndrew Geissler self.global_config_name = npmrc 95*eff27476SAndrew Geissler else: 96*eff27476SAndrew Geissler self.global_config_name = "/dev/null" 97*eff27476SAndrew Geissler 98*eff27476SAndrew Geissler def __del__(self): 99*eff27476SAndrew Geissler if self.user_config: 100*eff27476SAndrew Geissler self.user_config.close() 10182c905dcSAndrew Geissler 10282c905dcSAndrew Geissler def run(self, cmd, args=None, configs=None, workdir=None): 10382c905dcSAndrew Geissler """Run npm command in a controlled environment""" 10482c905dcSAndrew Geissler with tempfile.TemporaryDirectory() as tmpdir: 10582c905dcSAndrew Geissler d = bb.data.createCopy(self.d) 10682c905dcSAndrew Geissler d.setVar("HOME", tmpdir) 10782c905dcSAndrew Geissler 10882c905dcSAndrew Geissler if not workdir: 10982c905dcSAndrew Geissler workdir = tmpdir 11082c905dcSAndrew Geissler 11182c905dcSAndrew Geissler def _run(cmd): 112*eff27476SAndrew Geissler cmd = "NPM_CONFIG_USERCONFIG=%s " % (self.user_config_name) + cmd 113*eff27476SAndrew Geissler cmd = "NPM_CONFIG_GLOBALCONFIG=%s " % (self.global_config_name) + cmd 11482c905dcSAndrew Geissler return runfetchcmd(cmd, d, workdir=workdir) 11582c905dcSAndrew Geissler 11682c905dcSAndrew Geissler if configs: 117*eff27476SAndrew Geissler bb.warn("Use of configs argument of NpmEnvironment.run() function" 118*eff27476SAndrew Geissler " is deprecated. Please use args argument instead.") 11982c905dcSAndrew Geissler for key, value in configs: 120*eff27476SAndrew Geissler cmd += " --%s=%s" % (key, shlex.quote(value)) 12182c905dcSAndrew Geissler 12282c905dcSAndrew Geissler if args: 12382c905dcSAndrew Geissler for key, value in args: 12482c905dcSAndrew Geissler cmd += " --%s=%s" % (key, shlex.quote(value)) 12582c905dcSAndrew Geissler 12682c905dcSAndrew Geissler return _run(cmd) 127eb8dc403SDave Cobbley 128eb8dc403SDave Cobbleyclass Npm(FetchMethod): 12982c905dcSAndrew Geissler """Class to fetch a package from a npm registry""" 130eb8dc403SDave Cobbley 131eb8dc403SDave Cobbley def supports(self, ud, d): 13282c905dcSAndrew Geissler """Check if a given url can be fetched with npm""" 13382c905dcSAndrew Geissler return ud.type in ["npm"] 134eb8dc403SDave Cobbley 135eb8dc403SDave Cobbley def urldata_init(self, ud, d): 13682c905dcSAndrew Geissler """Init npm specific variables within url data""" 13782c905dcSAndrew Geissler ud.package = None 13882c905dcSAndrew Geissler ud.version = None 13982c905dcSAndrew Geissler ud.registry = None 140eb8dc403SDave Cobbley 14182c905dcSAndrew Geissler # Get the 'package' parameter 14282c905dcSAndrew Geissler if "package" in ud.parm: 14382c905dcSAndrew Geissler ud.package = ud.parm.get("package") 14482c905dcSAndrew Geissler 14582c905dcSAndrew Geissler if not ud.package: 14682c905dcSAndrew Geissler raise MissingParameterError("Parameter 'package' required", ud.url) 14782c905dcSAndrew Geissler 14882c905dcSAndrew Geissler # Get the 'version' parameter 14982c905dcSAndrew Geissler if "version" in ud.parm: 15082c905dcSAndrew Geissler ud.version = ud.parm.get("version") 15182c905dcSAndrew Geissler 152eb8dc403SDave Cobbley if not ud.version: 15382c905dcSAndrew Geissler raise MissingParameterError("Parameter 'version' required", ud.url) 154eb8dc403SDave Cobbley 15582c905dcSAndrew Geissler if not is_semver(ud.version) and not ud.version == "latest": 15682c905dcSAndrew Geissler raise ParameterError("Invalid 'version' parameter", ud.url) 157eb8dc403SDave Cobbley 15882c905dcSAndrew Geissler # Extract the 'registry' part of the url 15982c905dcSAndrew Geissler ud.registry = re.sub(r"^npm://", "http://", ud.url.split(";")[0]) 16082c905dcSAndrew Geissler 16182c905dcSAndrew Geissler # Using the 'downloadfilename' parameter as local filename 16282c905dcSAndrew Geissler # or the npm package name. 16382c905dcSAndrew Geissler if "downloadfilename" in ud.parm: 16482c905dcSAndrew Geissler ud.localfile = d.expand(ud.parm["downloadfilename"]) 16582c905dcSAndrew Geissler else: 16682c905dcSAndrew Geissler ud.localfile = npm_localfile(ud.package, ud.version) 16782c905dcSAndrew Geissler 16882c905dcSAndrew Geissler # Get the base 'npm' command 16982c905dcSAndrew Geissler ud.basecmd = d.getVar("FETCHCMD_npm") or "npm" 17082c905dcSAndrew Geissler 17182c905dcSAndrew Geissler # This fetcher resolves a URI from a npm package name and version and 17282c905dcSAndrew Geissler # then forwards it to a proxy fetcher. A resolve file containing the 17382c905dcSAndrew Geissler # resolved URI is created to avoid unwanted network access (if the file 17482c905dcSAndrew Geissler # already exists). The management of the donestamp file, the lockfile 17582c905dcSAndrew Geissler # and the checksums are forwarded to the proxy fetcher. 17682c905dcSAndrew Geissler ud.proxy = None 17782c905dcSAndrew Geissler ud.needdonestamp = False 17882c905dcSAndrew Geissler ud.resolvefile = self.localpath(ud, d) + ".resolved" 17982c905dcSAndrew Geissler 18082c905dcSAndrew Geissler def _resolve_proxy_url(self, ud, d): 18182c905dcSAndrew Geissler def _npm_view(): 182*eff27476SAndrew Geissler args = [] 183*eff27476SAndrew Geissler args.append(("json", "true")) 184*eff27476SAndrew Geissler args.append(("registry", ud.registry)) 18582c905dcSAndrew Geissler pkgver = shlex.quote(ud.package + "@" + ud.version) 18682c905dcSAndrew Geissler cmd = ud.basecmd + " view %s" % pkgver 18782c905dcSAndrew Geissler env = NpmEnvironment(d) 18882c905dcSAndrew Geissler check_network_access(d, cmd, ud.registry) 189*eff27476SAndrew Geissler view_string = env.run(cmd, args=args) 19082c905dcSAndrew Geissler 19182c905dcSAndrew Geissler if not view_string: 19282c905dcSAndrew Geissler raise FetchError("Unavailable package %s" % pkgver, ud.url) 19382c905dcSAndrew Geissler 19482c905dcSAndrew Geissler try: 19582c905dcSAndrew Geissler view = json.loads(view_string) 19682c905dcSAndrew Geissler 19782c905dcSAndrew Geissler error = view.get("error") 19882c905dcSAndrew Geissler if error is not None: 19982c905dcSAndrew Geissler raise FetchError(error.get("summary"), ud.url) 20082c905dcSAndrew Geissler 20182c905dcSAndrew Geissler if ud.version == "latest": 20282c905dcSAndrew Geissler bb.warn("The npm package %s is using the latest " \ 20382c905dcSAndrew Geissler "version available. This could lead to " \ 20482c905dcSAndrew Geissler "non-reproducible builds." % pkgver) 20582c905dcSAndrew Geissler elif ud.version != view.get("version"): 20682c905dcSAndrew Geissler raise ParameterError("Invalid 'version' parameter", ud.url) 20782c905dcSAndrew Geissler 20882c905dcSAndrew Geissler return view 20982c905dcSAndrew Geissler 21082c905dcSAndrew Geissler except Exception as e: 21182c905dcSAndrew Geissler raise FetchError("Invalid view from npm: %s" % str(e), ud.url) 21282c905dcSAndrew Geissler 21382c905dcSAndrew Geissler def _get_url(view): 21482c905dcSAndrew Geissler tarball_url = view.get("dist", {}).get("tarball") 21582c905dcSAndrew Geissler 21682c905dcSAndrew Geissler if tarball_url is None: 21782c905dcSAndrew Geissler raise FetchError("Invalid 'dist.tarball' in view", ud.url) 21882c905dcSAndrew Geissler 21982c905dcSAndrew Geissler uri = URI(tarball_url) 22082c905dcSAndrew Geissler uri.params["downloadfilename"] = ud.localfile 22182c905dcSAndrew Geissler 22282c905dcSAndrew Geissler integrity = view.get("dist", {}).get("integrity") 22382c905dcSAndrew Geissler shasum = view.get("dist", {}).get("shasum") 22482c905dcSAndrew Geissler 22582c905dcSAndrew Geissler if integrity is not None: 22682c905dcSAndrew Geissler checksum_name, checksum_expected = npm_integrity(integrity) 22782c905dcSAndrew Geissler uri.params[checksum_name] = checksum_expected 22882c905dcSAndrew Geissler elif shasum is not None: 22982c905dcSAndrew Geissler uri.params["sha1sum"] = shasum 23082c905dcSAndrew Geissler else: 23182c905dcSAndrew Geissler raise FetchError("Invalid 'dist.integrity' in view", ud.url) 23282c905dcSAndrew Geissler 23382c905dcSAndrew Geissler return str(uri) 23482c905dcSAndrew Geissler 23582c905dcSAndrew Geissler url = _get_url(_npm_view()) 23682c905dcSAndrew Geissler 23782c905dcSAndrew Geissler bb.utils.mkdirhier(os.path.dirname(ud.resolvefile)) 23882c905dcSAndrew Geissler with open(ud.resolvefile, "w") as f: 23982c905dcSAndrew Geissler f.write(url) 24082c905dcSAndrew Geissler 24182c905dcSAndrew Geissler def _setup_proxy(self, ud, d): 24282c905dcSAndrew Geissler if ud.proxy is None: 24382c905dcSAndrew Geissler if not os.path.exists(ud.resolvefile): 24482c905dcSAndrew Geissler self._resolve_proxy_url(ud, d) 24582c905dcSAndrew Geissler 24682c905dcSAndrew Geissler with open(ud.resolvefile, "r") as f: 24782c905dcSAndrew Geissler url = f.read() 24882c905dcSAndrew Geissler 24982c905dcSAndrew Geissler # Avoid conflicts between the environment data and: 25082c905dcSAndrew Geissler # - the proxy url checksum 25182c905dcSAndrew Geissler data = bb.data.createCopy(d) 25282c905dcSAndrew Geissler data.delVarFlags("SRC_URI") 25382c905dcSAndrew Geissler ud.proxy = Fetch([url], data) 25482c905dcSAndrew Geissler 25582c905dcSAndrew Geissler def _get_proxy_method(self, ud, d): 25682c905dcSAndrew Geissler self._setup_proxy(ud, d) 25782c905dcSAndrew Geissler proxy_url = ud.proxy.urls[0] 25882c905dcSAndrew Geissler proxy_ud = ud.proxy.ud[proxy_url] 25982c905dcSAndrew Geissler proxy_d = ud.proxy.d 26082c905dcSAndrew Geissler proxy_ud.setup_localpath(proxy_d) 26182c905dcSAndrew Geissler return proxy_ud.method, proxy_ud, proxy_d 26282c905dcSAndrew Geissler 26382c905dcSAndrew Geissler def verify_donestamp(self, ud, d): 26482c905dcSAndrew Geissler """Verify the donestamp file""" 26582c905dcSAndrew Geissler proxy_m, proxy_ud, proxy_d = self._get_proxy_method(ud, d) 26682c905dcSAndrew Geissler return proxy_m.verify_donestamp(proxy_ud, proxy_d) 26782c905dcSAndrew Geissler 26882c905dcSAndrew Geissler def update_donestamp(self, ud, d): 26982c905dcSAndrew Geissler """Update the donestamp file""" 27082c905dcSAndrew Geissler proxy_m, proxy_ud, proxy_d = self._get_proxy_method(ud, d) 27182c905dcSAndrew Geissler proxy_m.update_donestamp(proxy_ud, proxy_d) 272eb8dc403SDave Cobbley 273eb8dc403SDave Cobbley def need_update(self, ud, d): 27482c905dcSAndrew Geissler """Force a fetch, even if localpath exists ?""" 27582c905dcSAndrew Geissler if not os.path.exists(ud.resolvefile): 276eb8dc403SDave Cobbley return True 27782c905dcSAndrew Geissler if ud.version == "latest": 27882c905dcSAndrew Geissler return True 27982c905dcSAndrew Geissler proxy_m, proxy_ud, proxy_d = self._get_proxy_method(ud, d) 28082c905dcSAndrew Geissler return proxy_m.need_update(proxy_ud, proxy_d) 281eb8dc403SDave Cobbley 28282c905dcSAndrew Geissler def try_mirrors(self, fetch, ud, d, mirrors): 28382c905dcSAndrew Geissler """Try to use a mirror""" 28482c905dcSAndrew Geissler proxy_m, proxy_ud, proxy_d = self._get_proxy_method(ud, d) 28582c905dcSAndrew Geissler return proxy_m.try_mirrors(fetch, proxy_ud, proxy_d, mirrors) 286eb8dc403SDave Cobbley 287eb8dc403SDave Cobbley def download(self, ud, d): 288eb8dc403SDave Cobbley """Fetch url""" 28982c905dcSAndrew Geissler self._setup_proxy(ud, d) 29082c905dcSAndrew Geissler ud.proxy.download() 291eb8dc403SDave Cobbley 29282c905dcSAndrew Geissler def unpack(self, ud, rootdir, d): 29382c905dcSAndrew Geissler """Unpack the downloaded archive""" 29482c905dcSAndrew Geissler destsuffix = ud.parm.get("destsuffix", "npm") 29582c905dcSAndrew Geissler destdir = os.path.join(rootdir, destsuffix) 29682c905dcSAndrew Geissler npm_unpack(ud.localpath, destdir, d) 297eb8dc403SDave Cobbley 29882c905dcSAndrew Geissler def clean(self, ud, d): 29982c905dcSAndrew Geissler """Clean any existing full or partial download""" 30082c905dcSAndrew Geissler if os.path.exists(ud.resolvefile): 30182c905dcSAndrew Geissler self._setup_proxy(ud, d) 30282c905dcSAndrew Geissler ud.proxy.clean() 30382c905dcSAndrew Geissler bb.utils.remove(ud.resolvefile) 304eb8dc403SDave Cobbley 30582c905dcSAndrew Geissler def done(self, ud, d): 30682c905dcSAndrew Geissler """Is the download done ?""" 30782c905dcSAndrew Geissler if not os.path.exists(ud.resolvefile): 30882c905dcSAndrew Geissler return False 30982c905dcSAndrew Geissler proxy_m, proxy_ud, proxy_d = self._get_proxy_method(ud, d) 31082c905dcSAndrew Geissler return proxy_m.done(proxy_ud, proxy_d) 311