182c905dcSAndrew Geissler# Copyright (C) 2020 Savoir-Faire Linux 282c905dcSAndrew Geissler# 382c905dcSAndrew Geissler# SPDX-License-Identifier: GPL-2.0-only 482c905dcSAndrew Geissler# 582c905dcSAndrew Geissler""" 682c905dcSAndrew GeisslerBitBake 'Fetch' npm shrinkwrap implementation 782c905dcSAndrew Geissler 882c905dcSAndrew Geisslernpm fetcher support the SRC_URI with format of: 982c905dcSAndrew GeisslerSRC_URI = "npmsw://some.registry.url;OptionA=xxx;OptionB=xxx;..." 1082c905dcSAndrew Geissler 1182c905dcSAndrew GeisslerSupported SRC_URI options are: 1282c905dcSAndrew Geissler 1382c905dcSAndrew Geissler- dev 1482c905dcSAndrew Geissler Set to 1 to also install devDependencies. 1582c905dcSAndrew Geissler 1682c905dcSAndrew Geissler- destsuffix 1782c905dcSAndrew Geissler Specifies the directory to use to unpack the dependencies (default: ${S}). 1882c905dcSAndrew Geissler""" 1982c905dcSAndrew Geissler 2082c905dcSAndrew Geisslerimport json 2182c905dcSAndrew Geisslerimport os 2282c905dcSAndrew Geisslerimport re 2382c905dcSAndrew Geisslerimport bb 2482c905dcSAndrew Geisslerfrom bb.fetch2 import Fetch 2582c905dcSAndrew Geisslerfrom bb.fetch2 import FetchMethod 2682c905dcSAndrew Geisslerfrom bb.fetch2 import ParameterError 27eff27476SAndrew Geisslerfrom bb.fetch2 import runfetchcmd 2882c905dcSAndrew Geisslerfrom bb.fetch2 import URI 2982c905dcSAndrew Geisslerfrom bb.fetch2.npm import npm_integrity 3082c905dcSAndrew Geisslerfrom bb.fetch2.npm import npm_localfile 3182c905dcSAndrew Geisslerfrom bb.fetch2.npm import npm_unpack 3282c905dcSAndrew Geisslerfrom bb.utils import is_semver 33eff27476SAndrew Geisslerfrom bb.utils import lockfile 34eff27476SAndrew Geisslerfrom bb.utils import unlockfile 3582c905dcSAndrew Geissler 3682c905dcSAndrew Geisslerdef foreach_dependencies(shrinkwrap, callback=None, dev=False): 3782c905dcSAndrew Geissler """ 3882c905dcSAndrew Geissler Run a callback for each dependencies of a shrinkwrap file. 3982c905dcSAndrew Geissler The callback is using the format: 4082c905dcSAndrew Geissler callback(name, params, deptree) 4182c905dcSAndrew Geissler with: 4282c905dcSAndrew Geissler name = the package name (string) 4382c905dcSAndrew Geissler params = the package parameters (dictionary) 448f840685SAndrew Geissler destdir = the destination of the package (string) 4582c905dcSAndrew Geissler """ 468f840685SAndrew Geissler # For handling old style dependencies entries in shinkwrap files 4782c905dcSAndrew Geissler def _walk_deps(deps, deptree): 4882c905dcSAndrew Geissler for name in deps: 4982c905dcSAndrew Geissler subtree = [*deptree, name] 5082c905dcSAndrew Geissler _walk_deps(deps[name].get("dependencies", {}), subtree) 5182c905dcSAndrew Geissler if callback is not None: 5282c905dcSAndrew Geissler if deps[name].get("dev", False) and not dev: 5382c905dcSAndrew Geissler continue 5482c905dcSAndrew Geissler elif deps[name].get("bundled", False): 5582c905dcSAndrew Geissler continue 568f840685SAndrew Geissler destsubdirs = [os.path.join("node_modules", dep) for dep in subtree] 578f840685SAndrew Geissler destsuffix = os.path.join(*destsubdirs) 588f840685SAndrew Geissler callback(name, deps[name], destsuffix) 5982c905dcSAndrew Geissler 608f840685SAndrew Geissler # packages entry means new style shrinkwrap file, else use dependencies 618f840685SAndrew Geissler packages = shrinkwrap.get("packages", None) 628f840685SAndrew Geissler if packages is not None: 638f840685SAndrew Geissler for package in packages: 648f840685SAndrew Geissler if package != "": 658f840685SAndrew Geissler name = package.split('node_modules/')[-1] 668f840685SAndrew Geissler package_infos = packages.get(package, {}) 678f840685SAndrew Geissler if dev == False and package_infos.get("dev", False): 688f840685SAndrew Geissler continue 698f840685SAndrew Geissler callback(name, package_infos, package) 708f840685SAndrew Geissler else: 7182c905dcSAndrew Geissler _walk_deps(shrinkwrap.get("dependencies", {}), []) 7282c905dcSAndrew Geissler 7382c905dcSAndrew Geisslerclass NpmShrinkWrap(FetchMethod): 7482c905dcSAndrew Geissler """Class to fetch all package from a shrinkwrap file""" 7582c905dcSAndrew Geissler 7682c905dcSAndrew Geissler def supports(self, ud, d): 7782c905dcSAndrew Geissler """Check if a given url can be fetched with npmsw""" 7882c905dcSAndrew Geissler return ud.type in ["npmsw"] 7982c905dcSAndrew Geissler 8082c905dcSAndrew Geissler def urldata_init(self, ud, d): 8182c905dcSAndrew Geissler """Init npmsw specific variables within url data""" 8282c905dcSAndrew Geissler 8382c905dcSAndrew Geissler # Get the 'shrinkwrap' parameter 8482c905dcSAndrew Geissler ud.shrinkwrap_file = re.sub(r"^npmsw://", "", ud.url.split(";")[0]) 8582c905dcSAndrew Geissler 8682c905dcSAndrew Geissler # Get the 'dev' parameter 8782c905dcSAndrew Geissler ud.dev = bb.utils.to_boolean(ud.parm.get("dev"), False) 8882c905dcSAndrew Geissler 8982c905dcSAndrew Geissler # Resolve the dependencies 9082c905dcSAndrew Geissler ud.deps = [] 9182c905dcSAndrew Geissler 928f840685SAndrew Geissler def _resolve_dependency(name, params, destsuffix): 9382c905dcSAndrew Geissler url = None 9482c905dcSAndrew Geissler localpath = None 9582c905dcSAndrew Geissler extrapaths = [] 96eff27476SAndrew Geissler unpack = True 9782c905dcSAndrew Geissler 9882c905dcSAndrew Geissler integrity = params.get("integrity", None) 9982c905dcSAndrew Geissler resolved = params.get("resolved", None) 10082c905dcSAndrew Geissler version = params.get("version", None) 10182c905dcSAndrew Geissler 10282c905dcSAndrew Geissler # Handle registry sources 103595f6308SAndrew Geissler if is_semver(version) and integrity: 104595f6308SAndrew Geissler # Handle duplicate dependencies without url 105595f6308SAndrew Geissler if not resolved: 106595f6308SAndrew Geissler return 107595f6308SAndrew Geissler 10882c905dcSAndrew Geissler localfile = npm_localfile(name, version) 10982c905dcSAndrew Geissler 11082c905dcSAndrew Geissler uri = URI(resolved) 11182c905dcSAndrew Geissler uri.params["downloadfilename"] = localfile 11282c905dcSAndrew Geissler 11382c905dcSAndrew Geissler checksum_name, checksum_expected = npm_integrity(integrity) 11482c905dcSAndrew Geissler uri.params[checksum_name] = checksum_expected 11582c905dcSAndrew Geissler 11682c905dcSAndrew Geissler url = str(uri) 11782c905dcSAndrew Geissler 11882c905dcSAndrew Geissler localpath = os.path.join(d.getVar("DL_DIR"), localfile) 11982c905dcSAndrew Geissler 12082c905dcSAndrew Geissler # Create a resolve file to mimic the npm fetcher and allow 12182c905dcSAndrew Geissler # re-usability of the downloaded file. 12282c905dcSAndrew Geissler resolvefile = localpath + ".resolved" 12382c905dcSAndrew Geissler 12482c905dcSAndrew Geissler bb.utils.mkdirhier(os.path.dirname(resolvefile)) 12582c905dcSAndrew Geissler with open(resolvefile, "w") as f: 12682c905dcSAndrew Geissler f.write(url) 12782c905dcSAndrew Geissler 12882c905dcSAndrew Geissler extrapaths.append(resolvefile) 12982c905dcSAndrew Geissler 13082c905dcSAndrew Geissler # Handle http tarball sources 13182c905dcSAndrew Geissler elif version.startswith("http") and integrity: 1327e0e3c0cSAndrew Geissler localfile = npm_localfile(os.path.basename(version)) 13382c905dcSAndrew Geissler 13482c905dcSAndrew Geissler uri = URI(version) 13582c905dcSAndrew Geissler uri.params["downloadfilename"] = localfile 13682c905dcSAndrew Geissler 13782c905dcSAndrew Geissler checksum_name, checksum_expected = npm_integrity(integrity) 13882c905dcSAndrew Geissler uri.params[checksum_name] = checksum_expected 13982c905dcSAndrew Geissler 14082c905dcSAndrew Geissler url = str(uri) 14182c905dcSAndrew Geissler 14282c905dcSAndrew Geissler localpath = os.path.join(d.getVar("DL_DIR"), localfile) 14382c905dcSAndrew Geissler 1446aa7eec5SAndrew Geissler # Handle local tarball and link sources 1456aa7eec5SAndrew Geissler elif version.startswith("file"): 1466aa7eec5SAndrew Geissler localpath = version[5:] 1476aa7eec5SAndrew Geissler if not version.endswith(".tgz"): 1486aa7eec5SAndrew Geissler unpack = False 1496aa7eec5SAndrew Geissler 15082c905dcSAndrew Geissler # Handle git sources 1516aa7eec5SAndrew Geissler elif version.startswith(("git", "bitbucket","gist")) or ( 1526aa7eec5SAndrew Geissler not version.endswith((".tgz", ".tar", ".tar.gz")) 1536aa7eec5SAndrew Geissler and not version.startswith((".", "@", "/")) 1546aa7eec5SAndrew Geissler and "/" in version 1556aa7eec5SAndrew Geissler ): 156595f6308SAndrew Geissler if version.startswith("github:"): 157595f6308SAndrew Geissler version = "git+https://github.com/" + version[len("github:"):] 1586aa7eec5SAndrew Geissler elif version.startswith("gist:"): 1596aa7eec5SAndrew Geissler version = "git+https://gist.github.com/" + version[len("gist:"):] 1606aa7eec5SAndrew Geissler elif version.startswith("bitbucket:"): 1616aa7eec5SAndrew Geissler version = "git+https://bitbucket.org/" + version[len("bitbucket:"):] 1626aa7eec5SAndrew Geissler elif version.startswith("gitlab:"): 1636aa7eec5SAndrew Geissler version = "git+https://gitlab.com/" + version[len("gitlab:"):] 1646aa7eec5SAndrew Geissler elif not version.startswith(("git+","git:")): 1656aa7eec5SAndrew Geissler version = "git+https://github.com/" + version 16682c905dcSAndrew Geissler regex = re.compile(r""" 16782c905dcSAndrew Geissler ^ 16882c905dcSAndrew Geissler git\+ 16982c905dcSAndrew Geissler (?P<protocol>[a-z]+) 17082c905dcSAndrew Geissler :// 17182c905dcSAndrew Geissler (?P<url>[^#]+) 17282c905dcSAndrew Geissler \# 17382c905dcSAndrew Geissler (?P<rev>[0-9a-f]+) 17482c905dcSAndrew Geissler $ 17582c905dcSAndrew Geissler """, re.VERBOSE) 17682c905dcSAndrew Geissler 17782c905dcSAndrew Geissler match = regex.match(version) 17882c905dcSAndrew Geissler 17982c905dcSAndrew Geissler if not match: 18082c905dcSAndrew Geissler raise ParameterError("Invalid git url: %s" % version, ud.url) 18182c905dcSAndrew Geissler 18282c905dcSAndrew Geissler groups = match.groupdict() 18382c905dcSAndrew Geissler 18482c905dcSAndrew Geissler uri = URI("git://" + str(groups["url"])) 18582c905dcSAndrew Geissler uri.params["protocol"] = str(groups["protocol"]) 18682c905dcSAndrew Geissler uri.params["rev"] = str(groups["rev"]) 18782c905dcSAndrew Geissler uri.params["destsuffix"] = destsuffix 18882c905dcSAndrew Geissler 18982c905dcSAndrew Geissler url = str(uri) 19082c905dcSAndrew Geissler 19182c905dcSAndrew Geissler else: 19282c905dcSAndrew Geissler raise ParameterError("Unsupported dependency: %s" % name, ud.url) 19382c905dcSAndrew Geissler 194*ac13d5f3SPatrick Williams # name is needed by unpack tracer for module mapping 19582c905dcSAndrew Geissler ud.deps.append({ 196*ac13d5f3SPatrick Williams "name": name, 19782c905dcSAndrew Geissler "url": url, 19882c905dcSAndrew Geissler "localpath": localpath, 19982c905dcSAndrew Geissler "extrapaths": extrapaths, 20082c905dcSAndrew Geissler "destsuffix": destsuffix, 201eff27476SAndrew Geissler "unpack": unpack, 20282c905dcSAndrew Geissler }) 20382c905dcSAndrew Geissler 20482c905dcSAndrew Geissler try: 20582c905dcSAndrew Geissler with open(ud.shrinkwrap_file, "r") as f: 20682c905dcSAndrew Geissler shrinkwrap = json.load(f) 20782c905dcSAndrew Geissler except Exception as e: 20882c905dcSAndrew Geissler raise ParameterError("Invalid shrinkwrap file: %s" % str(e), ud.url) 20982c905dcSAndrew Geissler 21082c905dcSAndrew Geissler foreach_dependencies(shrinkwrap, _resolve_dependency, ud.dev) 21182c905dcSAndrew Geissler 21282c905dcSAndrew Geissler # Avoid conflicts between the environment data and: 21382c905dcSAndrew Geissler # - the proxy url revision 21482c905dcSAndrew Geissler # - the proxy url checksum 21582c905dcSAndrew Geissler data = bb.data.createCopy(d) 21682c905dcSAndrew Geissler data.delVar("SRCREV") 21782c905dcSAndrew Geissler data.delVarFlags("SRC_URI") 21882c905dcSAndrew Geissler 21982c905dcSAndrew Geissler # This fetcher resolves multiple URIs from a shrinkwrap file and then 22082c905dcSAndrew Geissler # forwards it to a proxy fetcher. The management of the donestamp file, 22182c905dcSAndrew Geissler # the lockfile and the checksums are forwarded to the proxy fetcher. 2228e7b46e2SPatrick Williams shrinkwrap_urls = [dep["url"] for dep in ud.deps if dep["url"]] 2238e7b46e2SPatrick Williams if shrinkwrap_urls: 2248e7b46e2SPatrick Williams ud.proxy = Fetch(shrinkwrap_urls, data) 22582c905dcSAndrew Geissler ud.needdonestamp = False 22682c905dcSAndrew Geissler 22782c905dcSAndrew Geissler @staticmethod 22882c905dcSAndrew Geissler def _foreach_proxy_method(ud, handle): 22982c905dcSAndrew Geissler returns = [] 2302a25492cSPatrick Williams #Check if there are dependencies before try to fetch them 2312a25492cSPatrick Williams if len(ud.deps) > 0: 23282c905dcSAndrew Geissler for proxy_url in ud.proxy.urls: 23382c905dcSAndrew Geissler proxy_ud = ud.proxy.ud[proxy_url] 23482c905dcSAndrew Geissler proxy_d = ud.proxy.d 23582c905dcSAndrew Geissler proxy_ud.setup_localpath(proxy_d) 236eff27476SAndrew Geissler lf = lockfile(proxy_ud.lockfile) 23782c905dcSAndrew Geissler returns.append(handle(proxy_ud.method, proxy_ud, proxy_d)) 238eff27476SAndrew Geissler unlockfile(lf) 23982c905dcSAndrew Geissler return returns 24082c905dcSAndrew Geissler 24182c905dcSAndrew Geissler def verify_donestamp(self, ud, d): 24282c905dcSAndrew Geissler """Verify the donestamp file""" 24382c905dcSAndrew Geissler def _handle(m, ud, d): 24482c905dcSAndrew Geissler return m.verify_donestamp(ud, d) 24582c905dcSAndrew Geissler return all(self._foreach_proxy_method(ud, _handle)) 24682c905dcSAndrew Geissler 24782c905dcSAndrew Geissler def update_donestamp(self, ud, d): 24882c905dcSAndrew Geissler """Update the donestamp file""" 24982c905dcSAndrew Geissler def _handle(m, ud, d): 25082c905dcSAndrew Geissler m.update_donestamp(ud, d) 25182c905dcSAndrew Geissler self._foreach_proxy_method(ud, _handle) 25282c905dcSAndrew Geissler 25382c905dcSAndrew Geissler def need_update(self, ud, d): 25482c905dcSAndrew Geissler """Force a fetch, even if localpath exists ?""" 25582c905dcSAndrew Geissler def _handle(m, ud, d): 25682c905dcSAndrew Geissler return m.need_update(ud, d) 25782c905dcSAndrew Geissler return all(self._foreach_proxy_method(ud, _handle)) 25882c905dcSAndrew Geissler 25982c905dcSAndrew Geissler def try_mirrors(self, fetch, ud, d, mirrors): 26082c905dcSAndrew Geissler """Try to use a mirror""" 26182c905dcSAndrew Geissler def _handle(m, ud, d): 26282c905dcSAndrew Geissler return m.try_mirrors(fetch, ud, d, mirrors) 26382c905dcSAndrew Geissler return all(self._foreach_proxy_method(ud, _handle)) 26482c905dcSAndrew Geissler 26582c905dcSAndrew Geissler def download(self, ud, d): 26682c905dcSAndrew Geissler """Fetch url""" 26782c905dcSAndrew Geissler ud.proxy.download() 26882c905dcSAndrew Geissler 26982c905dcSAndrew Geissler def unpack(self, ud, rootdir, d): 27082c905dcSAndrew Geissler """Unpack the downloaded dependencies""" 27182c905dcSAndrew Geissler destdir = d.getVar("S") 27282c905dcSAndrew Geissler destsuffix = ud.parm.get("destsuffix") 27382c905dcSAndrew Geissler if destsuffix: 27482c905dcSAndrew Geissler destdir = os.path.join(rootdir, destsuffix) 275*ac13d5f3SPatrick Williams ud.unpack_tracer.unpack("npm-shrinkwrap", destdir) 27682c905dcSAndrew Geissler 27782c905dcSAndrew Geissler bb.utils.mkdirhier(destdir) 27882c905dcSAndrew Geissler bb.utils.copyfile(ud.shrinkwrap_file, 27982c905dcSAndrew Geissler os.path.join(destdir, "npm-shrinkwrap.json")) 28082c905dcSAndrew Geissler 28182c905dcSAndrew Geissler auto = [dep["url"] for dep in ud.deps if not dep["localpath"]] 28282c905dcSAndrew Geissler manual = [dep for dep in ud.deps if dep["localpath"]] 28382c905dcSAndrew Geissler 28482c905dcSAndrew Geissler if auto: 28582c905dcSAndrew Geissler ud.proxy.unpack(destdir, auto) 28682c905dcSAndrew Geissler 28782c905dcSAndrew Geissler for dep in manual: 28882c905dcSAndrew Geissler depdestdir = os.path.join(destdir, dep["destsuffix"]) 289eff27476SAndrew Geissler if dep["url"]: 29082c905dcSAndrew Geissler npm_unpack(dep["localpath"], depdestdir, d) 291eff27476SAndrew Geissler else: 292eff27476SAndrew Geissler depsrcdir= os.path.join(destdir, dep["localpath"]) 293eff27476SAndrew Geissler if dep["unpack"]: 294eff27476SAndrew Geissler npm_unpack(depsrcdir, depdestdir, d) 295eff27476SAndrew Geissler else: 296eff27476SAndrew Geissler bb.utils.mkdirhier(depdestdir) 297eff27476SAndrew Geissler cmd = 'cp -fpPRH "%s/." .' % (depsrcdir) 298eff27476SAndrew Geissler runfetchcmd(cmd, d, workdir=depdestdir) 29982c905dcSAndrew Geissler 30082c905dcSAndrew Geissler def clean(self, ud, d): 30182c905dcSAndrew Geissler """Clean any existing full or partial download""" 30282c905dcSAndrew Geissler ud.proxy.clean() 30382c905dcSAndrew Geissler 30482c905dcSAndrew Geissler # Clean extra files 30582c905dcSAndrew Geissler for dep in ud.deps: 30682c905dcSAndrew Geissler for path in dep["extrapaths"]: 30782c905dcSAndrew Geissler bb.utils.remove(path) 30882c905dcSAndrew Geissler 30982c905dcSAndrew Geissler def done(self, ud, d): 31082c905dcSAndrew Geissler """Is the download done ?""" 31182c905dcSAndrew Geissler def _handle(m, ud, d): 31282c905dcSAndrew Geissler return m.done(ud, d) 31382c905dcSAndrew Geissler return all(self._foreach_proxy_method(ud, _handle)) 314