xref: /openbmc/openbmc/poky/bitbake/lib/bb/fetch2/npm.py (revision ac13d5f3) 
1# Copyright (C) 2020 Savoir-Faire Linux
2#
3# SPDX-License-Identifier: GPL-2.0-only
4#
5"""
6BitBake 'Fetch' npm implementation
7
8npm fetcher support the SRC_URI with format of:
9SRC_URI = "npm://some.registry.url;OptionA=xxx;OptionB=xxx;..."
10
11Supported SRC_URI options are:
12
13- package
14   The npm package name. This is a mandatory parameter.
15
16- version
17    The npm package version. This is a mandatory parameter.
18
19- downloadfilename
20    Specifies the filename used when storing the downloaded file.
21
22- destsuffix
23    Specifies the directory to use to unpack the package (default: npm).
24"""
25
26import base64
27import json
28import os
29import re
30import shlex
31import tempfile
32import bb
33from bb.fetch2 import Fetch
34from bb.fetch2 import FetchError
35from bb.fetch2 import FetchMethod
36from bb.fetch2 import MissingParameterError
37from bb.fetch2 import ParameterError
38from bb.fetch2 import URI
39from bb.fetch2 import check_network_access
40from bb.fetch2 import runfetchcmd
41from bb.utils import is_semver
42
43def npm_package(package):
44    """Convert the npm package name to remove unsupported character"""
45    # Scoped package names (with the @) use the same naming convention
46    # as the 'npm pack' command.
47    name = re.sub("/", "-", package)
48    name = name.lower()
49    name = re.sub(r"[^\-a-z0-9]", "", name)
50    name = name.strip("-")
51    return name
52
53
54def npm_filename(package, version):
55    """Get the filename of a npm package"""
56    return npm_package(package) + "-" + version + ".tgz"
57
58def npm_localfile(package, version=None):
59    """Get the local filename of a npm package"""
60    if version is not None:
61        filename = npm_filename(package, version)
62    else:
63        filename = package
64    return os.path.join("npm2", filename)
65
66def npm_integrity(integrity):
67    """
68    Get the checksum name and expected value from the subresource integrity
69        https://www.w3.org/TR/SRI/
70    """
71    algo, value = integrity.split("-", maxsplit=1)
72    return "%ssum" % algo, base64.b64decode(value).hex()
73
74def npm_unpack(tarball, destdir, d):
75    """Unpack a npm tarball"""
76    bb.utils.mkdirhier(destdir)
77    cmd = "tar --extract --gzip --file=%s" % shlex.quote(tarball)
78    cmd += " --no-same-owner"
79    cmd += " --delay-directory-restore"
80    cmd += " --strip-components=1"
81    runfetchcmd(cmd, d, workdir=destdir)
82    runfetchcmd("chmod -R +X '%s'" % (destdir), d, quiet=True, workdir=destdir)
83
84class NpmEnvironment(object):
85    """
86    Using a npm config file seems more reliable than using cli arguments.
87    This class allows to create a controlled environment for npm commands.
88    """
89    def __init__(self, d, configs=[], npmrc=None):
90        self.d = d
91
92        self.user_config = tempfile.NamedTemporaryFile(mode="w", buffering=1)
93        for key, value in configs:
94            self.user_config.write("%s=%s\n" % (key, value))
95
96        if npmrc:
97            self.global_config_name = npmrc
98        else:
99            self.global_config_name = "/dev/null"
100
101    def __del__(self):
102        if self.user_config:
103            self.user_config.close()
104
105    def run(self, cmd, args=None, configs=None, workdir=None):
106        """Run npm command in a controlled environment"""
107        with tempfile.TemporaryDirectory() as tmpdir:
108            d = bb.data.createCopy(self.d)
109            d.setVar("PATH", d.getVar("PATH"))  # PATH might contain $HOME - evaluate it before patching
110            d.setVar("HOME", tmpdir)
111
112            if not workdir:
113                workdir = tmpdir
114
115            def _run(cmd):
116                cmd = "NPM_CONFIG_USERCONFIG=%s " % (self.user_config.name) + cmd
117                cmd = "NPM_CONFIG_GLOBALCONFIG=%s " % (self.global_config_name) + cmd
118                return runfetchcmd(cmd, d, workdir=workdir)
119
120            if configs:
121                bb.warn("Use of configs argument of NpmEnvironment.run() function"
122                    " is deprecated. Please use args argument instead.")
123                for key, value in configs:
124                    cmd += " --%s=%s" % (key, shlex.quote(value))
125
126            if args:
127                for key, value in args:
128                    cmd += " --%s=%s" % (key, shlex.quote(value))
129
130            return _run(cmd)
131
132class Npm(FetchMethod):
133    """Class to fetch a package from a npm registry"""
134
135    def supports(self, ud, d):
136        """Check if a given url can be fetched with npm"""
137        return ud.type in ["npm"]
138
139    def urldata_init(self, ud, d):
140        """Init npm specific variables within url data"""
141        ud.package = None
142        ud.version = None
143        ud.registry = None
144
145        # Get the 'package' parameter
146        if "package" in ud.parm:
147            ud.package = ud.parm.get("package")
148
149        if not ud.package:
150            raise MissingParameterError("Parameter 'package' required", ud.url)
151
152        # Get the 'version' parameter
153        if "version" in ud.parm:
154            ud.version = ud.parm.get("version")
155
156        if not ud.version:
157            raise MissingParameterError("Parameter 'version' required", ud.url)
158
159        if not is_semver(ud.version) and not ud.version == "latest":
160            raise ParameterError("Invalid 'version' parameter", ud.url)
161
162        # Extract the 'registry' part of the url
163        ud.registry = re.sub(r"^npm://", "https://", ud.url.split(";")[0])
164
165        # Using the 'downloadfilename' parameter as local filename
166        # or the npm package name.
167        if "downloadfilename" in ud.parm:
168            ud.localfile = npm_localfile(d.expand(ud.parm["downloadfilename"]))
169        else:
170            ud.localfile = npm_localfile(ud.package, ud.version)
171
172        # Get the base 'npm' command
173        ud.basecmd = d.getVar("FETCHCMD_npm") or "npm"
174
175        # This fetcher resolves a URI from a npm package name and version and
176        # then forwards it to a proxy fetcher. A resolve file containing the
177        # resolved URI is created to avoid unwanted network access (if the file
178        # already exists). The management of the donestamp file, the lockfile
179        # and the checksums are forwarded to the proxy fetcher.
180        ud.proxy = None
181        ud.needdonestamp = False
182        ud.resolvefile = self.localpath(ud, d) + ".resolved"
183
184    def _resolve_proxy_url(self, ud, d):
185        def _npm_view():
186            args = []
187            args.append(("json", "true"))
188            args.append(("registry", ud.registry))
189            pkgver = shlex.quote(ud.package + "@" + ud.version)
190            cmd = ud.basecmd + " view %s" % pkgver
191            env = NpmEnvironment(d)
192            check_network_access(d, cmd, ud.registry)
193            view_string = env.run(cmd, args=args)
194
195            if not view_string:
196                raise FetchError("Unavailable package %s" % pkgver, ud.url)
197
198            try:
199                view = json.loads(view_string)
200
201                error = view.get("error")
202                if error is not None:
203                    raise FetchError(error.get("summary"), ud.url)
204
205                if ud.version == "latest":
206                    bb.warn("The npm package %s is using the latest " \
207                            "version available. This could lead to " \
208                            "non-reproducible builds." % pkgver)
209                elif ud.version != view.get("version"):
210                    raise ParameterError("Invalid 'version' parameter", ud.url)
211
212                return view
213
214            except Exception as e:
215                raise FetchError("Invalid view from npm: %s" % str(e), ud.url)
216
217        def _get_url(view):
218            tarball_url = view.get("dist", {}).get("tarball")
219
220            if tarball_url is None:
221                raise FetchError("Invalid 'dist.tarball' in view", ud.url)
222
223            uri = URI(tarball_url)
224            uri.params["downloadfilename"] = ud.localfile
225
226            integrity = view.get("dist", {}).get("integrity")
227            shasum = view.get("dist", {}).get("shasum")
228
229            if integrity is not None:
230                checksum_name, checksum_expected = npm_integrity(integrity)
231                uri.params[checksum_name] = checksum_expected
232            elif shasum is not None:
233                uri.params["sha1sum"] = shasum
234            else:
235                raise FetchError("Invalid 'dist.integrity' in view", ud.url)
236
237            return str(uri)
238
239        url = _get_url(_npm_view())
240
241        bb.utils.mkdirhier(os.path.dirname(ud.resolvefile))
242        with open(ud.resolvefile, "w") as f:
243            f.write(url)
244
245    def _setup_proxy(self, ud, d):
246        if ud.proxy is None:
247            if not os.path.exists(ud.resolvefile):
248                self._resolve_proxy_url(ud, d)
249
250            with open(ud.resolvefile, "r") as f:
251                url = f.read()
252
253            # Avoid conflicts between the environment data and:
254            # - the proxy url checksum
255            data = bb.data.createCopy(d)
256            data.delVarFlags("SRC_URI")
257            ud.proxy = Fetch([url], data)
258
259    def _get_proxy_method(self, ud, d):
260        self._setup_proxy(ud, d)
261        proxy_url = ud.proxy.urls[0]
262        proxy_ud = ud.proxy.ud[proxy_url]
263        proxy_d = ud.proxy.d
264        proxy_ud.setup_localpath(proxy_d)
265        return proxy_ud.method, proxy_ud, proxy_d
266
267    def verify_donestamp(self, ud, d):
268        """Verify the donestamp file"""
269        proxy_m, proxy_ud, proxy_d = self._get_proxy_method(ud, d)
270        return proxy_m.verify_donestamp(proxy_ud, proxy_d)
271
272    def update_donestamp(self, ud, d):
273        """Update the donestamp file"""
274        proxy_m, proxy_ud, proxy_d = self._get_proxy_method(ud, d)
275        proxy_m.update_donestamp(proxy_ud, proxy_d)
276
277    def need_update(self, ud, d):
278        """Force a fetch, even if localpath exists ?"""
279        if not os.path.exists(ud.resolvefile):
280            return True
281        if ud.version == "latest":
282            return True
283        proxy_m, proxy_ud, proxy_d = self._get_proxy_method(ud, d)
284        return proxy_m.need_update(proxy_ud, proxy_d)
285
286    def try_mirrors(self, fetch, ud, d, mirrors):
287        """Try to use a mirror"""
288        proxy_m, proxy_ud, proxy_d = self._get_proxy_method(ud, d)
289        return proxy_m.try_mirrors(fetch, proxy_ud, proxy_d, mirrors)
290
291    def download(self, ud, d):
292        """Fetch url"""
293        self._setup_proxy(ud, d)
294        ud.proxy.download()
295
296    def unpack(self, ud, rootdir, d):
297        """Unpack the downloaded archive"""
298        destsuffix = ud.parm.get("destsuffix", "npm")
299        destdir = os.path.join(rootdir, destsuffix)
300        npm_unpack(ud.localpath, destdir, d)
301        ud.unpack_tracer.unpack("npm", destdir)
302
303    def clean(self, ud, d):
304        """Clean any existing full or partial download"""
305        if os.path.exists(ud.resolvefile):
306            self._setup_proxy(ud, d)
307            ud.proxy.clean()
308            bb.utils.remove(ud.resolvefile)
309
310    def done(self, ud, d):
311        """Is the download done ?"""
312        if not os.path.exists(ud.resolvefile):
313            return False
314        proxy_m, proxy_ud, proxy_d = self._get_proxy_method(ud, d)
315        return proxy_m.done(proxy_ud, proxy_d)
316