1# Copyright (C) 2020 Savoir-Faire Linux 2# 3# SPDX-License-Identifier: GPL-2.0-only 4# 5""" 6BitBake 'Fetch' npm implementation 7 8npm fetcher support the SRC_URI with format of: 9SRC_URI = "npm://some.registry.url;OptionA=xxx;OptionB=xxx;..." 10 11Supported SRC_URI options are: 12 13- package 14 The npm package name. This is a mandatory parameter. 15 16- version 17 The npm package version. This is a mandatory parameter. 18 19- downloadfilename 20 Specifies the filename used when storing the downloaded file. 21 22- destsuffix 23 Specifies the directory to use to unpack the package (default: npm). 24""" 25 26import base64 27import json 28import os 29import re 30import shlex 31import tempfile 32import bb 33from bb.fetch2 import Fetch 34from bb.fetch2 import FetchError 35from bb.fetch2 import FetchMethod 36from bb.fetch2 import MissingParameterError 37from bb.fetch2 import ParameterError 38from bb.fetch2 import URI 39from bb.fetch2 import check_network_access 40from bb.fetch2 import runfetchcmd 41from bb.utils import is_semver 42 43def npm_package(package): 44 """Convert the npm package name to remove unsupported character""" 45 # For scoped package names ('@user/package') the '/' is replaced by a '-'. 46 # This is similar to what 'npm pack' does, but 'npm pack' also strips the 47 # leading '@', which can lead to ambiguous package names. 48 name = re.sub("/", "-", package) 49 name = name.lower() 50 name = re.sub(r"[^\-a-z0-9@]", "", name) 51 name = name.strip("-") 52 return name 53 54 55def npm_filename(package, version): 56 """Get the filename of a npm package""" 57 return npm_package(package) + "-" + version + ".tgz" 58 59def npm_localfile(package, version=None): 60 """Get the local filename of a npm package""" 61 if version is not None: 62 filename = npm_filename(package, version) 63 else: 64 filename = package 65 return os.path.join("npm2", filename) 66 67def npm_integrity(integrity): 68 """ 69 Get the checksum name and expected value from the subresource integrity 70 https://www.w3.org/TR/SRI/ 71 """ 72 algo, value = integrity.split("-", maxsplit=1) 73 return "%ssum" % algo, base64.b64decode(value).hex() 74 75def npm_unpack(tarball, destdir, d): 76 """Unpack a npm tarball""" 77 bb.utils.mkdirhier(destdir) 78 cmd = "tar --extract --gzip --file=%s" % shlex.quote(tarball) 79 cmd += " --no-same-owner" 80 cmd += " --delay-directory-restore" 81 cmd += " --strip-components=1" 82 runfetchcmd(cmd, d, workdir=destdir) 83 runfetchcmd("chmod -R +X '%s'" % (destdir), d, quiet=True, workdir=destdir) 84 85class NpmEnvironment(object): 86 """ 87 Using a npm config file seems more reliable than using cli arguments. 88 This class allows to create a controlled environment for npm commands. 89 """ 90 def __init__(self, d, configs=[], npmrc=None): 91 self.d = d 92 93 self.user_config = tempfile.NamedTemporaryFile(mode="w", buffering=1) 94 for key, value in configs: 95 self.user_config.write("%s=%s\n" % (key, value)) 96 97 if npmrc: 98 self.global_config_name = npmrc 99 else: 100 self.global_config_name = "/dev/null" 101 102 def __del__(self): 103 if self.user_config: 104 self.user_config.close() 105 106 def run(self, cmd, args=None, configs=None, workdir=None): 107 """Run npm command in a controlled environment""" 108 with tempfile.TemporaryDirectory() as tmpdir: 109 d = bb.data.createCopy(self.d) 110 d.setVar("PATH", d.getVar("PATH")) # PATH might contain $HOME - evaluate it before patching 111 d.setVar("HOME", tmpdir) 112 113 if not workdir: 114 workdir = tmpdir 115 116 def _run(cmd): 117 cmd = "NPM_CONFIG_USERCONFIG=%s " % (self.user_config.name) + cmd 118 cmd = "NPM_CONFIG_GLOBALCONFIG=%s " % (self.global_config_name) + cmd 119 return runfetchcmd(cmd, d, workdir=workdir) 120 121 if configs: 122 bb.warn("Use of configs argument of NpmEnvironment.run() function" 123 " is deprecated. Please use args argument instead.") 124 for key, value in configs: 125 cmd += " --%s=%s" % (key, shlex.quote(value)) 126 127 if args: 128 for key, value in args: 129 cmd += " --%s=%s" % (key, shlex.quote(value)) 130 131 return _run(cmd) 132 133class Npm(FetchMethod): 134 """Class to fetch a package from a npm registry""" 135 136 def supports(self, ud, d): 137 """Check if a given url can be fetched with npm""" 138 return ud.type in ["npm"] 139 140 def urldata_init(self, ud, d): 141 """Init npm specific variables within url data""" 142 ud.package = None 143 ud.version = None 144 ud.registry = None 145 146 # Get the 'package' parameter 147 if "package" in ud.parm: 148 ud.package = ud.parm.get("package") 149 150 if not ud.package: 151 raise MissingParameterError("Parameter 'package' required", ud.url) 152 153 # Get the 'version' parameter 154 if "version" in ud.parm: 155 ud.version = ud.parm.get("version") 156 157 if not ud.version: 158 raise MissingParameterError("Parameter 'version' required", ud.url) 159 160 if not is_semver(ud.version) and not ud.version == "latest": 161 raise ParameterError("Invalid 'version' parameter", ud.url) 162 163 # Extract the 'registry' part of the url 164 ud.registry = re.sub(r"^npm://", "https://", ud.url.split(";")[0]) 165 166 # Using the 'downloadfilename' parameter as local filename 167 # or the npm package name. 168 if "downloadfilename" in ud.parm: 169 ud.localfile = npm_localfile(d.expand(ud.parm["downloadfilename"])) 170 else: 171 ud.localfile = npm_localfile(ud.package, ud.version) 172 173 # Get the base 'npm' command 174 ud.basecmd = d.getVar("FETCHCMD_npm") or "npm" 175 176 # This fetcher resolves a URI from a npm package name and version and 177 # then forwards it to a proxy fetcher. A resolve file containing the 178 # resolved URI is created to avoid unwanted network access (if the file 179 # already exists). The management of the donestamp file, the lockfile 180 # and the checksums are forwarded to the proxy fetcher. 181 ud.proxy = None 182 ud.needdonestamp = False 183 ud.resolvefile = self.localpath(ud, d) + ".resolved" 184 185 def _resolve_proxy_url(self, ud, d): 186 def _npm_view(): 187 args = [] 188 args.append(("json", "true")) 189 args.append(("registry", ud.registry)) 190 pkgver = shlex.quote(ud.package + "@" + ud.version) 191 cmd = ud.basecmd + " view %s" % pkgver 192 env = NpmEnvironment(d) 193 check_network_access(d, cmd, ud.registry) 194 view_string = env.run(cmd, args=args) 195 196 if not view_string: 197 raise FetchError("Unavailable package %s" % pkgver, ud.url) 198 199 try: 200 view = json.loads(view_string) 201 202 error = view.get("error") 203 if error is not None: 204 raise FetchError(error.get("summary"), ud.url) 205 206 if ud.version == "latest": 207 bb.warn("The npm package %s is using the latest " \ 208 "version available. This could lead to " \ 209 "non-reproducible builds." % pkgver) 210 elif ud.version != view.get("version"): 211 raise ParameterError("Invalid 'version' parameter", ud.url) 212 213 return view 214 215 except Exception as e: 216 raise FetchError("Invalid view from npm: %s" % str(e), ud.url) 217 218 def _get_url(view): 219 tarball_url = view.get("dist", {}).get("tarball") 220 221 if tarball_url is None: 222 raise FetchError("Invalid 'dist.tarball' in view", ud.url) 223 224 uri = URI(tarball_url) 225 uri.params["downloadfilename"] = ud.localfile 226 227 integrity = view.get("dist", {}).get("integrity") 228 shasum = view.get("dist", {}).get("shasum") 229 230 if integrity is not None: 231 checksum_name, checksum_expected = npm_integrity(integrity) 232 uri.params[checksum_name] = checksum_expected 233 elif shasum is not None: 234 uri.params["sha1sum"] = shasum 235 else: 236 raise FetchError("Invalid 'dist.integrity' in view", ud.url) 237 238 return str(uri) 239 240 url = _get_url(_npm_view()) 241 242 bb.utils.mkdirhier(os.path.dirname(ud.resolvefile)) 243 with open(ud.resolvefile, "w") as f: 244 f.write(url) 245 246 def _setup_proxy(self, ud, d): 247 if ud.proxy is None: 248 if not os.path.exists(ud.resolvefile): 249 self._resolve_proxy_url(ud, d) 250 251 with open(ud.resolvefile, "r") as f: 252 url = f.read() 253 254 # Avoid conflicts between the environment data and: 255 # - the proxy url checksum 256 data = bb.data.createCopy(d) 257 data.delVarFlags("SRC_URI") 258 ud.proxy = Fetch([url], data) 259 260 def _get_proxy_method(self, ud, d): 261 self._setup_proxy(ud, d) 262 proxy_url = ud.proxy.urls[0] 263 proxy_ud = ud.proxy.ud[proxy_url] 264 proxy_d = ud.proxy.d 265 proxy_ud.setup_localpath(proxy_d) 266 return proxy_ud.method, proxy_ud, proxy_d 267 268 def verify_donestamp(self, ud, d): 269 """Verify the donestamp file""" 270 proxy_m, proxy_ud, proxy_d = self._get_proxy_method(ud, d) 271 return proxy_m.verify_donestamp(proxy_ud, proxy_d) 272 273 def update_donestamp(self, ud, d): 274 """Update the donestamp file""" 275 proxy_m, proxy_ud, proxy_d = self._get_proxy_method(ud, d) 276 proxy_m.update_donestamp(proxy_ud, proxy_d) 277 278 def need_update(self, ud, d): 279 """Force a fetch, even if localpath exists ?""" 280 if not os.path.exists(ud.resolvefile): 281 return True 282 if ud.version == "latest": 283 return True 284 proxy_m, proxy_ud, proxy_d = self._get_proxy_method(ud, d) 285 return proxy_m.need_update(proxy_ud, proxy_d) 286 287 def try_mirrors(self, fetch, ud, d, mirrors): 288 """Try to use a mirror""" 289 proxy_m, proxy_ud, proxy_d = self._get_proxy_method(ud, d) 290 return proxy_m.try_mirrors(fetch, proxy_ud, proxy_d, mirrors) 291 292 def download(self, ud, d): 293 """Fetch url""" 294 self._setup_proxy(ud, d) 295 ud.proxy.download() 296 297 def unpack(self, ud, rootdir, d): 298 """Unpack the downloaded archive""" 299 destsuffix = ud.parm.get("destsuffix", "npm") 300 destdir = os.path.join(rootdir, destsuffix) 301 npm_unpack(ud.localpath, destdir, d) 302 ud.unpack_tracer.unpack("npm", destdir) 303 304 def clean(self, ud, d): 305 """Clean any existing full or partial download""" 306 if os.path.exists(ud.resolvefile): 307 self._setup_proxy(ud, d) 308 ud.proxy.clean() 309 bb.utils.remove(ud.resolvefile) 310 311 def done(self, ud, d): 312 """Is the download done ?""" 313 if not os.path.exists(ud.resolvefile): 314 return False 315 proxy_m, proxy_ud, proxy_d = self._get_proxy_method(ud, d) 316 return proxy_m.done(proxy_ud, proxy_d) 317