1# Copyright (C) 2020 Savoir-Faire Linux 2# 3# SPDX-License-Identifier: GPL-2.0-only 4# 5""" 6BitBake 'Fetch' npm implementation 7 8npm fetcher support the SRC_URI with format of: 9SRC_URI = "npm://some.registry.url;OptionA=xxx;OptionB=xxx;..." 10 11Supported SRC_URI options are: 12 13- package 14 The npm package name. This is a mandatory parameter. 15 16- version 17 The npm package version. This is a mandatory parameter. 18 19- downloadfilename 20 Specifies the filename used when storing the downloaded file. 21 22- destsuffix 23 Specifies the directory to use to unpack the package (default: npm). 24""" 25 26import base64 27import json 28import os 29import re 30import shlex 31import tempfile 32import bb 33from bb.fetch2 import Fetch 34from bb.fetch2 import FetchError 35from bb.fetch2 import FetchMethod 36from bb.fetch2 import MissingParameterError 37from bb.fetch2 import ParameterError 38from bb.fetch2 import URI 39from bb.fetch2 import check_network_access 40from bb.fetch2 import runfetchcmd 41from bb.utils import is_semver 42 43def npm_package(package): 44 """Convert the npm package name to remove unsupported character""" 45 # Scoped package names (with the @) use the same naming convention 46 # as the 'npm pack' command. 47 if package.startswith("@"): 48 return re.sub("/", "-", package[1:]) 49 return package 50 51def npm_filename(package, version): 52 """Get the filename of a npm package""" 53 return npm_package(package) + "-" + version + ".tgz" 54 55def npm_localfile(package, version=None): 56 """Get the local filename of a npm package""" 57 if version is not None: 58 filename = npm_filename(package, version) 59 else: 60 filename = package 61 return os.path.join("npm2", filename) 62 63def npm_integrity(integrity): 64 """ 65 Get the checksum name and expected value from the subresource integrity 66 https://www.w3.org/TR/SRI/ 67 """ 68 algo, value = integrity.split("-", maxsplit=1) 69 return "%ssum" % algo, base64.b64decode(value).hex() 70 71def npm_unpack(tarball, destdir, d): 72 """Unpack a npm tarball""" 73 bb.utils.mkdirhier(destdir) 74 cmd = "tar --extract --gzip --file=%s" % shlex.quote(tarball) 75 cmd += " --no-same-owner" 76 cmd += " --delay-directory-restore" 77 cmd += " --strip-components=1" 78 runfetchcmd(cmd, d, workdir=destdir) 79 runfetchcmd("chmod -R +X '%s'" % (destdir), d, quiet=True, workdir=destdir) 80 81class NpmEnvironment(object): 82 """ 83 Using a npm config file seems more reliable than using cli arguments. 84 This class allows to create a controlled environment for npm commands. 85 """ 86 def __init__(self, d, configs=[], npmrc=None): 87 self.d = d 88 89 self.user_config = tempfile.NamedTemporaryFile(mode="w", buffering=1) 90 for key, value in configs: 91 self.user_config.write("%s=%s\n" % (key, value)) 92 93 if npmrc: 94 self.global_config_name = npmrc 95 else: 96 self.global_config_name = "/dev/null" 97 98 def __del__(self): 99 if self.user_config: 100 self.user_config.close() 101 102 def run(self, cmd, args=None, configs=None, workdir=None): 103 """Run npm command in a controlled environment""" 104 with tempfile.TemporaryDirectory() as tmpdir: 105 d = bb.data.createCopy(self.d) 106 d.setVar("HOME", tmpdir) 107 108 if not workdir: 109 workdir = tmpdir 110 111 def _run(cmd): 112 cmd = "NPM_CONFIG_USERCONFIG=%s " % (self.user_config.name) + cmd 113 cmd = "NPM_CONFIG_GLOBALCONFIG=%s " % (self.global_config_name) + cmd 114 return runfetchcmd(cmd, d, workdir=workdir) 115 116 if configs: 117 bb.warn("Use of configs argument of NpmEnvironment.run() function" 118 " is deprecated. Please use args argument instead.") 119 for key, value in configs: 120 cmd += " --%s=%s" % (key, shlex.quote(value)) 121 122 if args: 123 for key, value in args: 124 cmd += " --%s=%s" % (key, shlex.quote(value)) 125 126 return _run(cmd) 127 128class Npm(FetchMethod): 129 """Class to fetch a package from a npm registry""" 130 131 def supports(self, ud, d): 132 """Check if a given url can be fetched with npm""" 133 return ud.type in ["npm"] 134 135 def urldata_init(self, ud, d): 136 """Init npm specific variables within url data""" 137 ud.package = None 138 ud.version = None 139 ud.registry = None 140 141 # Get the 'package' parameter 142 if "package" in ud.parm: 143 ud.package = ud.parm.get("package") 144 145 if not ud.package: 146 raise MissingParameterError("Parameter 'package' required", ud.url) 147 148 # Get the 'version' parameter 149 if "version" in ud.parm: 150 ud.version = ud.parm.get("version") 151 152 if not ud.version: 153 raise MissingParameterError("Parameter 'version' required", ud.url) 154 155 if not is_semver(ud.version) and not ud.version == "latest": 156 raise ParameterError("Invalid 'version' parameter", ud.url) 157 158 # Extract the 'registry' part of the url 159 ud.registry = re.sub(r"^npm://", "http://", ud.url.split(";")[0]) 160 161 # Using the 'downloadfilename' parameter as local filename 162 # or the npm package name. 163 if "downloadfilename" in ud.parm: 164 ud.localfile = npm_localfile(d.expand(ud.parm["downloadfilename"])) 165 else: 166 ud.localfile = npm_localfile(ud.package, ud.version) 167 168 # Get the base 'npm' command 169 ud.basecmd = d.getVar("FETCHCMD_npm") or "npm" 170 171 # This fetcher resolves a URI from a npm package name and version and 172 # then forwards it to a proxy fetcher. A resolve file containing the 173 # resolved URI is created to avoid unwanted network access (if the file 174 # already exists). The management of the donestamp file, the lockfile 175 # and the checksums are forwarded to the proxy fetcher. 176 ud.proxy = None 177 ud.needdonestamp = False 178 ud.resolvefile = self.localpath(ud, d) + ".resolved" 179 180 def _resolve_proxy_url(self, ud, d): 181 def _npm_view(): 182 args = [] 183 args.append(("json", "true")) 184 args.append(("registry", ud.registry)) 185 pkgver = shlex.quote(ud.package + "@" + ud.version) 186 cmd = ud.basecmd + " view %s" % pkgver 187 env = NpmEnvironment(d) 188 check_network_access(d, cmd, ud.registry) 189 view_string = env.run(cmd, args=args) 190 191 if not view_string: 192 raise FetchError("Unavailable package %s" % pkgver, ud.url) 193 194 try: 195 view = json.loads(view_string) 196 197 error = view.get("error") 198 if error is not None: 199 raise FetchError(error.get("summary"), ud.url) 200 201 if ud.version == "latest": 202 bb.warn("The npm package %s is using the latest " \ 203 "version available. This could lead to " \ 204 "non-reproducible builds." % pkgver) 205 elif ud.version != view.get("version"): 206 raise ParameterError("Invalid 'version' parameter", ud.url) 207 208 return view 209 210 except Exception as e: 211 raise FetchError("Invalid view from npm: %s" % str(e), ud.url) 212 213 def _get_url(view): 214 tarball_url = view.get("dist", {}).get("tarball") 215 216 if tarball_url is None: 217 raise FetchError("Invalid 'dist.tarball' in view", ud.url) 218 219 uri = URI(tarball_url) 220 uri.params["downloadfilename"] = ud.localfile 221 222 integrity = view.get("dist", {}).get("integrity") 223 shasum = view.get("dist", {}).get("shasum") 224 225 if integrity is not None: 226 checksum_name, checksum_expected = npm_integrity(integrity) 227 uri.params[checksum_name] = checksum_expected 228 elif shasum is not None: 229 uri.params["sha1sum"] = shasum 230 else: 231 raise FetchError("Invalid 'dist.integrity' in view", ud.url) 232 233 return str(uri) 234 235 url = _get_url(_npm_view()) 236 237 bb.utils.mkdirhier(os.path.dirname(ud.resolvefile)) 238 with open(ud.resolvefile, "w") as f: 239 f.write(url) 240 241 def _setup_proxy(self, ud, d): 242 if ud.proxy is None: 243 if not os.path.exists(ud.resolvefile): 244 self._resolve_proxy_url(ud, d) 245 246 with open(ud.resolvefile, "r") as f: 247 url = f.read() 248 249 # Avoid conflicts between the environment data and: 250 # - the proxy url checksum 251 data = bb.data.createCopy(d) 252 data.delVarFlags("SRC_URI") 253 ud.proxy = Fetch([url], data) 254 255 def _get_proxy_method(self, ud, d): 256 self._setup_proxy(ud, d) 257 proxy_url = ud.proxy.urls[0] 258 proxy_ud = ud.proxy.ud[proxy_url] 259 proxy_d = ud.proxy.d 260 proxy_ud.setup_localpath(proxy_d) 261 return proxy_ud.method, proxy_ud, proxy_d 262 263 def verify_donestamp(self, ud, d): 264 """Verify the donestamp file""" 265 proxy_m, proxy_ud, proxy_d = self._get_proxy_method(ud, d) 266 return proxy_m.verify_donestamp(proxy_ud, proxy_d) 267 268 def update_donestamp(self, ud, d): 269 """Update the donestamp file""" 270 proxy_m, proxy_ud, proxy_d = self._get_proxy_method(ud, d) 271 proxy_m.update_donestamp(proxy_ud, proxy_d) 272 273 def need_update(self, ud, d): 274 """Force a fetch, even if localpath exists ?""" 275 if not os.path.exists(ud.resolvefile): 276 return True 277 if ud.version == "latest": 278 return True 279 proxy_m, proxy_ud, proxy_d = self._get_proxy_method(ud, d) 280 return proxy_m.need_update(proxy_ud, proxy_d) 281 282 def try_mirrors(self, fetch, ud, d, mirrors): 283 """Try to use a mirror""" 284 proxy_m, proxy_ud, proxy_d = self._get_proxy_method(ud, d) 285 return proxy_m.try_mirrors(fetch, proxy_ud, proxy_d, mirrors) 286 287 def download(self, ud, d): 288 """Fetch url""" 289 self._setup_proxy(ud, d) 290 ud.proxy.download() 291 292 def unpack(self, ud, rootdir, d): 293 """Unpack the downloaded archive""" 294 destsuffix = ud.parm.get("destsuffix", "npm") 295 destdir = os.path.join(rootdir, destsuffix) 296 npm_unpack(ud.localpath, destdir, d) 297 298 def clean(self, ud, d): 299 """Clean any existing full or partial download""" 300 if os.path.exists(ud.resolvefile): 301 self._setup_proxy(ud, d) 302 ud.proxy.clean() 303 bb.utils.remove(ud.resolvefile) 304 305 def done(self, ud, d): 306 """Is the download done ?""" 307 if not os.path.exists(ud.resolvefile): 308 return False 309 proxy_m, proxy_ud, proxy_d = self._get_proxy_method(ud, d) 310 return proxy_m.done(proxy_ud, proxy_d) 311