1# Copyright (C) 2020 Savoir-Faire Linux 2# 3# SPDX-License-Identifier: GPL-2.0-only 4# 5""" 6BitBake 'Fetch' npm implementation 7 8npm fetcher support the SRC_URI with format of: 9SRC_URI = "npm://some.registry.url;OptionA=xxx;OptionB=xxx;..." 10 11Supported SRC_URI options are: 12 13- package 14 The npm package name. This is a mandatory parameter. 15 16- version 17 The npm package version. This is a mandatory parameter. 18 19- downloadfilename 20 Specifies the filename used when storing the downloaded file. 21 22- destsuffix 23 Specifies the directory to use to unpack the package (default: npm). 24""" 25 26import base64 27import json 28import os 29import re 30import shlex 31import tempfile 32import bb 33from bb.fetch2 import Fetch 34from bb.fetch2 import FetchError 35from bb.fetch2 import FetchMethod 36from bb.fetch2 import MissingParameterError 37from bb.fetch2 import ParameterError 38from bb.fetch2 import URI 39from bb.fetch2 import check_network_access 40from bb.fetch2 import runfetchcmd 41from bb.utils import is_semver 42 43def npm_package(package): 44 """Convert the npm package name to remove unsupported character""" 45 # For scoped package names ('@user/package') the '/' is replaced by a '-'. 46 # This is similar to what 'npm pack' does, but 'npm pack' also strips the 47 # leading '@', which can lead to ambiguous package names. 48 name = re.sub("/", "-", package) 49 name = name.lower() 50 name = re.sub(r"[^\-a-z0-9@]", "", name) 51 name = name.strip("-") 52 return name 53 54 55def npm_filename(package, version): 56 """Get the filename of a npm package""" 57 return npm_package(package) + "-" + version + ".tgz" 58 59def npm_localfile(package, version=None): 60 """Get the local filename of a npm package""" 61 if version is not None: 62 filename = npm_filename(package, version) 63 else: 64 filename = package 65 return os.path.join("npm2", filename) 66 67def npm_integrity(integrity): 68 """ 69 Get the checksum name and expected value from the subresource integrity 70 https://www.w3.org/TR/SRI/ 71 """ 72 algo, value = integrity.split("-", maxsplit=1) 73 return "%ssum" % algo, base64.b64decode(value).hex() 74 75def npm_unpack(tarball, destdir, d): 76 """Unpack a npm tarball""" 77 bb.utils.mkdirhier(destdir) 78 cmd = "tar --extract --gzip --file=%s" % shlex.quote(tarball) 79 cmd += " --no-same-owner" 80 cmd += " --delay-directory-restore" 81 cmd += " --strip-components=1" 82 runfetchcmd(cmd, d, workdir=destdir) 83 runfetchcmd("chmod -R +X '%s'" % (destdir), d, quiet=True, workdir=destdir) 84 85class NpmEnvironment(object): 86 """ 87 Using a npm config file seems more reliable than using cli arguments. 88 This class allows to create a controlled environment for npm commands. 89 """ 90 def __init__(self, d, configs=[], npmrc=None): 91 self.d = d 92 93 self.user_config = tempfile.NamedTemporaryFile(mode="w", buffering=1) 94 95 hn = self._home_npmrc(d) 96 if hn is not None: 97 with open(hn, 'r') as hnf: 98 self.user_config.write(hnf.read()) 99 100 for key, value in configs: 101 self.user_config.write("%s=%s\n" % (key, value)) 102 103 if npmrc: 104 self.global_config_name = npmrc 105 else: 106 self.global_config_name = "/dev/null" 107 108 def __del__(self): 109 if self.user_config: 110 self.user_config.close() 111 112 def _home_npmrc(self, d): 113 """Function to return user's HOME .npmrc file (or None if it doesn't exist)""" 114 home_npmrc_file = os.path.join(os.environ.get("HOME"), ".npmrc") 115 if d.getVar("BB_USE_HOME_NPMRC") == "1" and os.path.exists(home_npmrc_file): 116 bb.warn(f"BB_USE_HOME_NPMRC flag set and valid .npmrc detected - "\ 117 f"npm fetcher will use {home_npmrc_file}") 118 return home_npmrc_file 119 return None 120 121 def run(self, cmd, args=None, configs=None, workdir=None): 122 """Run npm command in a controlled environment""" 123 with tempfile.TemporaryDirectory() as tmpdir: 124 d = bb.data.createCopy(self.d) 125 d.setVar("PATH", d.getVar("PATH")) # PATH might contain $HOME - evaluate it before patching 126 d.setVar("HOME", tmpdir) 127 128 if not workdir: 129 workdir = tmpdir 130 131 def _run(cmd): 132 cmd = "NPM_CONFIG_USERCONFIG=%s " % (self.user_config.name) + cmd 133 cmd = "NPM_CONFIG_GLOBALCONFIG=%s " % (self.global_config_name) + cmd 134 return runfetchcmd(cmd, d, workdir=workdir) 135 136 if configs: 137 bb.warn("Use of configs argument of NpmEnvironment.run() function" 138 " is deprecated. Please use args argument instead.") 139 for key, value in configs: 140 cmd += " --%s=%s" % (key, shlex.quote(value)) 141 142 if args: 143 for key, value in args: 144 cmd += " --%s=%s" % (key, shlex.quote(value)) 145 146 return _run(cmd) 147 148class Npm(FetchMethod): 149 """Class to fetch a package from a npm registry""" 150 151 def supports(self, ud, d): 152 """Check if a given url can be fetched with npm""" 153 return ud.type in ["npm"] 154 155 def urldata_init(self, ud, d): 156 """Init npm specific variables within url data""" 157 ud.package = None 158 ud.version = None 159 ud.registry = None 160 161 # Get the 'package' parameter 162 if "package" in ud.parm: 163 ud.package = ud.parm.get("package") 164 165 if not ud.package: 166 raise MissingParameterError("Parameter 'package' required", ud.url) 167 168 # Get the 'version' parameter 169 if "version" in ud.parm: 170 ud.version = ud.parm.get("version") 171 172 if not ud.version: 173 raise MissingParameterError("Parameter 'version' required", ud.url) 174 175 if not is_semver(ud.version) and not ud.version == "latest": 176 raise ParameterError("Invalid 'version' parameter", ud.url) 177 178 # Extract the 'registry' part of the url 179 ud.registry = re.sub(r"^npm://", "https://", ud.url.split(";")[0]) 180 181 # Using the 'downloadfilename' parameter as local filename 182 # or the npm package name. 183 if "downloadfilename" in ud.parm: 184 ud.localfile = npm_localfile(ud.parm["downloadfilename"]) 185 else: 186 ud.localfile = npm_localfile(ud.package, ud.version) 187 188 # Get the base 'npm' command 189 ud.basecmd = d.getVar("FETCHCMD_npm") or "npm" 190 191 # This fetcher resolves a URI from a npm package name and version and 192 # then forwards it to a proxy fetcher. A resolve file containing the 193 # resolved URI is created to avoid unwanted network access (if the file 194 # already exists). The management of the donestamp file, the lockfile 195 # and the checksums are forwarded to the proxy fetcher. 196 ud.proxy = None 197 ud.needdonestamp = False 198 ud.resolvefile = self.localpath(ud, d) + ".resolved" 199 200 def _resolve_proxy_url(self, ud, d): 201 def _npm_view(): 202 args = [] 203 args.append(("json", "true")) 204 args.append(("registry", ud.registry)) 205 pkgver = shlex.quote(ud.package + "@" + ud.version) 206 cmd = ud.basecmd + " view %s" % pkgver 207 env = NpmEnvironment(d) 208 check_network_access(d, cmd, ud.registry) 209 view_string = env.run(cmd, args=args) 210 211 if not view_string: 212 raise FetchError("Unavailable package %s" % pkgver, ud.url) 213 214 try: 215 view = json.loads(view_string) 216 217 error = view.get("error") 218 if error is not None: 219 raise FetchError(error.get("summary"), ud.url) 220 221 if ud.version == "latest": 222 bb.warn("The npm package %s is using the latest " \ 223 "version available. This could lead to " \ 224 "non-reproducible builds." % pkgver) 225 elif ud.version != view.get("version"): 226 raise ParameterError("Invalid 'version' parameter", ud.url) 227 228 return view 229 230 except Exception as e: 231 raise FetchError("Invalid view from npm: %s" % str(e), ud.url) 232 233 def _get_url(view): 234 tarball_url = view.get("dist", {}).get("tarball") 235 236 if tarball_url is None: 237 raise FetchError("Invalid 'dist.tarball' in view", ud.url) 238 239 uri = URI(tarball_url) 240 uri.params["downloadfilename"] = ud.localfile 241 242 integrity = view.get("dist", {}).get("integrity") 243 shasum = view.get("dist", {}).get("shasum") 244 245 if integrity is not None: 246 checksum_name, checksum_expected = npm_integrity(integrity) 247 uri.params[checksum_name] = checksum_expected 248 elif shasum is not None: 249 uri.params["sha1sum"] = shasum 250 else: 251 raise FetchError("Invalid 'dist.integrity' in view", ud.url) 252 253 return str(uri) 254 255 url = _get_url(_npm_view()) 256 257 bb.utils.mkdirhier(os.path.dirname(ud.resolvefile)) 258 with open(ud.resolvefile, "w") as f: 259 f.write(url) 260 261 def _setup_proxy(self, ud, d): 262 if ud.proxy is None: 263 if not os.path.exists(ud.resolvefile): 264 self._resolve_proxy_url(ud, d) 265 266 with open(ud.resolvefile, "r") as f: 267 url = f.read() 268 269 # Avoid conflicts between the environment data and: 270 # - the proxy url checksum 271 data = bb.data.createCopy(d) 272 data.delVarFlags("SRC_URI") 273 ud.proxy = Fetch([url], data) 274 275 def _get_proxy_method(self, ud, d): 276 self._setup_proxy(ud, d) 277 proxy_url = ud.proxy.urls[0] 278 proxy_ud = ud.proxy.ud[proxy_url] 279 proxy_d = ud.proxy.d 280 proxy_ud.setup_localpath(proxy_d) 281 return proxy_ud.method, proxy_ud, proxy_d 282 283 def verify_donestamp(self, ud, d): 284 """Verify the donestamp file""" 285 proxy_m, proxy_ud, proxy_d = self._get_proxy_method(ud, d) 286 return proxy_m.verify_donestamp(proxy_ud, proxy_d) 287 288 def update_donestamp(self, ud, d): 289 """Update the donestamp file""" 290 proxy_m, proxy_ud, proxy_d = self._get_proxy_method(ud, d) 291 proxy_m.update_donestamp(proxy_ud, proxy_d) 292 293 def need_update(self, ud, d): 294 """Force a fetch, even if localpath exists ?""" 295 if not os.path.exists(ud.resolvefile): 296 return True 297 if ud.version == "latest": 298 return True 299 proxy_m, proxy_ud, proxy_d = self._get_proxy_method(ud, d) 300 return proxy_m.need_update(proxy_ud, proxy_d) 301 302 def try_mirrors(self, fetch, ud, d, mirrors): 303 """Try to use a mirror""" 304 proxy_m, proxy_ud, proxy_d = self._get_proxy_method(ud, d) 305 return proxy_m.try_mirrors(fetch, proxy_ud, proxy_d, mirrors) 306 307 def download(self, ud, d): 308 """Fetch url""" 309 self._setup_proxy(ud, d) 310 ud.proxy.download() 311 312 def unpack(self, ud, rootdir, d): 313 """Unpack the downloaded archive""" 314 destsuffix = ud.parm.get("destsuffix", "npm") 315 destdir = os.path.join(rootdir, destsuffix) 316 npm_unpack(ud.localpath, destdir, d) 317 ud.unpack_tracer.unpack("npm", destdir) 318 319 def clean(self, ud, d): 320 """Clean any existing full or partial download""" 321 if os.path.exists(ud.resolvefile): 322 self._setup_proxy(ud, d) 323 ud.proxy.clean() 324 bb.utils.remove(ud.resolvefile) 325 326 def done(self, ud, d): 327 """Is the download done ?""" 328 if not os.path.exists(ud.resolvefile): 329 return False 330 proxy_m, proxy_ud, proxy_d = self._get_proxy_method(ud, d) 331 return proxy_m.done(proxy_ud, proxy_d) 332