1# Copyright (C) 2020 Savoir-Faire Linux 2# 3# SPDX-License-Identifier: GPL-2.0-only 4# 5# This bbclass builds and installs an npm package to the target. The package 6# sources files should be fetched in the calling recipe by using the SRC_URI 7# variable. The ${S} variable should be updated depending of your fetcher. 8# 9# Usage: 10# SRC_URI = "..." 11# inherit npm 12# 13# Optional variables: 14# NPM_ARCH: 15# Override the auto generated npm architecture. 16# 17# NPM_INSTALL_DEV: 18# Set to 1 to also install devDependencies. 19 20inherit python3native 21 22DEPENDS:prepend = "nodejs-native nodejs-oe-cache-native " 23RDEPENDS:${PN}:append:class-target = " nodejs" 24 25EXTRA_OENPM = "" 26 27NPM_INSTALL_DEV ?= "0" 28 29NPM_NODEDIR ?= "${RECIPE_SYSROOT_NATIVE}${prefix_native}" 30 31## must match mapping in nodejs.bb (openembedded-meta) 32def map_nodejs_arch(a, d): 33 import re 34 35 if re.match('i.86$', a): return 'ia32' 36 elif re.match('x86_64$', a): return 'x64' 37 elif re.match('aarch64$', a): return 'arm64' 38 elif re.match('(powerpc64|powerpc64le|ppc64le)$', a): return 'ppc64' 39 elif re.match('powerpc$', a): return 'ppc' 40 return a 41 42NPM_ARCH ?= "${@map_nodejs_arch(d.getVar("TARGET_ARCH"), d)}" 43 44NPM_PACKAGE = "${WORKDIR}/npm-package" 45NPM_CACHE = "${WORKDIR}/npm-cache" 46NPM_BUILD = "${WORKDIR}/npm-build" 47NPM_REGISTRY = "${WORKDIR}/npm-registry" 48 49def npm_global_configs(d): 50 """Get the npm global configuration""" 51 configs = [] 52 # Ensure no network access is done 53 configs.append(("offline", "true")) 54 configs.append(("proxy", "http://invalid")) 55 configs.append(("fund", False)) 56 configs.append(("audit", False)) 57 # Configure the cache directory 58 configs.append(("cache", d.getVar("NPM_CACHE"))) 59 return configs 60 61## 'npm pack' runs 'prepare' and 'prepack' scripts. Support for 62## 'ignore-scripts' which prevents this behavior has been removed 63## from nodejs 16. Use simple 'tar' instead of. 64def npm_pack(env, srcdir, workdir): 65 """Emulate 'npm pack' on a specified directory""" 66 import subprocess 67 import os 68 import json 69 70 src = os.path.join(srcdir, 'package.json') 71 with open(src) as f: 72 j = json.load(f) 73 74 # base does not really matter and is for documentation purposes 75 # only. But the 'version' part must exist because other parts of 76 # the bbclass rely on it. 77 base = j['name'].split('/')[-1] 78 tarball = os.path.join(workdir, "%s-%s.tgz" % (base, j['version'])); 79 80 # TODO: real 'npm pack' does not include directories while 'tar' 81 # does. But this does not seem to matter... 82 subprocess.run(['tar', 'czf', tarball, 83 '--exclude', './node-modules', 84 '--exclude-vcs', 85 '--transform', r's,^\./,package/,', 86 '--mtime', '1985-10-26T08:15:00.000Z', 87 '.'], 88 check = True, cwd = srcdir) 89 90 return (tarball, j) 91 92python npm_do_configure() { 93 """ 94 Step one: configure the npm cache and the main npm package 95 96 Every dependencies have been fetched and patched in the source directory. 97 They have to be packed (this remove unneeded files) and added to the npm 98 cache to be available for the next step. 99 100 The main package and its associated manifest file and shrinkwrap file have 101 to be configured to take into account these cached dependencies. 102 """ 103 import base64 104 import copy 105 import json 106 import re 107 import shlex 108 import stat 109 import tempfile 110 from bb.fetch2.npm import NpmEnvironment 111 from bb.fetch2.npm import npm_unpack 112 from bb.fetch2.npmsw import foreach_dependencies 113 from bb.progress import OutOfProgressHandler 114 from oe.npm_registry import NpmRegistry 115 116 bb.utils.remove(d.getVar("NPM_CACHE"), recurse=True) 117 bb.utils.remove(d.getVar("NPM_PACKAGE"), recurse=True) 118 119 env = NpmEnvironment(d, configs=npm_global_configs(d)) 120 registry = NpmRegistry(d.getVar('NPM_REGISTRY'), d.getVar('NPM_CACHE')) 121 122 def _npm_cache_add(tarball, pkg): 123 """Add tarball to local registry and register it in the 124 cache""" 125 registry.add_pkg(tarball, pkg) 126 127 def _npm_integrity(tarball): 128 """Return the npm integrity of a specified tarball""" 129 sha512 = bb.utils.sha512_file(tarball) 130 return "sha512-" + base64.b64encode(bytes.fromhex(sha512)).decode() 131 132 def _npmsw_dependency_dict(orig, deptree): 133 """ 134 Return the sub dictionary in the 'orig' dictionary corresponding to the 135 'deptree' dependency tree. This function follows the shrinkwrap file 136 format. 137 """ 138 ptr = orig 139 for dep in deptree: 140 if "dependencies" not in ptr: 141 ptr["dependencies"] = {} 142 ptr = ptr["dependencies"] 143 if dep not in ptr: 144 ptr[dep] = {} 145 ptr = ptr[dep] 146 return ptr 147 148 # Manage the manifest file and shrinkwrap files 149 orig_manifest_file = d.expand("${S}/package.json") 150 orig_shrinkwrap_file = d.expand("${S}/npm-shrinkwrap.json") 151 cached_manifest_file = d.expand("${NPM_PACKAGE}/package.json") 152 cached_shrinkwrap_file = d.expand("${NPM_PACKAGE}/npm-shrinkwrap.json") 153 154 with open(orig_manifest_file, "r") as f: 155 orig_manifest = json.load(f) 156 157 cached_manifest = copy.deepcopy(orig_manifest) 158 cached_manifest.pop("dependencies", None) 159 cached_manifest.pop("devDependencies", None) 160 161 has_shrinkwrap_file = True 162 163 try: 164 with open(orig_shrinkwrap_file, "r") as f: 165 orig_shrinkwrap = json.load(f) 166 except IOError: 167 has_shrinkwrap_file = False 168 169 if has_shrinkwrap_file: 170 cached_shrinkwrap = copy.deepcopy(orig_shrinkwrap) 171 cached_shrinkwrap.pop("dependencies", None) 172 173 # Manage the dependencies 174 progress = OutOfProgressHandler(d, r"^(\d+)/(\d+)$") 175 progress_total = 1 # also count the main package 176 progress_done = 0 177 178 def _count_dependency(name, params, deptree): 179 nonlocal progress_total 180 progress_total += 1 181 182 def _cache_dependency(name, params, deptree): 183 destsubdirs = [os.path.join("node_modules", dep) for dep in deptree] 184 destsuffix = os.path.join(*destsubdirs) 185 with tempfile.TemporaryDirectory() as tmpdir: 186 # Add the dependency to the npm cache 187 destdir = os.path.join(d.getVar("S"), destsuffix) 188 (tarball, pkg) = npm_pack(env, destdir, tmpdir) 189 _npm_cache_add(tarball, pkg) 190 # Add its signature to the cached shrinkwrap 191 dep = _npmsw_dependency_dict(cached_shrinkwrap, deptree) 192 dep["version"] = pkg['version'] 193 dep["integrity"] = _npm_integrity(tarball) 194 if params.get("dev", False): 195 dep["dev"] = True 196 # Display progress 197 nonlocal progress_done 198 progress_done += 1 199 progress.write("%d/%d" % (progress_done, progress_total)) 200 201 dev = bb.utils.to_boolean(d.getVar("NPM_INSTALL_DEV"), False) 202 203 if has_shrinkwrap_file: 204 foreach_dependencies(orig_shrinkwrap, _count_dependency, dev) 205 foreach_dependencies(orig_shrinkwrap, _cache_dependency, dev) 206 207 # Configure the main package 208 with tempfile.TemporaryDirectory() as tmpdir: 209 (tarball, _) = npm_pack(env, d.getVar("S"), tmpdir) 210 npm_unpack(tarball, d.getVar("NPM_PACKAGE"), d) 211 212 # Configure the cached manifest file and cached shrinkwrap file 213 def _update_manifest(depkey): 214 for name in orig_manifest.get(depkey, {}): 215 version = cached_shrinkwrap["dependencies"][name]["version"] 216 if depkey not in cached_manifest: 217 cached_manifest[depkey] = {} 218 cached_manifest[depkey][name] = version 219 220 if has_shrinkwrap_file: 221 _update_manifest("dependencies") 222 223 if dev: 224 if has_shrinkwrap_file: 225 _update_manifest("devDependencies") 226 227 os.chmod(cached_manifest_file, os.stat(cached_manifest_file).st_mode | stat.S_IWUSR) 228 with open(cached_manifest_file, "w") as f: 229 json.dump(cached_manifest, f, indent=2) 230 231 if has_shrinkwrap_file: 232 with open(cached_shrinkwrap_file, "w") as f: 233 json.dump(cached_shrinkwrap, f, indent=2) 234} 235 236python npm_do_compile() { 237 """ 238 Step two: install the npm package 239 240 Use the configured main package and the cached dependencies to run the 241 installation process. The installation is done in a directory which is 242 not the destination directory yet. 243 244 A combination of 'npm pack' and 'npm install' is used to ensure that the 245 installed files are actual copies instead of symbolic links (which is the 246 default npm behavior). 247 """ 248 import shlex 249 import tempfile 250 from bb.fetch2.npm import NpmEnvironment 251 252 bb.utils.remove(d.getVar("NPM_BUILD"), recurse=True) 253 254 with tempfile.TemporaryDirectory() as tmpdir: 255 args = [] 256 configs = npm_global_configs(d) 257 258 if bb.utils.to_boolean(d.getVar("NPM_INSTALL_DEV"), False): 259 configs.append(("also", "development")) 260 else: 261 configs.append(("only", "production")) 262 263 # Report as many logs as possible for debugging purpose 264 configs.append(("loglevel", "silly")) 265 266 # Configure the installation to be done globally in the build directory 267 configs.append(("global", "true")) 268 configs.append(("prefix", d.getVar("NPM_BUILD"))) 269 270 # Add node-gyp configuration 271 configs.append(("arch", d.getVar("NPM_ARCH"))) 272 configs.append(("release", "true")) 273 configs.append(("nodedir", d.getVar("NPM_NODEDIR"))) 274 configs.append(("python", d.getVar("PYTHON"))) 275 276 env = NpmEnvironment(d, configs) 277 278 # Add node-pre-gyp configuration 279 args.append(("target_arch", d.getVar("NPM_ARCH"))) 280 args.append(("build-from-source", "true")) 281 282 # Pack and install the main package 283 (tarball, _) = npm_pack(env, d.getVar("NPM_PACKAGE"), tmpdir) 284 cmd = "npm install %s %s" % (shlex.quote(tarball), d.getVar("EXTRA_OENPM")) 285 env.run(cmd, args=args) 286} 287 288npm_do_install() { 289 # Step three: final install 290 # 291 # The previous installation have to be filtered to remove some extra files. 292 293 rm -rf ${D} 294 295 # Copy the entire lib and bin directories 296 install -d ${D}/${nonarch_libdir} 297 cp --no-preserve=ownership --recursive ${NPM_BUILD}/lib/. ${D}/${nonarch_libdir} 298 299 if [ -d "${NPM_BUILD}/bin" ] 300 then 301 install -d ${D}/${bindir} 302 cp --no-preserve=ownership --recursive ${NPM_BUILD}/bin/. ${D}/${bindir} 303 fi 304 305 # If the package (or its dependencies) uses node-gyp to build native addons, 306 # object files, static libraries or other temporary files can be hidden in 307 # the lib directory. To reduce the package size and to avoid QA issues 308 # (staticdev with static library files) these files must be removed. 309 local GYP_REGEX=".*/build/Release/[^/]*.node" 310 311 # Remove any node-gyp directory in ${D} to remove temporary build files 312 for GYP_D_FILE in $(find ${D} -regex "${GYP_REGEX}") 313 do 314 local GYP_D_DIR=${GYP_D_FILE%/Release/*} 315 316 rm --recursive --force ${GYP_D_DIR} 317 done 318 319 # Copy only the node-gyp release files 320 for GYP_B_FILE in $(find ${NPM_BUILD} -regex "${GYP_REGEX}") 321 do 322 local GYP_D_FILE=${D}/${prefix}/${GYP_B_FILE#${NPM_BUILD}} 323 324 install -d ${GYP_D_FILE%/*} 325 install -m 755 ${GYP_B_FILE} ${GYP_D_FILE} 326 done 327 328 # Remove the shrinkwrap file which does not need to be packed 329 rm -f ${D}/${nonarch_libdir}/node_modules/*/npm-shrinkwrap.json 330 rm -f ${D}/${nonarch_libdir}/node_modules/@*/*/npm-shrinkwrap.json 331} 332 333FILES:${PN} += " \ 334 ${bindir} \ 335 ${nonarch_libdir} \ 336" 337 338EXPORT_FUNCTIONS do_configure do_compile do_install 339