1# Copyright (C) 2020 Savoir-Faire Linux
2#
3# SPDX-License-Identifier: GPL-2.0-only
4#
5# This bbclass builds and installs an npm package to the target. The package
6# sources files should be fetched in the calling recipe by using the SRC_URI
7# variable. The ${S} variable should be updated depending of your fetcher.
8#
9# Usage:
10#  SRC_URI = "..."
11#  inherit npm
12#
13# Optional variables:
14#  NPM_ARCH:
15#       Override the auto generated npm architecture.
16#
17#  NPM_INSTALL_DEV:
18#       Set to 1 to also install devDependencies.
19
20inherit python3native
21
22DEPENDS:prepend = "nodejs-native nodejs-oe-cache-native "
23RDEPENDS:${PN}:append:class-target = " nodejs"
24
25EXTRA_OENPM = ""
26
27NPM_INSTALL_DEV ?= "0"
28
29NPM_NODEDIR ?= "${RECIPE_SYSROOT_NATIVE}${prefix_native}"
30
31## must match mapping in nodejs.bb (openembedded-meta)
32def map_nodejs_arch(a, d):
33    import re
34
35    if   re.match('i.86$', a): return 'ia32'
36    elif re.match('x86_64$', a): return 'x64'
37    elif re.match('aarch64$', a): return 'arm64'
38    elif re.match('(powerpc64|powerpc64le|ppc64le)$', a): return 'ppc64'
39    elif re.match('powerpc$', a): return 'ppc'
40    return a
41
42NPM_ARCH ?= "${@map_nodejs_arch(d.getVar("TARGET_ARCH"), d)}"
43
44NPM_PACKAGE = "${WORKDIR}/npm-package"
45NPM_CACHE = "${WORKDIR}/npm-cache"
46NPM_BUILD = "${WORKDIR}/npm-build"
47NPM_REGISTRY = "${WORKDIR}/npm-registry"
48
49def npm_global_configs(d):
50    """Get the npm global configuration"""
51    configs = []
52    # Ensure no network access is done
53    configs.append(("offline", "true"))
54    configs.append(("proxy", "http://invalid"))
55    configs.append(("fund", False))
56    configs.append(("audit", False))
57    # Configure the cache directory
58    configs.append(("cache", d.getVar("NPM_CACHE")))
59    return configs
60
61## 'npm pack' runs 'prepare' and 'prepack' scripts. Support for
62## 'ignore-scripts' which prevents this behavior has been removed
63## from nodejs 16.  Use simple 'tar' instead of.
64def npm_pack(env, srcdir, workdir):
65    """Emulate 'npm pack' on a specified directory"""
66    import subprocess
67    import os
68    import json
69
70    src = os.path.join(srcdir, 'package.json')
71    with open(src) as f:
72        j = json.load(f)
73
74    # base does not really matter and is for documentation purposes
75    # only.  But the 'version' part must exist because other parts of
76    # the bbclass rely on it.
77    base = j['name'].split('/')[-1]
78    tarball = os.path.join(workdir, "%s-%s.tgz" % (base, j['version']));
79
80    # TODO: real 'npm pack' does not include directories while 'tar'
81    # does.  But this does not seem to matter...
82    subprocess.run(['tar', 'czf', tarball,
83                    '--exclude', './node-modules',
84                    '--exclude-vcs',
85                    '--transform', r's,^\./,package/,',
86                    '--mtime', '1985-10-26T08:15:00.000Z',
87                    '.'],
88                   check = True, cwd = srcdir)
89
90    return (tarball, j)
91
92python npm_do_configure() {
93    """
94    Step one: configure the npm cache and the main npm package
95
96    Every dependencies have been fetched and patched in the source directory.
97    They have to be packed (this remove unneeded files) and added to the npm
98    cache to be available for the next step.
99
100    The main package and its associated manifest file and shrinkwrap file have
101    to be configured to take into account these cached dependencies.
102    """
103    import base64
104    import copy
105    import json
106    import re
107    import shlex
108    import stat
109    import tempfile
110    from bb.fetch2.npm import NpmEnvironment
111    from bb.fetch2.npm import npm_unpack
112    from bb.fetch2.npm import npm_package
113    from bb.fetch2.npmsw import foreach_dependencies
114    from bb.progress import OutOfProgressHandler
115    from oe.npm_registry import NpmRegistry
116
117    bb.utils.remove(d.getVar("NPM_CACHE"), recurse=True)
118    bb.utils.remove(d.getVar("NPM_PACKAGE"), recurse=True)
119
120    env = NpmEnvironment(d, configs=npm_global_configs(d))
121    registry = NpmRegistry(d.getVar('NPM_REGISTRY'), d.getVar('NPM_CACHE'))
122
123    def _npm_cache_add(tarball, pkg):
124        """Add tarball to local registry and register it in the
125           cache"""
126        registry.add_pkg(tarball, pkg)
127
128    def _npm_integrity(tarball):
129        """Return the npm integrity of a specified tarball"""
130        sha512 = bb.utils.sha512_file(tarball)
131        return "sha512-" + base64.b64encode(bytes.fromhex(sha512)).decode()
132
133    # Manage the manifest file and shrinkwrap files
134    orig_manifest_file = d.expand("${S}/package.json")
135    orig_shrinkwrap_file = d.expand("${S}/npm-shrinkwrap.json")
136    cached_manifest_file = d.expand("${NPM_PACKAGE}/package.json")
137    cached_shrinkwrap_file = d.expand("${NPM_PACKAGE}/npm-shrinkwrap.json")
138
139    with open(orig_manifest_file, "r") as f:
140        orig_manifest = json.load(f)
141
142    cached_manifest = copy.deepcopy(orig_manifest)
143    cached_manifest.pop("dependencies", None)
144    cached_manifest.pop("devDependencies", None)
145
146    has_shrinkwrap_file = True
147
148    try:
149        with open(orig_shrinkwrap_file, "r") as f:
150            orig_shrinkwrap = json.load(f)
151    except IOError:
152        has_shrinkwrap_file = False
153
154    if has_shrinkwrap_file:
155       cached_shrinkwrap = copy.deepcopy(orig_shrinkwrap)
156       for package in orig_shrinkwrap["packages"]:
157            if package != "":
158                cached_shrinkwrap["packages"].pop(package, None)
159       cached_shrinkwrap["packages"][""].pop("dependencies", None)
160       cached_shrinkwrap["packages"][""].pop("devDependencies", None)
161       cached_shrinkwrap["packages"][""].pop("peerDependencies", None)
162
163    # Manage the dependencies
164    progress = OutOfProgressHandler(d, r"^(\d+)/(\d+)$")
165    progress_total = 1 # also count the main package
166    progress_done = 0
167
168    def _count_dependency(name, params, destsuffix):
169        nonlocal progress_total
170        progress_total += 1
171
172    def _cache_dependency(name, params, destsuffix):
173        with tempfile.TemporaryDirectory() as tmpdir:
174            # Add the dependency to the npm cache
175            destdir = os.path.join(d.getVar("S"), destsuffix)
176            (tarball, pkg) = npm_pack(env, destdir, tmpdir)
177            _npm_cache_add(tarball, pkg)
178            # Add its signature to the cached shrinkwrap
179            dep = params
180            dep["version"] = pkg['version']
181            dep["integrity"] = _npm_integrity(tarball)
182            if params.get("dev", False):
183                dep["dev"] = True
184                if "devDependencies" not in cached_shrinkwrap["packages"][""]:
185                    cached_shrinkwrap["packages"][""]["devDependencies"] = {}
186                cached_shrinkwrap["packages"][""]["devDependencies"][name] = pkg['version']
187
188            else:
189                if "dependencies" not in cached_shrinkwrap["packages"][""]:
190                    cached_shrinkwrap["packages"][""]["dependencies"] = {}
191                cached_shrinkwrap["packages"][""]["dependencies"][name] = pkg['version']
192
193            cached_shrinkwrap["packages"][destsuffix] = dep
194            # Display progress
195            nonlocal progress_done
196            progress_done += 1
197            progress.write("%d/%d" % (progress_done, progress_total))
198
199    dev = bb.utils.to_boolean(d.getVar("NPM_INSTALL_DEV"), False)
200
201    if has_shrinkwrap_file:
202        foreach_dependencies(orig_shrinkwrap, _count_dependency, dev)
203        foreach_dependencies(orig_shrinkwrap, _cache_dependency, dev)
204
205    # Manage Peer Dependencies
206    if has_shrinkwrap_file:
207        packages = orig_shrinkwrap.get("packages", {})
208        peer_deps = packages.get("", {}).get("peerDependencies", {})
209        package_runtime_dependencies = d.getVar("RDEPENDS:%s" % d.getVar("PN"))
210
211        for peer_dep in peer_deps:
212            peer_dep_yocto_name = npm_package(peer_dep)
213            if peer_dep_yocto_name not in package_runtime_dependencies:
214                bb.warn(peer_dep + " is a peer dependencie that is not in RDEPENDS variable. " +
215                "Please add this peer dependencie to the RDEPENDS variable as %s and generate its recipe with devtool"
216                % peer_dep_yocto_name)
217
218    # Configure the main package
219    with tempfile.TemporaryDirectory() as tmpdir:
220        (tarball, _) = npm_pack(env, d.getVar("S"), tmpdir)
221        npm_unpack(tarball, d.getVar("NPM_PACKAGE"), d)
222
223    # Configure the cached manifest file and cached shrinkwrap file
224    def _update_manifest(depkey):
225        for name in orig_manifest.get(depkey, {}):
226            version = cached_shrinkwrap["packages"][""][depkey][name]
227            if depkey not in cached_manifest:
228                cached_manifest[depkey] = {}
229            cached_manifest[depkey][name] = version
230
231    if has_shrinkwrap_file:
232        _update_manifest("dependencies")
233
234    if dev:
235        if has_shrinkwrap_file:
236            _update_manifest("devDependencies")
237
238    os.chmod(cached_manifest_file, os.stat(cached_manifest_file).st_mode | stat.S_IWUSR)
239    with open(cached_manifest_file, "w") as f:
240        json.dump(cached_manifest, f, indent=2)
241
242    if has_shrinkwrap_file:
243        with open(cached_shrinkwrap_file, "w") as f:
244            json.dump(cached_shrinkwrap, f, indent=2)
245}
246
247python npm_do_compile() {
248    """
249    Step two: install the npm package
250
251    Use the configured main package and the cached dependencies to run the
252    installation process. The installation is done in a directory which is
253    not the destination directory yet.
254
255    A combination of 'npm pack' and 'npm install' is used to ensure that the
256    installed files are actual copies instead of symbolic links (which is the
257    default npm behavior).
258    """
259    import shlex
260    import tempfile
261    from bb.fetch2.npm import NpmEnvironment
262
263    bb.utils.remove(d.getVar("NPM_BUILD"), recurse=True)
264
265    with tempfile.TemporaryDirectory() as tmpdir:
266        args = []
267        configs = npm_global_configs(d)
268
269        if bb.utils.to_boolean(d.getVar("NPM_INSTALL_DEV"), False):
270            configs.append(("also", "development"))
271        else:
272            configs.append(("only", "production"))
273
274        # Report as many logs as possible for debugging purpose
275        configs.append(("loglevel", "silly"))
276
277        # Configure the installation to be done globally in the build directory
278        configs.append(("global", "true"))
279        configs.append(("prefix", d.getVar("NPM_BUILD")))
280
281        # Add node-gyp configuration
282        configs.append(("arch", d.getVar("NPM_ARCH")))
283        configs.append(("release", "true"))
284        configs.append(("nodedir", d.getVar("NPM_NODEDIR")))
285        configs.append(("python", d.getVar("PYTHON")))
286
287        env = NpmEnvironment(d, configs)
288
289        # Add node-pre-gyp configuration
290        args.append(("target_arch", d.getVar("NPM_ARCH")))
291        args.append(("build-from-source", "true"))
292
293        # Don't install peer dependencies as they should be in RDEPENDS variable
294        args.append(("legacy-peer-deps", "true"))
295
296        # Pack and install the main package
297        (tarball, _) = npm_pack(env, d.getVar("NPM_PACKAGE"), tmpdir)
298        cmd = "npm install %s %s" % (shlex.quote(tarball), d.getVar("EXTRA_OENPM"))
299        env.run(cmd, args=args)
300}
301
302npm_do_install() {
303    # Step three: final install
304    #
305    # The previous installation have to be filtered to remove some extra files.
306
307    rm -rf ${D}
308
309    # Copy the entire lib and bin directories
310    install -d ${D}/${nonarch_libdir}
311    cp --no-preserve=ownership --recursive ${NPM_BUILD}/lib/. ${D}/${nonarch_libdir}
312
313    if [ -d "${NPM_BUILD}/bin" ]
314    then
315        install -d ${D}/${bindir}
316        cp --no-preserve=ownership --recursive ${NPM_BUILD}/bin/. ${D}/${bindir}
317    fi
318
319    # If the package (or its dependencies) uses node-gyp to build native addons,
320    # object files, static libraries or other temporary files can be hidden in
321    # the lib directory. To reduce the package size and to avoid QA issues
322    # (staticdev with static library files) these files must be removed.
323    local GYP_REGEX=".*/build/Release/[^/]*.node"
324
325    # Remove any node-gyp directory in ${D} to remove temporary build files
326    for GYP_D_FILE in $(find ${D} -regex "${GYP_REGEX}")
327    do
328        local GYP_D_DIR=${GYP_D_FILE%/Release/*}
329
330        rm --recursive --force ${GYP_D_DIR}
331    done
332
333    # Copy only the node-gyp release files
334    for GYP_B_FILE in $(find ${NPM_BUILD} -regex "${GYP_REGEX}")
335    do
336        local GYP_D_FILE=${D}/${prefix}/${GYP_B_FILE#${NPM_BUILD}}
337
338        install -d ${GYP_D_FILE%/*}
339        install -m 755 ${GYP_B_FILE} ${GYP_D_FILE}
340    done
341
342    # Remove the shrinkwrap file which does not need to be packed
343    rm -f ${D}/${nonarch_libdir}/node_modules/*/npm-shrinkwrap.json
344    rm -f ${D}/${nonarch_libdir}/node_modules/@*/*/npm-shrinkwrap.json
345}
346
347FILES:${PN} += " \
348    ${bindir} \
349    ${nonarch_libdir} \
350"
351
352EXPORT_FUNCTIONS do_configure do_compile do_install
353