1# Copyright (C) 2020 Savoir-Faire Linux
2#
3# SPDX-License-Identifier: GPL-2.0-only
4#
5# This bbclass builds and installs an npm package to the target. The package
6# sources files should be fetched in the calling recipe by using the SRC_URI
7# variable. The ${S} variable should be updated depending of your fetcher.
8#
9# Usage:
10#  SRC_URI = "..."
11#  inherit npm
12#
13# Optional variables:
14#  NPM_ARCH:
15#       Override the auto generated npm architecture.
16#
17#  NPM_INSTALL_DEV:
18#       Set to 1 to also install devDependencies.
19
20inherit python3native
21
22DEPENDS:prepend = "nodejs-native nodejs-oe-cache-native "
23RDEPENDS:${PN}:append:class-target = " nodejs"
24
25EXTRA_OENPM = ""
26
27NPM_INSTALL_DEV ?= "0"
28
29NPM_NODEDIR ?= "${RECIPE_SYSROOT_NATIVE}${prefix_native}"
30
31## must match mapping in nodejs.bb (openembedded-meta)
32def map_nodejs_arch(a, d):
33    import re
34
35    if   re.match('i.86$', a): return 'ia32'
36    elif re.match('x86_64$', a): return 'x64'
37    elif re.match('aarch64$', a): return 'arm64'
38    elif re.match('(powerpc64|powerpc64le|ppc64le)$', a): return 'ppc64'
39    elif re.match('powerpc$', a): return 'ppc'
40    return a
41
42NPM_ARCH ?= "${@map_nodejs_arch(d.getVar("TARGET_ARCH"), d)}"
43
44NPM_PACKAGE = "${WORKDIR}/npm-package"
45NPM_CACHE = "${WORKDIR}/npm-cache"
46NPM_BUILD = "${WORKDIR}/npm-build"
47NPM_REGISTRY = "${WORKDIR}/npm-registry"
48
49def npm_global_configs(d):
50    """Get the npm global configuration"""
51    configs = []
52    # Ensure no network access is done
53    configs.append(("offline", "true"))
54    configs.append(("proxy", "http://invalid"))
55    configs.append(("fund", False))
56    configs.append(("audit", False))
57    # Configure the cache directory
58    configs.append(("cache", d.getVar("NPM_CACHE")))
59    return configs
60
61## 'npm pack' runs 'prepare' and 'prepack' scripts. Support for
62## 'ignore-scripts' which prevents this behavior has been removed
63## from nodejs 16.  Use simple 'tar' instead of.
64def npm_pack(env, srcdir, workdir):
65    """Emulate 'npm pack' on a specified directory"""
66    import subprocess
67    import os
68    import json
69
70    src = os.path.join(srcdir, 'package.json')
71    with open(src) as f:
72        j = json.load(f)
73
74    # base does not really matter and is for documentation purposes
75    # only.  But the 'version' part must exist because other parts of
76    # the bbclass rely on it.
77    base = j['name'].split('/')[-1]
78    tarball = os.path.join(workdir, "%s-%s.tgz" % (base, j['version']));
79
80    # TODO: real 'npm pack' does not include directories while 'tar'
81    # does.  But this does not seem to matter...
82    subprocess.run(['tar', 'czf', tarball,
83                    '--exclude', './node-modules',
84                    '--exclude-vcs',
85                    '--transform', r's,^\./,package/,',
86                    '--mtime', '1985-10-26T08:15:00.000Z',
87                    '.'],
88                   check = True, cwd = srcdir)
89
90    return (tarball, j)
91
92python npm_do_configure() {
93    """
94    Step one: configure the npm cache and the main npm package
95
96    Every dependencies have been fetched and patched in the source directory.
97    They have to be packed (this remove unneeded files) and added to the npm
98    cache to be available for the next step.
99
100    The main package and its associated manifest file and shrinkwrap file have
101    to be configured to take into account these cached dependencies.
102    """
103    import base64
104    import copy
105    import json
106    import re
107    import shlex
108    import stat
109    import tempfile
110    from bb.fetch2.npm import NpmEnvironment
111    from bb.fetch2.npm import npm_unpack
112    from bb.fetch2.npmsw import foreach_dependencies
113    from bb.progress import OutOfProgressHandler
114    from oe.npm_registry import NpmRegistry
115
116    bb.utils.remove(d.getVar("NPM_CACHE"), recurse=True)
117    bb.utils.remove(d.getVar("NPM_PACKAGE"), recurse=True)
118
119    env = NpmEnvironment(d, configs=npm_global_configs(d))
120    registry = NpmRegistry(d.getVar('NPM_REGISTRY'), d.getVar('NPM_CACHE'))
121
122    def _npm_cache_add(tarball, pkg):
123        """Add tarball to local registry and register it in the
124           cache"""
125        registry.add_pkg(tarball, pkg)
126
127    def _npm_integrity(tarball):
128        """Return the npm integrity of a specified tarball"""
129        sha512 = bb.utils.sha512_file(tarball)
130        return "sha512-" + base64.b64encode(bytes.fromhex(sha512)).decode()
131
132    def _npmsw_dependency_dict(orig, deptree):
133        """
134        Return the sub dictionary in the 'orig' dictionary corresponding to the
135        'deptree' dependency tree. This function follows the shrinkwrap file
136        format.
137        """
138        ptr = orig
139        for dep in deptree:
140            if "dependencies" not in ptr:
141                ptr["dependencies"] = {}
142            ptr = ptr["dependencies"]
143            if dep not in ptr:
144                ptr[dep] = {}
145            ptr = ptr[dep]
146        return ptr
147
148    # Manage the manifest file and shrinkwrap files
149    orig_manifest_file = d.expand("${S}/package.json")
150    orig_shrinkwrap_file = d.expand("${S}/npm-shrinkwrap.json")
151    cached_manifest_file = d.expand("${NPM_PACKAGE}/package.json")
152    cached_shrinkwrap_file = d.expand("${NPM_PACKAGE}/npm-shrinkwrap.json")
153
154    with open(orig_manifest_file, "r") as f:
155        orig_manifest = json.load(f)
156
157    cached_manifest = copy.deepcopy(orig_manifest)
158    cached_manifest.pop("dependencies", None)
159    cached_manifest.pop("devDependencies", None)
160
161    has_shrinkwrap_file = True
162
163    try:
164        with open(orig_shrinkwrap_file, "r") as f:
165            orig_shrinkwrap = json.load(f)
166    except IOError:
167        has_shrinkwrap_file = False
168
169    if has_shrinkwrap_file:
170       cached_shrinkwrap = copy.deepcopy(orig_shrinkwrap)
171       cached_shrinkwrap.pop("dependencies", None)
172
173    # Manage the dependencies
174    progress = OutOfProgressHandler(d, r"^(\d+)/(\d+)$")
175    progress_total = 1 # also count the main package
176    progress_done = 0
177
178    def _count_dependency(name, params, deptree):
179        nonlocal progress_total
180        progress_total += 1
181
182    def _cache_dependency(name, params, deptree):
183        destsubdirs = [os.path.join("node_modules", dep) for dep in deptree]
184        destsuffix = os.path.join(*destsubdirs)
185        with tempfile.TemporaryDirectory() as tmpdir:
186            # Add the dependency to the npm cache
187            destdir = os.path.join(d.getVar("S"), destsuffix)
188            (tarball, pkg) = npm_pack(env, destdir, tmpdir)
189            _npm_cache_add(tarball, pkg)
190            # Add its signature to the cached shrinkwrap
191            dep = _npmsw_dependency_dict(cached_shrinkwrap, deptree)
192            dep["version"] = pkg['version']
193            dep["integrity"] = _npm_integrity(tarball)
194            if params.get("dev", False):
195                dep["dev"] = True
196            # Display progress
197            nonlocal progress_done
198            progress_done += 1
199            progress.write("%d/%d" % (progress_done, progress_total))
200
201    dev = bb.utils.to_boolean(d.getVar("NPM_INSTALL_DEV"), False)
202
203    if has_shrinkwrap_file:
204        foreach_dependencies(orig_shrinkwrap, _count_dependency, dev)
205        foreach_dependencies(orig_shrinkwrap, _cache_dependency, dev)
206
207    # Configure the main package
208    with tempfile.TemporaryDirectory() as tmpdir:
209        (tarball, _) = npm_pack(env, d.getVar("S"), tmpdir)
210        npm_unpack(tarball, d.getVar("NPM_PACKAGE"), d)
211
212    # Configure the cached manifest file and cached shrinkwrap file
213    def _update_manifest(depkey):
214        for name in orig_manifest.get(depkey, {}):
215            version = cached_shrinkwrap["dependencies"][name]["version"]
216            if depkey not in cached_manifest:
217                cached_manifest[depkey] = {}
218            cached_manifest[depkey][name] = version
219
220    if has_shrinkwrap_file:
221        _update_manifest("dependencies")
222
223    if dev:
224        if has_shrinkwrap_file:
225            _update_manifest("devDependencies")
226
227    os.chmod(cached_manifest_file, os.stat(cached_manifest_file).st_mode | stat.S_IWUSR)
228    with open(cached_manifest_file, "w") as f:
229        json.dump(cached_manifest, f, indent=2)
230
231    if has_shrinkwrap_file:
232        with open(cached_shrinkwrap_file, "w") as f:
233            json.dump(cached_shrinkwrap, f, indent=2)
234}
235
236python npm_do_compile() {
237    """
238    Step two: install the npm package
239
240    Use the configured main package and the cached dependencies to run the
241    installation process. The installation is done in a directory which is
242    not the destination directory yet.
243
244    A combination of 'npm pack' and 'npm install' is used to ensure that the
245    installed files are actual copies instead of symbolic links (which is the
246    default npm behavior).
247    """
248    import shlex
249    import tempfile
250    from bb.fetch2.npm import NpmEnvironment
251
252    bb.utils.remove(d.getVar("NPM_BUILD"), recurse=True)
253
254    with tempfile.TemporaryDirectory() as tmpdir:
255        args = []
256        configs = npm_global_configs(d)
257
258        if bb.utils.to_boolean(d.getVar("NPM_INSTALL_DEV"), False):
259            configs.append(("also", "development"))
260        else:
261            configs.append(("only", "production"))
262
263        # Report as many logs as possible for debugging purpose
264        configs.append(("loglevel", "silly"))
265
266        # Configure the installation to be done globally in the build directory
267        configs.append(("global", "true"))
268        configs.append(("prefix", d.getVar("NPM_BUILD")))
269
270        # Add node-gyp configuration
271        configs.append(("arch", d.getVar("NPM_ARCH")))
272        configs.append(("release", "true"))
273        configs.append(("nodedir", d.getVar("NPM_NODEDIR")))
274        configs.append(("python", d.getVar("PYTHON")))
275
276        env = NpmEnvironment(d, configs)
277
278        # Add node-pre-gyp configuration
279        args.append(("target_arch", d.getVar("NPM_ARCH")))
280        args.append(("build-from-source", "true"))
281
282        # Pack and install the main package
283        (tarball, _) = npm_pack(env, d.getVar("NPM_PACKAGE"), tmpdir)
284        cmd = "npm install %s %s" % (shlex.quote(tarball), d.getVar("EXTRA_OENPM"))
285        env.run(cmd, args=args)
286}
287
288npm_do_install() {
289    # Step three: final install
290    #
291    # The previous installation have to be filtered to remove some extra files.
292
293    rm -rf ${D}
294
295    # Copy the entire lib and bin directories
296    install -d ${D}/${nonarch_libdir}
297    cp --no-preserve=ownership --recursive ${NPM_BUILD}/lib/. ${D}/${nonarch_libdir}
298
299    if [ -d "${NPM_BUILD}/bin" ]
300    then
301        install -d ${D}/${bindir}
302        cp --no-preserve=ownership --recursive ${NPM_BUILD}/bin/. ${D}/${bindir}
303    fi
304
305    # If the package (or its dependencies) uses node-gyp to build native addons,
306    # object files, static libraries or other temporary files can be hidden in
307    # the lib directory. To reduce the package size and to avoid QA issues
308    # (staticdev with static library files) these files must be removed.
309    local GYP_REGEX=".*/build/Release/[^/]*.node"
310
311    # Remove any node-gyp directory in ${D} to remove temporary build files
312    for GYP_D_FILE in $(find ${D} -regex "${GYP_REGEX}")
313    do
314        local GYP_D_DIR=${GYP_D_FILE%/Release/*}
315
316        rm --recursive --force ${GYP_D_DIR}
317    done
318
319    # Copy only the node-gyp release files
320    for GYP_B_FILE in $(find ${NPM_BUILD} -regex "${GYP_REGEX}")
321    do
322        local GYP_D_FILE=${D}/${prefix}/${GYP_B_FILE#${NPM_BUILD}}
323
324        install -d ${GYP_D_FILE%/*}
325        install -m 755 ${GYP_B_FILE} ${GYP_D_FILE}
326    done
327
328    # Remove the shrinkwrap file which does not need to be packed
329    rm -f ${D}/${nonarch_libdir}/node_modules/*/npm-shrinkwrap.json
330    rm -f ${D}/${nonarch_libdir}/node_modules/@*/*/npm-shrinkwrap.json
331}
332
333FILES:${PN} += " \
334    ${bindir} \
335    ${nonarch_libdir} \
336"
337
338EXPORT_FUNCTIONS do_configure do_compile do_install
339