1# Copyright (C) 2016 Intel Corporation
2# Copyright (C) 2020 Savoir-Faire Linux
3#
4# SPDX-License-Identifier: GPL-2.0-only
5#
6"""Recipe creation tool - npm module support plugin"""
7
8import json
9import logging
10import os
11import re
12import sys
13import tempfile
14import bb
15from bb.fetch2.npm import NpmEnvironment
16from bb.fetch2.npm import npm_package
17from bb.fetch2.npmsw import foreach_dependencies
18from recipetool.create import RecipeHandler
19from recipetool.create import get_license_md5sums
20from recipetool.create import guess_license
21from recipetool.create import split_pkg_licenses
22logger = logging.getLogger('recipetool')
23
24TINFOIL = None
25
26def tinfoil_init(instance):
27    """Initialize tinfoil"""
28    global TINFOIL
29    TINFOIL = instance
30
31class NpmRecipeHandler(RecipeHandler):
32    """Class to handle the npm recipe creation"""
33
34    @staticmethod
35    def _get_registry(lines):
36        """Get the registry value from the 'npm://registry' url"""
37        registry = None
38
39        def _handle_registry(varname, origvalue, op, newlines):
40            nonlocal registry
41            if origvalue.startswith("npm://"):
42                registry = re.sub(r"^npm://", "http://", origvalue.split(";")[0])
43            return origvalue, None, 0, True
44
45        bb.utils.edit_metadata(lines, ["SRC_URI"], _handle_registry)
46
47        return registry
48
49    @staticmethod
50    def _ensure_npm():
51        """Check if the 'npm' command is available in the recipes"""
52        if not TINFOIL.recipes_parsed:
53            TINFOIL.parse_recipes()
54
55        try:
56            d = TINFOIL.parse_recipe("nodejs-native")
57        except bb.providers.NoProvider:
58            bb.error("Nothing provides 'nodejs-native' which is required for the build")
59            bb.note("You will likely need to add a layer that provides nodejs")
60            sys.exit(14)
61
62        bindir = d.getVar("STAGING_BINDIR_NATIVE")
63        npmpath = os.path.join(bindir, "npm")
64
65        if not os.path.exists(npmpath):
66            TINFOIL.build_targets("nodejs-native", "addto_recipe_sysroot")
67
68            if not os.path.exists(npmpath):
69                bb.error("Failed to add 'npm' to sysroot")
70                sys.exit(14)
71
72        return bindir
73
74    @staticmethod
75    def _npm_global_configs(dev):
76        """Get the npm global configuration"""
77        configs = []
78
79        if dev:
80            configs.append(("also", "development"))
81        else:
82            configs.append(("only", "production"))
83
84        configs.append(("save", "false"))
85        configs.append(("package-lock", "false"))
86        configs.append(("shrinkwrap", "false"))
87        return configs
88
89    def _run_npm_install(self, d, srctree, registry, dev):
90        """Run the 'npm install' command without building the addons"""
91        configs = self._npm_global_configs(dev)
92        configs.append(("ignore-scripts", "true"))
93
94        if registry:
95            configs.append(("registry", registry))
96
97        bb.utils.remove(os.path.join(srctree, "node_modules"), recurse=True)
98
99        env = NpmEnvironment(d, configs=configs)
100        env.run("npm install", workdir=srctree)
101
102    def _generate_shrinkwrap(self, d, srctree, dev):
103        """Check and generate the 'npm-shrinkwrap.json' file if needed"""
104        configs = self._npm_global_configs(dev)
105
106        env = NpmEnvironment(d, configs=configs)
107        env.run("npm shrinkwrap", workdir=srctree)
108
109        return os.path.join(srctree, "npm-shrinkwrap.json")
110
111    def _handle_licenses(self, srctree, shrinkwrap_file, dev):
112        """Return the extra license files and the list of packages"""
113        licfiles = []
114        packages = {}
115
116        # Handle the parent package
117        packages["${PN}"] = ""
118
119        def _licfiles_append_fallback_readme_files(destdir):
120            """Append README files as fallback to license files if a license files is missing"""
121
122            fallback = True
123            readmes = []
124            basedir = os.path.join(srctree, destdir)
125            for fn in os.listdir(basedir):
126                upper = fn.upper()
127                if upper.startswith("README"):
128                    fullpath = os.path.join(basedir, fn)
129                    readmes.append(fullpath)
130                if upper.startswith("COPYING") or "LICENCE" in upper or "LICENSE" in upper:
131                    fallback = False
132            if fallback:
133                for readme in readmes:
134                    licfiles.append(os.path.relpath(readme, srctree))
135
136        # Handle the dependencies
137        def _handle_dependency(name, params, destdir):
138            deptree = destdir.split('node_modules/')
139            suffix = "-".join([npm_package(dep) for dep in deptree])
140            packages["${PN}" + suffix] = destdir
141            _licfiles_append_fallback_readme_files(destdir)
142
143        with open(shrinkwrap_file, "r") as f:
144            shrinkwrap = json.load(f)
145
146        foreach_dependencies(shrinkwrap, _handle_dependency, dev)
147
148        return licfiles, packages
149
150    # Handle the peer dependencies
151    def _handle_peer_dependency(self, shrinkwrap_file):
152        """Check if package has peer dependencies and show warning if it is the case"""
153        with open(shrinkwrap_file, "r") as f:
154            shrinkwrap = json.load(f)
155
156        packages = shrinkwrap.get("packages", {})
157        peer_deps = packages.get("", {}).get("peerDependencies", {})
158
159        for peer_dep in peer_deps:
160            peer_dep_yocto_name = npm_package(peer_dep)
161            bb.warn(peer_dep + " is a peer dependencie of the actual package. " +
162            "Please add this peer dependencie to the RDEPENDS variable as %s and generate its recipe with devtool"
163            % peer_dep_yocto_name)
164
165
166
167    def process(self, srctree, classes, lines_before, lines_after, handled, extravalues):
168        """Handle the npm recipe creation"""
169
170        if "buildsystem" in handled:
171            return False
172
173        files = RecipeHandler.checkfiles(srctree, ["package.json"])
174
175        if not files:
176            return False
177
178        with open(files[0], "r") as f:
179            data = json.load(f)
180
181        if "name" not in data or "version" not in data:
182            return False
183
184        extravalues["PN"] = npm_package(data["name"])
185        extravalues["PV"] = data["version"]
186
187        if "description" in data:
188            extravalues["SUMMARY"] = data["description"]
189
190        if "homepage" in data:
191            extravalues["HOMEPAGE"] = data["homepage"]
192
193        dev = bb.utils.to_boolean(str(extravalues.get("NPM_INSTALL_DEV", "0")), False)
194        registry = self._get_registry(lines_before)
195
196        bb.note("Checking if npm is available ...")
197        # The native npm is used here (and not the host one) to ensure that the
198        # npm version is high enough to ensure an efficient dependency tree
199        # resolution and avoid issue with the shrinkwrap file format.
200        # Moreover the native npm is mandatory for the build.
201        bindir = self._ensure_npm()
202
203        d = bb.data.createCopy(TINFOIL.config_data)
204        d.prependVar("PATH", bindir + ":")
205        d.setVar("S", srctree)
206
207        bb.note("Generating shrinkwrap file ...")
208        # To generate the shrinkwrap file the dependencies have to be installed
209        # first. During the generation process some files may be updated /
210        # deleted. By default devtool tracks the diffs in the srctree and raises
211        # errors when finishing the recipe if some diffs are found.
212        git_exclude_file = os.path.join(srctree, ".git", "info", "exclude")
213        if os.path.exists(git_exclude_file):
214            with open(git_exclude_file, "r+") as f:
215                lines = f.readlines()
216                for line in ["/node_modules/", "/npm-shrinkwrap.json"]:
217                    if line not in lines:
218                        f.write(line + "\n")
219
220        lock_file = os.path.join(srctree, "package-lock.json")
221        lock_copy = lock_file + ".copy"
222        if os.path.exists(lock_file):
223            bb.utils.copyfile(lock_file, lock_copy)
224
225        self._run_npm_install(d, srctree, registry, dev)
226        shrinkwrap_file = self._generate_shrinkwrap(d, srctree, dev)
227
228        with open(shrinkwrap_file, "r") as f:
229            shrinkwrap = json.load(f)
230
231        if os.path.exists(lock_copy):
232            bb.utils.movefile(lock_copy, lock_file)
233
234        # Add the shrinkwrap file as 'extrafiles'
235        shrinkwrap_copy = shrinkwrap_file + ".copy"
236        bb.utils.copyfile(shrinkwrap_file, shrinkwrap_copy)
237        extravalues.setdefault("extrafiles", {})
238        extravalues["extrafiles"]["npm-shrinkwrap.json"] = shrinkwrap_copy
239
240        url_local = "npmsw://%s" % shrinkwrap_file
241        url_recipe= "npmsw://${THISDIR}/${BPN}/npm-shrinkwrap.json"
242
243        if dev:
244            url_local += ";dev=1"
245            url_recipe += ";dev=1"
246
247        # Add the npmsw url in the SRC_URI of the generated recipe
248        def _handle_srcuri(varname, origvalue, op, newlines):
249            """Update the version value and add the 'npmsw://' url"""
250            value = origvalue.replace("version=" + data["version"], "version=${PV}")
251            value = value.replace("version=latest", "version=${PV}")
252            values = [line.strip() for line in value.strip('\n').splitlines()]
253            if "dependencies" in shrinkwrap.get("packages", {}).get("", {}):
254                values.append(url_recipe)
255            return values, None, 4, False
256
257        (_, newlines) = bb.utils.edit_metadata(lines_before, ["SRC_URI"], _handle_srcuri)
258        lines_before[:] = [line.rstrip('\n') for line in newlines]
259
260        # In order to generate correct licence checksums in the recipe the
261        # dependencies have to be fetched again using the npmsw url
262        bb.note("Fetching npm dependencies ...")
263        bb.utils.remove(os.path.join(srctree, "node_modules"), recurse=True)
264        fetcher = bb.fetch2.Fetch([url_local], d)
265        fetcher.download()
266        fetcher.unpack(srctree)
267
268        bb.note("Handling licences ...")
269        (licfiles, packages) = self._handle_licenses(srctree, shrinkwrap_file, dev)
270
271        def _guess_odd_license(licfiles):
272            import bb
273
274            md5sums = get_license_md5sums(d, linenumbers=True)
275
276            chksums = []
277            licenses = []
278            for licfile in licfiles:
279                f = os.path.join(srctree, licfile)
280                md5value = bb.utils.md5_file(f)
281                (license, beginline, endline, md5) = md5sums.get(md5value,
282                    (None, "", "", ""))
283                if not license:
284                    license = "Unknown"
285                    logger.info("Please add the following line for '%s' to a "
286                        "'lib/recipetool/licenses.csv' and replace `Unknown`, "
287                        "`X`, `Y` and `MD5` with the license, begin line, "
288                        "end line and partial MD5 checksum:\n" \
289                        "%s,Unknown,X,Y,MD5" % (licfile, md5value))
290                chksums.append("file://%s%s%s;md5=%s" % (licfile,
291                    ";beginline=%s" % (beginline) if beginline else "",
292                    ";endline=%s" % (endline) if endline else "",
293                    md5 if md5 else md5value))
294                licenses.append((license, licfile, md5value))
295            return (licenses, chksums)
296
297        (licenses, extravalues["LIC_FILES_CHKSUM"]) = _guess_odd_license(licfiles)
298        split_pkg_licenses([*licenses, *guess_license(srctree, d)], packages, lines_after)
299
300        classes.append("npm")
301        handled.append("buildsystem")
302
303        # Check if package has peer dependencies and inform the user
304        self._handle_peer_dependency(shrinkwrap_file)
305
306        return True
307
308def register_recipe_handlers(handlers):
309    """Register the npm handler"""
310    handlers.append((NpmRecipeHandler(), 60))
311