xref: /openbmc/openbmc/poky/meta/lib/oe/spdx_common.py (revision c9537f57ab488bf5d90132917b0184e2527970a5) 
1#
2# Copyright OpenEmbedded Contributors
3#
4# SPDX-License-Identifier: GPL-2.0-only
5#
6
7import bb
8import collections
9import json
10import oe.packagedata
11import re
12import shutil
13
14from pathlib import Path
15from dataclasses import dataclass
16
17LIC_REGEX = re.compile(
18    rb"^\W*SPDX-License-Identifier:\s*([ \w\d.()+-]+?)(?:\s+\W*)?$",
19    re.MULTILINE,
20)
21
22
23def extract_licenses(filename):
24    """
25    Extract SPDX License identifiers from a file
26    """
27    try:
28        with open(filename, "rb") as f:
29            size = min(15000, os.stat(filename).st_size)
30            txt = f.read(size)
31            licenses = re.findall(LIC_REGEX, txt)
32            if licenses:
33                ascii_licenses = [lic.decode("ascii") for lic in licenses]
34                return ascii_licenses
35    except Exception as e:
36        bb.warn(f"Exception reading {filename}: {e}")
37    return []
38
39
40def is_work_shared_spdx(d):
41    return '/work-shared/' in d.getVar('S')
42
43
44def load_spdx_license_data(d):
45    with open(d.getVar("SPDX_LICENSES"), "r") as f:
46        data = json.load(f)
47        # Transform the license array to a dictionary
48        data["licenses"] = {l["licenseId"]: l for l in data["licenses"]}
49
50    return data
51
52
53def process_sources(d):
54    """
55    Returns True if the sources for this recipe should be included in the SPDX
56    or False if not
57    """
58    pn = d.getVar("PN")
59    assume_provided = (d.getVar("ASSUME_PROVIDED") or "").split()
60    if pn in assume_provided:
61        for p in d.getVar("PROVIDES").split():
62            if p != pn:
63                pn = p
64                break
65
66    # glibc-locale: do_fetch, do_unpack and do_patch tasks have been deleted,
67    # so avoid archiving source here.
68    if pn.startswith("glibc-locale"):
69        return False
70    if d.getVar("PN") == "libtool-cross":
71        return False
72    if d.getVar("PN") == "libgcc-initial":
73        return False
74    if d.getVar("PN") == "shadow-sysroot":
75        return False
76
77    return True
78
79
80@dataclass(frozen=True)
81class Dep(object):
82    pn: str
83    hashfn: str
84    in_taskhash: bool
85
86
87def collect_direct_deps(d, dep_task):
88    """
89    Find direct dependencies of current task
90
91    Returns the list of recipes that have a dep_task that the current task
92    depends on
93    """
94    current_task = "do_" + d.getVar("BB_CURRENTTASK")
95    pn = d.getVar("PN")
96
97    taskdepdata = d.getVar("BB_TASKDEPDATA", False)
98
99    for this_dep in taskdepdata.values():
100        if this_dep[0] == pn and this_dep[1] == current_task:
101            break
102    else:
103        bb.fatal(f"Unable to find this {pn}:{current_task} in taskdepdata")
104
105    deps = set()
106
107    for dep_name in this_dep.deps:
108        dep_data = taskdepdata[dep_name]
109        if dep_data.taskname == dep_task and dep_data.pn != pn:
110            deps.add((dep_data.pn, dep_data.hashfn, dep_name in this_dep.taskhash_deps))
111
112    return sorted(deps)
113
114
115def get_spdx_deps(d):
116    """
117    Reads the SPDX dependencies JSON file and returns the data
118    """
119    spdx_deps_file = Path(d.getVar("SPDXDEPS"))
120
121    deps = []
122    with spdx_deps_file.open("r") as f:
123        for d in json.load(f):
124            deps.append(Dep(*d))
125    return deps
126
127
128def collect_package_providers(d):
129    """
130    Returns a dictionary where each RPROVIDES is mapped to the package that
131    provides it
132    """
133    deploy_dir_spdx = Path(d.getVar("DEPLOY_DIR_SPDX"))
134
135    providers = {}
136
137    deps = collect_direct_deps(d, "do_create_spdx")
138    deps.append((d.getVar("PN"), d.getVar("BB_HASHFILENAME"), True))
139
140    for dep_pn, dep_hashfn, _ in deps:
141        localdata = d
142        recipe_data = oe.packagedata.read_pkgdata(dep_pn, localdata)
143        if not recipe_data:
144            localdata = bb.data.createCopy(d)
145            localdata.setVar("PKGDATA_DIR", "${PKGDATA_DIR_SDK}")
146            recipe_data = oe.packagedata.read_pkgdata(dep_pn, localdata)
147
148        for pkg in recipe_data.get("PACKAGES", "").split():
149            pkg_data = oe.packagedata.read_subpkgdata_dict(pkg, localdata)
150            rprovides = set(
151                n
152                for n, _ in bb.utils.explode_dep_versions2(
153                    pkg_data.get("RPROVIDES", "")
154                ).items()
155            )
156            rprovides.add(pkg)
157
158            if "PKG" in pkg_data:
159                pkg = pkg_data["PKG"]
160                rprovides.add(pkg)
161
162            for r in rprovides:
163                providers[r] = (pkg, dep_hashfn)
164
165    return providers
166
167
168def get_patched_src(d):
169    """
170    Save patched source of the recipe in SPDX_WORKDIR.
171    """
172    spdx_workdir = d.getVar("SPDXWORK")
173    spdx_sysroot_native = d.getVar("STAGING_DIR_NATIVE")
174    pn = d.getVar("PN")
175
176    workdir = d.getVar("WORKDIR")
177
178    try:
179        # The kernel class functions require it to be on work-shared, so we dont change WORKDIR
180        if not is_work_shared_spdx(d):
181            # Change the WORKDIR to make do_unpack do_patch run in another dir.
182            d.setVar("WORKDIR", spdx_workdir)
183            # Restore the original path to recipe's native sysroot (it's relative to WORKDIR).
184            d.setVar("STAGING_DIR_NATIVE", spdx_sysroot_native)
185
186            # The changed 'WORKDIR' also caused 'B' changed, create dir 'B' for the
187            # possibly requiring of the following tasks (such as some recipes's
188            # do_patch required 'B' existed).
189            bb.utils.mkdirhier(d.getVar("B"))
190
191            bb.build.exec_func("do_unpack", d)
192
193            if d.getVar("SRC_URI") != "":
194                if bb.data.inherits_class('dos2unix', d):
195                    bb.build.exec_func('do_convert_crlf_to_lf', d)
196                bb.build.exec_func("do_patch", d)
197
198        # Copy source from work-share to spdx_workdir
199        if is_work_shared_spdx(d):
200            share_src = d.getVar('S')
201            d.setVar("WORKDIR", spdx_workdir)
202            d.setVar("STAGING_DIR_NATIVE", spdx_sysroot_native)
203            # Copy source to ${SPDXWORK}, same basename dir of ${S};
204            src_dir = (
205                spdx_workdir
206                + "/"
207                + os.path.basename(share_src)
208            )
209            # For kernel souce, rename suffix dir 'kernel-source'
210            # to ${BP} (${BPN}-${PV})
211            if bb.data.inherits_class("kernel", d):
212                src_dir = spdx_workdir + "/" + d.getVar('BP')
213
214            bb.note(f"copyhardlinktree {share_src} to {src_dir}")
215            oe.path.copyhardlinktree(share_src, src_dir)
216
217        # Some userland has no source.
218        if not os.path.exists(spdx_workdir):
219            bb.utils.mkdirhier(spdx_workdir)
220    finally:
221        d.setVar("WORKDIR", workdir)
222
223
224def has_task(d, task):
225    return bool(d.getVarFlag(task, "task", False)) and not bool(d.getVarFlag(task, "noexec", False))
226
227
228def fetch_data_to_uri(fd, name):
229    """
230    Translates a bitbake FetchData to a string URI
231    """
232    uri = fd.type
233    # Map gitsm to git, since gitsm:// is not a valid URI protocol
234    if uri == "gitsm":
235        uri = "git"
236    proto = getattr(fd, "proto", None)
237    if proto is not None:
238        uri = uri + "+" + proto
239    uri = uri + "://" + fd.host + fd.path
240
241    if fd.method.supports_srcrev():
242        uri = uri + "@" + fd.revision
243
244    return uri
245