xref: /openbmc/openbmc/poky/meta/lib/oe/license.py (revision c124f4f2e04dca16a428a76c89677328bc7bf908)
1#
2# Copyright OpenEmbedded Contributors
3#
4# SPDX-License-Identifier: GPL-2.0-only
5#
6"""Code for parsing OpenEmbedded license strings"""
7
8import ast
9import re
10import oe.qa
11from fnmatch import fnmatchcase as fnmatch
12
13def license_ok(license, dont_want_licenses):
14    """ Return False if License exist in dont_want_licenses else True """
15    for dwl in dont_want_licenses:
16        if fnmatch(license, dwl):
17            return False
18    return True
19
20def obsolete_license_list():
21    return ["AGPL-3", "AGPL-3+", "AGPLv3", "AGPLv3+", "AGPLv3.0", "AGPLv3.0+", "AGPL-3.0", "AGPL-3.0+", "BSD-0-Clause",
22            "GPL-1", "GPL-1+", "GPLv1", "GPLv1+", "GPLv1.0", "GPLv1.0+", "GPL-1.0", "GPL-1.0+", "GPL-2", "GPL-2+", "GPLv2",
23            "GPLv2+", "GPLv2.0", "GPLv2.0+", "GPL-2.0", "GPL-2.0+", "GPL-3", "GPL-3+", "GPLv3", "GPLv3+", "GPLv3.0", "GPLv3.0+",
24            "GPL-3.0", "GPL-3.0+", "LGPLv2", "LGPLv2+", "LGPLv2.0", "LGPLv2.0+", "LGPL-2.0", "LGPL-2.0+", "LGPL2.1", "LGPL2.1+",
25            "LGPLv2.1", "LGPLv2.1+", "LGPL-2.1", "LGPL-2.1+", "LGPLv3", "LGPLv3+", "LGPL-3.0", "LGPL-3.0+", "MPL-1", "MPLv1",
26            "MPLv1.1", "MPLv2", "MIT-X", "MIT-style", "openssl", "PSF", "PSFv2", "Python-2", "Apachev2", "Apache-2", "Artisticv1",
27            "Artistic-1", "AFL-2", "AFL-1", "AFLv2", "AFLv1", "CDDLv1", "CDDL-1", "EPLv1.0", "FreeType", "Nauman",
28            "tcl", "vim", "SGIv1"]
29
30class LicenseError(Exception):
31    pass
32
33class LicenseSyntaxError(LicenseError):
34    def __init__(self, licensestr, exc):
35        self.licensestr = licensestr
36        self.exc = exc
37        LicenseError.__init__(self)
38
39    def __str__(self):
40        return "error in '%s': %s" % (self.licensestr, self.exc)
41
42class InvalidLicense(LicenseError):
43    def __init__(self, license):
44        self.license = license
45        LicenseError.__init__(self)
46
47    def __str__(self):
48        return "invalid characters in license '%s'" % self.license
49
50license_operator_chars = '&|() '
51license_operator = re.compile(r'([' + license_operator_chars + '])')
52license_pattern = re.compile(r'[a-zA-Z0-9.+_\-]+$')
53
54class LicenseVisitor(ast.NodeVisitor):
55    """Get elements based on OpenEmbedded license strings"""
56    def get_elements(self, licensestr):
57        new_elements = []
58        elements = list([x for x in license_operator.split(licensestr) if x.strip()])
59        for pos, element in enumerate(elements):
60            if license_pattern.match(element):
61                if pos > 0 and license_pattern.match(elements[pos-1]):
62                    new_elements.append('&')
63                element = '"' + element + '"'
64            elif not license_operator.match(element):
65                raise InvalidLicense(element)
66            new_elements.append(element)
67
68        return new_elements
69
70    """Syntax tree visitor which can accept elements previously generated with
71    OpenEmbedded license string"""
72    def visit_elements(self, elements):
73        self.visit(ast.parse(' '.join(elements)))
74
75    """Syntax tree visitor which can accept OpenEmbedded license strings"""
76    def visit_string(self, licensestr):
77        self.visit_elements(self.get_elements(licensestr))
78
79class FlattenVisitor(LicenseVisitor):
80    """Flatten a license tree (parsed from a string) by selecting one of each
81    set of OR options, in the way the user specifies"""
82    def __init__(self, choose_licenses):
83        self.choose_licenses = choose_licenses
84        self.licenses = []
85        LicenseVisitor.__init__(self)
86
87    def visit_Str(self, node):
88        self.licenses.append(node.s)
89
90    def visit_Constant(self, node):
91        self.licenses.append(node.value)
92
93    def visit_BinOp(self, node):
94        if isinstance(node.op, ast.BitOr):
95            left = FlattenVisitor(self.choose_licenses)
96            left.visit(node.left)
97
98            right = FlattenVisitor(self.choose_licenses)
99            right.visit(node.right)
100
101            selected = self.choose_licenses(left.licenses, right.licenses)
102            self.licenses.extend(selected)
103        else:
104            self.generic_visit(node)
105
106def flattened_licenses(licensestr, choose_licenses):
107    """Given a license string and choose_licenses function, return a flat list of licenses"""
108    flatten = FlattenVisitor(choose_licenses)
109    try:
110        flatten.visit_string(licensestr)
111    except SyntaxError as exc:
112        raise LicenseSyntaxError(licensestr, exc)
113    return flatten.licenses
114
115def is_included(licensestr, include_licenses=None, exclude_licenses=None):
116    """Given a license string, a list of licenses to include and a list of
117    licenses to exclude, determine if the license string matches the include
118    list and does not match the exclude list.
119
120    Returns a tuple holding the boolean state and a list of the applicable
121    licenses that were excluded if state is False, or the licenses that were
122    included if the state is True."""
123
124    def include_license(license):
125        return any(fnmatch(license, pattern) for pattern in include_licenses)
126
127    def exclude_license(license):
128        return any(fnmatch(license, pattern) for pattern in exclude_licenses)
129
130    def choose_licenses(alpha, beta):
131        """Select the option in an OR which is the 'best' (has the most
132        included licenses and no excluded licenses)."""
133        # The factor 1000 below is arbitrary, just expected to be much larger
134        # than the number of licenses actually specified. That way the weight
135        # will be negative if the list of licenses contains an excluded license,
136        # but still gives a higher weight to the list with the most included
137        # licenses.
138        alpha_weight = (len(list(filter(include_license, alpha))) -
139                        1000 * (len(list(filter(exclude_license, alpha))) > 0))
140        beta_weight = (len(list(filter(include_license, beta))) -
141                       1000 * (len(list(filter(exclude_license, beta))) > 0))
142        if alpha_weight >= beta_weight:
143            return alpha
144        else:
145            return beta
146
147    if not include_licenses:
148        include_licenses = ['*']
149
150    if not exclude_licenses:
151        exclude_licenses = []
152
153    licenses = flattened_licenses(licensestr, choose_licenses)
154    excluded = [lic for lic in licenses if exclude_license(lic)]
155    included = [lic for lic in licenses if include_license(lic)]
156    if excluded:
157        return False, excluded
158    else:
159        return True, included
160
161class ManifestVisitor(LicenseVisitor):
162    """Walk license tree (parsed from a string) removing the incompatible
163    licenses specified"""
164    def __init__(self, dont_want_licenses, canonical_license, d):
165        self._dont_want_licenses = dont_want_licenses
166        self._canonical_license = canonical_license
167        self._d = d
168        self._operators = []
169
170        self.licenses = []
171        self.licensestr = ''
172
173        LicenseVisitor.__init__(self)
174
175    def visit(self, node):
176        if isinstance(node, ast.Str):
177            lic = node.s
178
179            if license_ok(self._canonical_license(self._d, lic),
180                    self._dont_want_licenses) == True:
181                if self._operators:
182                    ops = []
183                    for op in self._operators:
184                        if op == '[':
185                            ops.append(op)
186                        elif op == ']':
187                            ops.append(op)
188                        else:
189                            if not ops:
190                                ops.append(op)
191                            elif ops[-1] in ['[', ']']:
192                                ops.append(op)
193                            else:
194                                ops[-1] = op
195
196                    for op in ops:
197                        if op == '[' or op == ']':
198                            self.licensestr += op
199                        elif self.licenses:
200                            self.licensestr += ' ' + op + ' '
201
202                    self._operators = []
203
204                self.licensestr += lic
205                self.licenses.append(lic)
206        elif isinstance(node, ast.BitAnd):
207            self._operators.append("&")
208        elif isinstance(node, ast.BitOr):
209            self._operators.append("|")
210        elif isinstance(node, ast.List):
211            self._operators.append("[")
212        elif isinstance(node, ast.Load):
213            self.licensestr += "]"
214
215        self.generic_visit(node)
216
217def manifest_licenses(licensestr, dont_want_licenses, canonical_license, d):
218    """Given a license string and dont_want_licenses list,
219       return license string filtered and a list of licenses"""
220    manifest = ManifestVisitor(dont_want_licenses, canonical_license, d)
221
222    try:
223        elements = manifest.get_elements(licensestr)
224
225        # Replace '()' to '[]' for handle in ast as List and Load types.
226        elements = ['[' if e == '(' else e for e in elements]
227        elements = [']' if e == ')' else e for e in elements]
228
229        manifest.visit_elements(elements)
230    except SyntaxError as exc:
231        raise LicenseSyntaxError(licensestr, exc)
232
233    # Replace '[]' to '()' for output correct license.
234    manifest.licensestr = manifest.licensestr.replace('[', '(').replace(']', ')')
235
236    return (manifest.licensestr, manifest.licenses)
237
238class ListVisitor(LicenseVisitor):
239    """Record all different licenses found in the license string"""
240    def __init__(self):
241        self.licenses = set()
242
243    def visit_Str(self, node):
244        self.licenses.add(node.s)
245
246    def visit_Constant(self, node):
247        self.licenses.add(node.value)
248
249def list_licenses(licensestr):
250    """Simply get a list of all licenses mentioned in a license string.
251       Binary operators are not applied or taken into account in any way"""
252    visitor = ListVisitor()
253    try:
254        visitor.visit_string(licensestr)
255    except SyntaxError as exc:
256        raise LicenseSyntaxError(licensestr, exc)
257    return visitor.licenses
258
259def apply_pkg_license_exception(pkg, bad_licenses, exceptions):
260    """Return remaining bad licenses after removing any package exceptions"""
261
262    return [lic for lic in bad_licenses if pkg + ':' + lic not in exceptions]
263
264def return_spdx(d, license):
265    """
266    This function returns the spdx mapping of a license if it exists.
267     """
268    return d.getVarFlag('SPDXLICENSEMAP', license)
269
270def canonical_license(d, license):
271    """
272    Return the canonical (SPDX) form of the license if available (so GPLv3
273    becomes GPL-3.0-only) or the passed license if there is no canonical form.
274    """
275    return d.getVarFlag('SPDXLICENSEMAP', license) or license
276
277def expand_wildcard_licenses(d, wildcard_licenses):
278    """
279    There are some common wildcard values users may want to use. Support them
280    here.
281    """
282    licenses = set(wildcard_licenses)
283    mapping = {
284        "AGPL-3.0*" : ["AGPL-3.0-only", "AGPL-3.0-or-later"],
285        "GPL-3.0*" : ["GPL-3.0-only", "GPL-3.0-or-later"],
286        "LGPL-3.0*" : ["LGPL-3.0-only", "LGPL-3.0-or-later"],
287    }
288    for k in mapping:
289        if k in wildcard_licenses:
290            licenses.remove(k)
291            for item in mapping[k]:
292                licenses.add(item)
293
294    for l in licenses:
295        if l in obsolete_license_list():
296            bb.fatal("Error, %s is an obsolete license, please use an SPDX reference in INCOMPATIBLE_LICENSE" % l)
297        if "*" in l:
298            bb.fatal("Error, %s is an invalid license wildcard entry" % l)
299
300    return list(licenses)
301
302def incompatible_license_contains(license, truevalue, falsevalue, d):
303    license = canonical_license(d, license)
304    bad_licenses = (d.getVar('INCOMPATIBLE_LICENSE') or "").split()
305    bad_licenses = expand_wildcard_licenses(d, bad_licenses)
306    return truevalue if license in bad_licenses else falsevalue
307
308def incompatible_pkg_license(d, dont_want_licenses, license):
309    # Handles an "or" or two license sets provided by
310    # flattened_licenses(), pick one that works if possible.
311    def choose_lic_set(a, b):
312        return a if all(license_ok(canonical_license(d, lic),
313                            dont_want_licenses) for lic in a) else b
314
315    try:
316        licenses = flattened_licenses(license, choose_lic_set)
317    except LicenseError as exc:
318        bb.fatal('%s: %s' % (d.getVar('P'), exc))
319
320    incompatible_lic = []
321    for l in licenses:
322        license = canonical_license(d, l)
323        if not license_ok(license, dont_want_licenses):
324            incompatible_lic.append(license)
325
326    return sorted(incompatible_lic)
327
328def incompatible_license(d, dont_want_licenses, package=None):
329    """
330    This function checks if a recipe has only incompatible licenses. It also
331    take into consideration 'or' operand.  dont_want_licenses should be passed
332    as canonical (SPDX) names.
333    """
334    license = d.getVar("LICENSE:%s" % package) if package else None
335    if not license:
336        license = d.getVar('LICENSE')
337
338    return incompatible_pkg_license(d, dont_want_licenses, license)
339
340def check_license_flags(d):
341    """
342    This function checks if a recipe has any LICENSE_FLAGS that
343    aren't acceptable.
344
345    If it does, it returns the all LICENSE_FLAGS missing from the list
346    of acceptable license flags, or all of the LICENSE_FLAGS if there
347    is no list of acceptable flags.
348
349    If everything is is acceptable, it returns None.
350    """
351
352    def license_flag_matches(flag, acceptlist, pn):
353        """
354        Return True if flag matches something in acceptlist, None if not.
355
356        Before we test a flag against the acceptlist, we append _${PN}
357        to it.  We then try to match that string against the
358        acceptlist.  This covers the normal case, where we expect
359        LICENSE_FLAGS to be a simple string like 'commercial', which
360        the user typically matches exactly in the acceptlist by
361        explicitly appending the package name e.g 'commercial_foo'.
362        If we fail the match however, we then split the flag across
363        '_' and append each fragment and test until we either match or
364        run out of fragments.
365        """
366        flag_pn = ("%s_%s" % (flag, pn))
367        for candidate in acceptlist:
368            if flag_pn == candidate:
369                    return True
370
371        flag_cur = ""
372        flagments = flag_pn.split("_")
373        flagments.pop() # we've already tested the full string
374        for flagment in flagments:
375            if flag_cur:
376                flag_cur += "_"
377            flag_cur += flagment
378            for candidate in acceptlist:
379                if flag_cur == candidate:
380                    return True
381        return False
382
383    def all_license_flags_match(license_flags, acceptlist):
384        """ Return all unmatched flags, None if all flags match """
385        pn = d.getVar('PN')
386        split_acceptlist = acceptlist.split()
387        flags = []
388        for flag in license_flags.split():
389            if not license_flag_matches(flag, split_acceptlist, pn):
390                flags.append(flag)
391        return flags if flags else None
392
393    license_flags = d.getVar('LICENSE_FLAGS')
394    if license_flags:
395        acceptlist = d.getVar('LICENSE_FLAGS_ACCEPTED')
396        if not acceptlist:
397            return license_flags.split()
398        unmatched_flags = all_license_flags_match(license_flags, acceptlist)
399        if unmatched_flags:
400            return unmatched_flags
401    return None
402
403def check_license_format(d):
404    """
405    This function checks if LICENSE is well defined,
406        Validate operators in LICENSES.
407        No spaces are allowed between LICENSES.
408    """
409    pn = d.getVar('PN')
410    licenses = d.getVar('LICENSE')
411
412    elements = list(filter(lambda x: x.strip(), license_operator.split(licenses)))
413    for pos, element in enumerate(elements):
414        if license_pattern.match(element):
415            if pos > 0 and license_pattern.match(elements[pos - 1]):
416                oe.qa.handle_error('license-format',
417                        '%s: LICENSE value "%s" has an invalid format - license names ' \
418                        'must be separated by the following characters to indicate ' \
419                        'the license selection: %s' %
420                        (pn, licenses, license_operator_chars), d)
421        elif not license_operator.match(element):
422            oe.qa.handle_error('license-format',
423                    '%s: LICENSE value "%s" has an invalid separator "%s" that is not ' \
424                    'in the valid list of separators (%s)' %
425                    (pn, licenses, element, license_operator_chars), d)
426
427def skip_incompatible_package_licenses(d, pkgs):
428    if not pkgs:
429        return {}
430
431    pn = d.getVar("PN")
432
433    check_license = False if pn.startswith("nativesdk-") else True
434    for t in ["-native", "-cross-${TARGET_ARCH}", "-cross-initial-${TARGET_ARCH}",
435            "-crosssdk-${SDK_SYS}", "-crosssdk-initial-${SDK_SYS}",
436            "-cross-canadian-${TRANSLATED_TARGET_ARCH}"]:
437        if pn.endswith(d.expand(t)):
438            check_license = False
439    if pn.startswith("gcc-source-"):
440        check_license = False
441
442    bad_licenses = (d.getVar('INCOMPATIBLE_LICENSE') or "").split()
443    if not check_license or not bad_licenses:
444        return {}
445
446    bad_licenses = expand_wildcard_licenses(d, bad_licenses)
447
448    exceptions = (d.getVar("INCOMPATIBLE_LICENSE_EXCEPTIONS") or "").split()
449
450    for lic_exception in exceptions:
451        if ":" in lic_exception:
452            lic_exception = lic_exception.split(":")[1]
453        if lic_exception in obsolete_license_list():
454            bb.fatal("Obsolete license %s used in INCOMPATIBLE_LICENSE_EXCEPTIONS" % lic_exception)
455
456    skipped_pkgs = {}
457    for pkg in pkgs:
458        remaining_bad_licenses = apply_pkg_license_exception(pkg, bad_licenses, exceptions)
459
460        incompatible_lic = incompatible_license(d, remaining_bad_licenses, pkg)
461        if incompatible_lic:
462            skipped_pkgs[pkg] = incompatible_lic
463
464    return skipped_pkgs
465