1# 2# Copyright OpenEmbedded Contributors 3# 4# SPDX-License-Identifier: GPL-2.0-only 5# 6"""Code for parsing OpenEmbedded license strings""" 7 8import ast 9import re 10import oe.qa 11from fnmatch import fnmatchcase as fnmatch 12 13def license_ok(license, dont_want_licenses): 14 """ Return False if License exist in dont_want_licenses else True """ 15 for dwl in dont_want_licenses: 16 if fnmatch(license, dwl): 17 return False 18 return True 19 20def obsolete_license_list(): 21 return ["AGPL-3", "AGPL-3+", "AGPLv3", "AGPLv3+", "AGPLv3.0", "AGPLv3.0+", "AGPL-3.0", "AGPL-3.0+", "BSD-0-Clause", 22 "GPL-1", "GPL-1+", "GPLv1", "GPLv1+", "GPLv1.0", "GPLv1.0+", "GPL-1.0", "GPL-1.0+", "GPL-2", "GPL-2+", "GPLv2", 23 "GPLv2+", "GPLv2.0", "GPLv2.0+", "GPL-2.0", "GPL-2.0+", "GPL-3", "GPL-3+", "GPLv3", "GPLv3+", "GPLv3.0", "GPLv3.0+", 24 "GPL-3.0", "GPL-3.0+", "LGPLv2", "LGPLv2+", "LGPLv2.0", "LGPLv2.0+", "LGPL-2.0", "LGPL-2.0+", "LGPL2.1", "LGPL2.1+", 25 "LGPLv2.1", "LGPLv2.1+", "LGPL-2.1", "LGPL-2.1+", "LGPLv3", "LGPLv3+", "LGPL-3.0", "LGPL-3.0+", "MPL-1", "MPLv1", 26 "MPLv1.1", "MPLv2", "MIT-X", "MIT-style", "openssl", "PSF", "PSFv2", "Python-2", "Apachev2", "Apache-2", "Artisticv1", 27 "Artistic-1", "AFL-2", "AFL-1", "AFLv2", "AFLv1", "CDDLv1", "CDDL-1", "EPLv1.0", "FreeType", "Nauman", 28 "tcl", "vim", "SGIv1"] 29 30class LicenseError(Exception): 31 pass 32 33class LicenseSyntaxError(LicenseError): 34 def __init__(self, licensestr, exc): 35 self.licensestr = licensestr 36 self.exc = exc 37 LicenseError.__init__(self) 38 39 def __str__(self): 40 return "error in '%s': %s" % (self.licensestr, self.exc) 41 42class InvalidLicense(LicenseError): 43 def __init__(self, license): 44 self.license = license 45 LicenseError.__init__(self) 46 47 def __str__(self): 48 return "invalid characters in license '%s'" % self.license 49 50license_operator_chars = '&|() ' 51license_operator = re.compile(r'([' + license_operator_chars + '])') 52license_pattern = re.compile(r'[a-zA-Z0-9.+_\-]+$') 53 54class LicenseVisitor(ast.NodeVisitor): 55 """Get elements based on OpenEmbedded license strings""" 56 def get_elements(self, licensestr): 57 new_elements = [] 58 elements = list([x for x in license_operator.split(licensestr) if x.strip()]) 59 for pos, element in enumerate(elements): 60 if license_pattern.match(element): 61 if pos > 0 and license_pattern.match(elements[pos-1]): 62 new_elements.append('&') 63 element = '"' + element + '"' 64 elif not license_operator.match(element): 65 raise InvalidLicense(element) 66 new_elements.append(element) 67 68 return new_elements 69 70 """Syntax tree visitor which can accept elements previously generated with 71 OpenEmbedded license string""" 72 def visit_elements(self, elements): 73 self.visit(ast.parse(' '.join(elements))) 74 75 """Syntax tree visitor which can accept OpenEmbedded license strings""" 76 def visit_string(self, licensestr): 77 self.visit_elements(self.get_elements(licensestr)) 78 79class FlattenVisitor(LicenseVisitor): 80 """Flatten a license tree (parsed from a string) by selecting one of each 81 set of OR options, in the way the user specifies""" 82 def __init__(self, choose_licenses): 83 self.choose_licenses = choose_licenses 84 self.licenses = [] 85 LicenseVisitor.__init__(self) 86 87 def visit_Str(self, node): 88 self.licenses.append(node.s) 89 90 def visit_Constant(self, node): 91 self.licenses.append(node.value) 92 93 def visit_BinOp(self, node): 94 if isinstance(node.op, ast.BitOr): 95 left = FlattenVisitor(self.choose_licenses) 96 left.visit(node.left) 97 98 right = FlattenVisitor(self.choose_licenses) 99 right.visit(node.right) 100 101 selected = self.choose_licenses(left.licenses, right.licenses) 102 self.licenses.extend(selected) 103 else: 104 self.generic_visit(node) 105 106def flattened_licenses(licensestr, choose_licenses): 107 """Given a license string and choose_licenses function, return a flat list of licenses""" 108 flatten = FlattenVisitor(choose_licenses) 109 try: 110 flatten.visit_string(licensestr) 111 except SyntaxError as exc: 112 raise LicenseSyntaxError(licensestr, exc) 113 return flatten.licenses 114 115def is_included(licensestr, include_licenses=None, exclude_licenses=None): 116 """Given a license string, a list of licenses to include and a list of 117 licenses to exclude, determine if the license string matches the include 118 list and does not match the exclude list. 119 120 Returns a tuple holding the boolean state and a list of the applicable 121 licenses that were excluded if state is False, or the licenses that were 122 included if the state is True.""" 123 124 def include_license(license): 125 return any(fnmatch(license, pattern) for pattern in include_licenses) 126 127 def exclude_license(license): 128 return any(fnmatch(license, pattern) for pattern in exclude_licenses) 129 130 def choose_licenses(alpha, beta): 131 """Select the option in an OR which is the 'best' (has the most 132 included licenses and no excluded licenses).""" 133 # The factor 1000 below is arbitrary, just expected to be much larger 134 # than the number of licenses actually specified. That way the weight 135 # will be negative if the list of licenses contains an excluded license, 136 # but still gives a higher weight to the list with the most included 137 # licenses. 138 alpha_weight = (len(list(filter(include_license, alpha))) - 139 1000 * (len(list(filter(exclude_license, alpha))) > 0)) 140 beta_weight = (len(list(filter(include_license, beta))) - 141 1000 * (len(list(filter(exclude_license, beta))) > 0)) 142 if alpha_weight >= beta_weight: 143 return alpha 144 else: 145 return beta 146 147 if not include_licenses: 148 include_licenses = ['*'] 149 150 if not exclude_licenses: 151 exclude_licenses = [] 152 153 licenses = flattened_licenses(licensestr, choose_licenses) 154 excluded = [lic for lic in licenses if exclude_license(lic)] 155 included = [lic for lic in licenses if include_license(lic)] 156 if excluded: 157 return False, excluded 158 else: 159 return True, included 160 161class ManifestVisitor(LicenseVisitor): 162 """Walk license tree (parsed from a string) removing the incompatible 163 licenses specified""" 164 def __init__(self, dont_want_licenses, canonical_license, d): 165 self._dont_want_licenses = dont_want_licenses 166 self._canonical_license = canonical_license 167 self._d = d 168 self._operators = [] 169 170 self.licenses = [] 171 self.licensestr = '' 172 173 LicenseVisitor.__init__(self) 174 175 def visit(self, node): 176 if isinstance(node, ast.Str): 177 lic = node.s 178 179 if license_ok(self._canonical_license(self._d, lic), 180 self._dont_want_licenses) == True: 181 if self._operators: 182 ops = [] 183 for op in self._operators: 184 if op == '[': 185 ops.append(op) 186 elif op == ']': 187 ops.append(op) 188 else: 189 if not ops: 190 ops.append(op) 191 elif ops[-1] in ['[', ']']: 192 ops.append(op) 193 else: 194 ops[-1] = op 195 196 for op in ops: 197 if op == '[' or op == ']': 198 self.licensestr += op 199 elif self.licenses: 200 self.licensestr += ' ' + op + ' ' 201 202 self._operators = [] 203 204 self.licensestr += lic 205 self.licenses.append(lic) 206 elif isinstance(node, ast.BitAnd): 207 self._operators.append("&") 208 elif isinstance(node, ast.BitOr): 209 self._operators.append("|") 210 elif isinstance(node, ast.List): 211 self._operators.append("[") 212 elif isinstance(node, ast.Load): 213 self.licensestr += "]" 214 215 self.generic_visit(node) 216 217def manifest_licenses(licensestr, dont_want_licenses, canonical_license, d): 218 """Given a license string and dont_want_licenses list, 219 return license string filtered and a list of licenses""" 220 manifest = ManifestVisitor(dont_want_licenses, canonical_license, d) 221 222 try: 223 elements = manifest.get_elements(licensestr) 224 225 # Replace '()' to '[]' for handle in ast as List and Load types. 226 elements = ['[' if e == '(' else e for e in elements] 227 elements = [']' if e == ')' else e for e in elements] 228 229 manifest.visit_elements(elements) 230 except SyntaxError as exc: 231 raise LicenseSyntaxError(licensestr, exc) 232 233 # Replace '[]' to '()' for output correct license. 234 manifest.licensestr = manifest.licensestr.replace('[', '(').replace(']', ')') 235 236 return (manifest.licensestr, manifest.licenses) 237 238class ListVisitor(LicenseVisitor): 239 """Record all different licenses found in the license string""" 240 def __init__(self): 241 self.licenses = set() 242 243 def visit_Str(self, node): 244 self.licenses.add(node.s) 245 246 def visit_Constant(self, node): 247 self.licenses.add(node.value) 248 249def list_licenses(licensestr): 250 """Simply get a list of all licenses mentioned in a license string. 251 Binary operators are not applied or taken into account in any way""" 252 visitor = ListVisitor() 253 try: 254 visitor.visit_string(licensestr) 255 except SyntaxError as exc: 256 raise LicenseSyntaxError(licensestr, exc) 257 return visitor.licenses 258 259def apply_pkg_license_exception(pkg, bad_licenses, exceptions): 260 """Return remaining bad licenses after removing any package exceptions""" 261 262 return [lic for lic in bad_licenses if pkg + ':' + lic not in exceptions] 263 264def return_spdx(d, license): 265 """ 266 This function returns the spdx mapping of a license if it exists. 267 """ 268 return d.getVarFlag('SPDXLICENSEMAP', license) 269 270def canonical_license(d, license): 271 """ 272 Return the canonical (SPDX) form of the license if available (so GPLv3 273 becomes GPL-3.0-only) or the passed license if there is no canonical form. 274 """ 275 return d.getVarFlag('SPDXLICENSEMAP', license) or license 276 277def expand_wildcard_licenses(d, wildcard_licenses): 278 """ 279 There are some common wildcard values users may want to use. Support them 280 here. 281 """ 282 licenses = set(wildcard_licenses) 283 mapping = { 284 "AGPL-3.0*" : ["AGPL-3.0-only", "AGPL-3.0-or-later"], 285 "GPL-3.0*" : ["GPL-3.0-only", "GPL-3.0-or-later"], 286 "LGPL-3.0*" : ["LGPL-3.0-only", "LGPL-3.0-or-later"], 287 } 288 for k in mapping: 289 if k in wildcard_licenses: 290 licenses.remove(k) 291 for item in mapping[k]: 292 licenses.add(item) 293 294 for l in licenses: 295 if l in obsolete_license_list(): 296 bb.fatal("Error, %s is an obsolete license, please use an SPDX reference in INCOMPATIBLE_LICENSE" % l) 297 if "*" in l: 298 bb.fatal("Error, %s is an invalid license wildcard entry" % l) 299 300 return list(licenses) 301 302def incompatible_license_contains(license, truevalue, falsevalue, d): 303 license = canonical_license(d, license) 304 bad_licenses = (d.getVar('INCOMPATIBLE_LICENSE') or "").split() 305 bad_licenses = expand_wildcard_licenses(d, bad_licenses) 306 return truevalue if license in bad_licenses else falsevalue 307 308def incompatible_pkg_license(d, dont_want_licenses, license): 309 # Handles an "or" or two license sets provided by 310 # flattened_licenses(), pick one that works if possible. 311 def choose_lic_set(a, b): 312 return a if all(license_ok(canonical_license(d, lic), 313 dont_want_licenses) for lic in a) else b 314 315 try: 316 licenses = flattened_licenses(license, choose_lic_set) 317 except LicenseError as exc: 318 bb.fatal('%s: %s' % (d.getVar('P'), exc)) 319 320 incompatible_lic = [] 321 for l in licenses: 322 license = canonical_license(d, l) 323 if not license_ok(license, dont_want_licenses): 324 incompatible_lic.append(license) 325 326 return sorted(incompatible_lic) 327 328def incompatible_license(d, dont_want_licenses, package=None): 329 """ 330 This function checks if a recipe has only incompatible licenses. It also 331 take into consideration 'or' operand. dont_want_licenses should be passed 332 as canonical (SPDX) names. 333 """ 334 license = d.getVar("LICENSE:%s" % package) if package else None 335 if not license: 336 license = d.getVar('LICENSE') 337 338 return incompatible_pkg_license(d, dont_want_licenses, license) 339 340def check_license_flags(d): 341 """ 342 This function checks if a recipe has any LICENSE_FLAGS that 343 aren't acceptable. 344 345 If it does, it returns the all LICENSE_FLAGS missing from the list 346 of acceptable license flags, or all of the LICENSE_FLAGS if there 347 is no list of acceptable flags. 348 349 If everything is is acceptable, it returns None. 350 """ 351 352 def license_flag_matches(flag, acceptlist, pn): 353 """ 354 Return True if flag matches something in acceptlist, None if not. 355 356 Before we test a flag against the acceptlist, we append _${PN} 357 to it. We then try to match that string against the 358 acceptlist. This covers the normal case, where we expect 359 LICENSE_FLAGS to be a simple string like 'commercial', which 360 the user typically matches exactly in the acceptlist by 361 explicitly appending the package name e.g 'commercial_foo'. 362 If we fail the match however, we then split the flag across 363 '_' and append each fragment and test until we either match or 364 run out of fragments. 365 """ 366 flag_pn = ("%s_%s" % (flag, pn)) 367 for candidate in acceptlist: 368 if flag_pn == candidate: 369 return True 370 371 flag_cur = "" 372 flagments = flag_pn.split("_") 373 flagments.pop() # we've already tested the full string 374 for flagment in flagments: 375 if flag_cur: 376 flag_cur += "_" 377 flag_cur += flagment 378 for candidate in acceptlist: 379 if flag_cur == candidate: 380 return True 381 return False 382 383 def all_license_flags_match(license_flags, acceptlist): 384 """ Return all unmatched flags, None if all flags match """ 385 pn = d.getVar('PN') 386 split_acceptlist = acceptlist.split() 387 flags = [] 388 for flag in license_flags.split(): 389 if not license_flag_matches(flag, split_acceptlist, pn): 390 flags.append(flag) 391 return flags if flags else None 392 393 license_flags = d.getVar('LICENSE_FLAGS') 394 if license_flags: 395 acceptlist = d.getVar('LICENSE_FLAGS_ACCEPTED') 396 if not acceptlist: 397 return license_flags.split() 398 unmatched_flags = all_license_flags_match(license_flags, acceptlist) 399 if unmatched_flags: 400 return unmatched_flags 401 return None 402 403def check_license_format(d): 404 """ 405 This function checks if LICENSE is well defined, 406 Validate operators in LICENSES. 407 No spaces are allowed between LICENSES. 408 """ 409 pn = d.getVar('PN') 410 licenses = d.getVar('LICENSE') 411 412 elements = list(filter(lambda x: x.strip(), license_operator.split(licenses))) 413 for pos, element in enumerate(elements): 414 if license_pattern.match(element): 415 if pos > 0 and license_pattern.match(elements[pos - 1]): 416 oe.qa.handle_error('license-format', 417 '%s: LICENSE value "%s" has an invalid format - license names ' \ 418 'must be separated by the following characters to indicate ' \ 419 'the license selection: %s' % 420 (pn, licenses, license_operator_chars), d) 421 elif not license_operator.match(element): 422 oe.qa.handle_error('license-format', 423 '%s: LICENSE value "%s" has an invalid separator "%s" that is not ' \ 424 'in the valid list of separators (%s)' % 425 (pn, licenses, element, license_operator_chars), d) 426 427def skip_incompatible_package_licenses(d, pkgs): 428 if not pkgs: 429 return {} 430 431 pn = d.getVar("PN") 432 433 check_license = False if pn.startswith("nativesdk-") else True 434 for t in ["-native", "-cross-${TARGET_ARCH}", "-cross-initial-${TARGET_ARCH}", 435 "-crosssdk-${SDK_SYS}", "-crosssdk-initial-${SDK_SYS}", 436 "-cross-canadian-${TRANSLATED_TARGET_ARCH}"]: 437 if pn.endswith(d.expand(t)): 438 check_license = False 439 if pn.startswith("gcc-source-"): 440 check_license = False 441 442 bad_licenses = (d.getVar('INCOMPATIBLE_LICENSE') or "").split() 443 if not check_license or not bad_licenses: 444 return {} 445 446 bad_licenses = expand_wildcard_licenses(d, bad_licenses) 447 448 exceptions = (d.getVar("INCOMPATIBLE_LICENSE_EXCEPTIONS") or "").split() 449 450 for lic_exception in exceptions: 451 if ":" in lic_exception: 452 lic_exception = lic_exception.split(":")[1] 453 if lic_exception in obsolete_license_list(): 454 bb.fatal("Obsolete license %s used in INCOMPATIBLE_LICENSE_EXCEPTIONS" % lic_exception) 455 456 skipped_pkgs = {} 457 for pkg in pkgs: 458 remaining_bad_licenses = apply_pkg_license_exception(pkg, bad_licenses, exceptions) 459 460 incompatible_lic = incompatible_license(d, remaining_bad_licenses, pkg) 461 if incompatible_lic: 462 skipped_pkgs[pkg] = incompatible_lic 463 464 return skipped_pkgs 465