1# 2# Copyright OpenEmbedded Contributors 3# 4# SPDX-License-Identifier: GPL-2.0-only 5# 6 7inherit spdx-common 8 9SPDX_VERSION = "2.2" 10 11SPDX_ORG ??= "OpenEmbedded ()" 12SPDX_SUPPLIER ??= "Organization: ${SPDX_ORG}" 13SPDX_SUPPLIER[doc] = "The SPDX PackageSupplier field for SPDX packages created from \ 14 this recipe. For SPDX documents create using this class during the build, this \ 15 is the contact information for the person or organization who is doing the \ 16 build." 17 18SPDX_ARCHIVE_SOURCES ??= "0" 19SPDX_ARCHIVE_PACKAGED ??= "0" 20 21def get_namespace(d, name): 22 import uuid 23 namespace_uuid = uuid.uuid5(uuid.NAMESPACE_DNS, d.getVar("SPDX_UUID_NAMESPACE")) 24 return "%s/%s-%s" % (d.getVar("SPDX_NAMESPACE_PREFIX"), name, str(uuid.uuid5(namespace_uuid, name))) 25 26 27def create_annotation(d, comment): 28 from datetime import datetime, timezone 29 30 creation_time = datetime.now(tz=timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ") 31 annotation = oe.spdx.SPDXAnnotation() 32 annotation.annotationDate = creation_time 33 annotation.annotationType = "OTHER" 34 annotation.annotator = "Tool: %s - %s" % (d.getVar("SPDX_TOOL_NAME"), d.getVar("SPDX_TOOL_VERSION")) 35 annotation.comment = comment 36 return annotation 37 38def recipe_spdx_is_native(d, recipe): 39 return any(a.annotationType == "OTHER" and 40 a.annotator == "Tool: %s - %s" % (d.getVar("SPDX_TOOL_NAME"), d.getVar("SPDX_TOOL_VERSION")) and 41 a.comment == "isNative" for a in recipe.annotations) 42 43def get_json_indent(d): 44 if d.getVar("SPDX_PRETTY") == "1": 45 return 2 46 return None 47 48 49def convert_license_to_spdx(lic, license_data, document, d, existing={}): 50 from pathlib import Path 51 import oe.spdx 52 53 extracted = {} 54 55 def add_extracted_license(ident, name): 56 nonlocal document 57 58 if name in extracted: 59 return 60 61 extracted_info = oe.spdx.SPDXExtractedLicensingInfo() 62 extracted_info.name = name 63 extracted_info.licenseId = ident 64 extracted_info.extractedText = None 65 66 if name == "PD": 67 # Special-case this. 68 extracted_info.extractedText = "Software released to the public domain" 69 else: 70 # Seach for the license in COMMON_LICENSE_DIR and LICENSE_PATH 71 for directory in [d.getVar('COMMON_LICENSE_DIR')] + (d.getVar('LICENSE_PATH') or '').split(): 72 try: 73 with (Path(directory) / name).open(errors="replace") as f: 74 extracted_info.extractedText = f.read() 75 break 76 except FileNotFoundError: 77 pass 78 if extracted_info.extractedText is None: 79 # If it's not SPDX or PD, then NO_GENERIC_LICENSE must be set 80 entry = d.getVarFlag('NO_GENERIC_LICENSE', name).split(';') 81 filename = entry[0] 82 params = {i.split('=')[0]: i.split('=')[1] for i in entry[1:] if '=' in i} 83 beginline = int(params.get('beginline', 1)) 84 endline = params.get('endline', None) 85 if endline: 86 endline = int(endline) 87 if filename: 88 filename = d.expand("${S}/" + filename) 89 with open(filename, errors="replace") as f: 90 extracted_info.extractedText = "".join(line for idx, line in enumerate(f, 1) if beginline <= idx and idx <= (endline or idx)) 91 else: 92 bb.fatal("Cannot find any text for license %s" % name) 93 94 extracted[name] = extracted_info 95 document.hasExtractedLicensingInfos.append(extracted_info) 96 97 def convert(l): 98 if l == "(" or l == ")": 99 return l 100 101 if l == "&": 102 return "AND" 103 104 if l == "|": 105 return "OR" 106 107 if l == "CLOSED": 108 return "NONE" 109 110 spdx_license = d.getVarFlag("SPDXLICENSEMAP", l) or l 111 if spdx_license in license_data["licenses"]: 112 return spdx_license 113 114 try: 115 spdx_license = existing[l] 116 except KeyError: 117 spdx_license = "LicenseRef-" + l 118 add_extracted_license(spdx_license, l) 119 120 return spdx_license 121 122 lic_split = lic.replace("(", " ( ").replace(")", " ) ").replace("|", " | ").replace("&", " & ").split() 123 124 return ' '.join(convert(l) for l in lic_split) 125 126def add_package_files(d, doc, spdx_pkg, topdir, get_spdxid, get_types, *, archive=None, ignore_dirs=[], ignore_top_level_dirs=[]): 127 from pathlib import Path 128 import oe.spdx 129 import oe.spdx_common 130 import hashlib 131 132 source_date_epoch = d.getVar("SOURCE_DATE_EPOCH") 133 if source_date_epoch: 134 source_date_epoch = int(source_date_epoch) 135 136 sha1s = [] 137 spdx_files = [] 138 139 file_counter = 1 140 for subdir, dirs, files in os.walk(topdir): 141 dirs[:] = [d for d in dirs if d not in ignore_dirs] 142 if subdir == str(topdir): 143 dirs[:] = [d for d in dirs if d not in ignore_top_level_dirs] 144 145 for file in files: 146 filepath = Path(subdir) / file 147 filename = str(filepath.relative_to(topdir)) 148 149 if not filepath.is_symlink() and filepath.is_file(): 150 spdx_file = oe.spdx.SPDXFile() 151 spdx_file.SPDXID = get_spdxid(file_counter) 152 for t in get_types(filepath): 153 spdx_file.fileTypes.append(t) 154 spdx_file.fileName = filename 155 156 if archive is not None: 157 with filepath.open("rb") as f: 158 info = archive.gettarinfo(fileobj=f) 159 info.name = filename 160 info.uid = 0 161 info.gid = 0 162 info.uname = "root" 163 info.gname = "root" 164 165 if source_date_epoch is not None and info.mtime > source_date_epoch: 166 info.mtime = source_date_epoch 167 168 archive.addfile(info, f) 169 170 sha1 = bb.utils.sha1_file(filepath) 171 sha1s.append(sha1) 172 spdx_file.checksums.append(oe.spdx.SPDXChecksum( 173 algorithm="SHA1", 174 checksumValue=sha1, 175 )) 176 spdx_file.checksums.append(oe.spdx.SPDXChecksum( 177 algorithm="SHA256", 178 checksumValue=bb.utils.sha256_file(filepath), 179 )) 180 181 if "SOURCE" in spdx_file.fileTypes: 182 extracted_lics = oe.spdx_common.extract_licenses(filepath) 183 if extracted_lics: 184 spdx_file.licenseInfoInFiles = extracted_lics 185 186 doc.files.append(spdx_file) 187 doc.add_relationship(spdx_pkg, "CONTAINS", spdx_file) 188 spdx_pkg.hasFiles.append(spdx_file.SPDXID) 189 190 spdx_files.append(spdx_file) 191 192 file_counter += 1 193 194 sha1s.sort() 195 verifier = hashlib.sha1() 196 for v in sha1s: 197 verifier.update(v.encode("utf-8")) 198 spdx_pkg.packageVerificationCode.packageVerificationCodeValue = verifier.hexdigest() 199 200 return spdx_files 201 202 203def add_package_sources_from_debug(d, package_doc, spdx_package, package, package_files, sources): 204 from pathlib import Path 205 import hashlib 206 import oe.packagedata 207 import oe.spdx 208 209 debug_search_paths = [ 210 Path(d.getVar('PKGD')), 211 Path(d.getVar('STAGING_DIR_TARGET')), 212 Path(d.getVar('STAGING_DIR_NATIVE')), 213 Path(d.getVar('STAGING_KERNEL_DIR')), 214 ] 215 216 pkg_data = oe.packagedata.read_subpkgdata_extended(package, d) 217 218 if pkg_data is None: 219 return 220 221 for file_path, file_data in pkg_data["files_info"].items(): 222 if not "debugsrc" in file_data: 223 continue 224 225 for pkg_file in package_files: 226 if file_path.lstrip("/") == pkg_file.fileName.lstrip("/"): 227 break 228 else: 229 bb.fatal("No package file found for %s in %s; SPDX found: %s" % (str(file_path), package, 230 " ".join(p.fileName for p in package_files))) 231 continue 232 233 for debugsrc in file_data["debugsrc"]: 234 ref_id = "NOASSERTION" 235 for search in debug_search_paths: 236 if debugsrc.startswith("/usr/src/kernel"): 237 debugsrc_path = search / debugsrc.replace('/usr/src/kernel/', '') 238 else: 239 debugsrc_path = search / debugsrc.lstrip("/") 240 # We can only hash files below, skip directories, links, etc. 241 if not os.path.isfile(debugsrc_path): 242 continue 243 244 file_sha256 = bb.utils.sha256_file(debugsrc_path) 245 246 if file_sha256 in sources: 247 source_file = sources[file_sha256] 248 249 doc_ref = package_doc.find_external_document_ref(source_file.doc.documentNamespace) 250 if doc_ref is None: 251 doc_ref = oe.spdx.SPDXExternalDocumentRef() 252 doc_ref.externalDocumentId = "DocumentRef-dependency-" + source_file.doc.name 253 doc_ref.spdxDocument = source_file.doc.documentNamespace 254 doc_ref.checksum.algorithm = "SHA1" 255 doc_ref.checksum.checksumValue = source_file.doc_sha1 256 package_doc.externalDocumentRefs.append(doc_ref) 257 258 ref_id = "%s:%s" % (doc_ref.externalDocumentId, source_file.file.SPDXID) 259 else: 260 bb.debug(1, "Debug source %s with SHA256 %s not found in any dependency" % (str(debugsrc_path), file_sha256)) 261 break 262 else: 263 bb.debug(1, "Debug source %s not found" % debugsrc) 264 265 package_doc.add_relationship(pkg_file, "GENERATED_FROM", ref_id, comment=debugsrc) 266 267add_package_sources_from_debug[vardepsexclude] += "STAGING_KERNEL_DIR" 268 269def collect_dep_recipes(d, doc, spdx_recipe): 270 import json 271 from pathlib import Path 272 import oe.sbom 273 import oe.spdx 274 import oe.spdx_common 275 276 deploy_dir_spdx = Path(d.getVar("DEPLOY_DIR_SPDX")) 277 package_archs = d.getVar("SPDX_MULTILIB_SSTATE_ARCHS").split() 278 package_archs.reverse() 279 280 dep_recipes = [] 281 282 deps = oe.spdx_common.get_spdx_deps(d) 283 284 for dep in deps: 285 # If this dependency is not calculated in the taskhash skip it. 286 # Otherwise, it can result in broken links since this task won't 287 # rebuild and see the new SPDX ID if the dependency changes 288 if not dep.in_taskhash: 289 continue 290 291 dep_recipe_path = oe.sbom.doc_find_by_hashfn(deploy_dir_spdx, package_archs, "recipe-" + dep.pn, dep.hashfn) 292 if not dep_recipe_path: 293 bb.fatal("Cannot find any SPDX file for recipe %s, %s" % (dep.pn, dep.hashfn)) 294 295 spdx_dep_doc, spdx_dep_sha1 = oe.sbom.read_doc(dep_recipe_path) 296 297 for pkg in spdx_dep_doc.packages: 298 if pkg.name == dep.pn: 299 spdx_dep_recipe = pkg 300 break 301 else: 302 continue 303 304 dep_recipes.append(oe.sbom.DepRecipe(spdx_dep_doc, spdx_dep_sha1, spdx_dep_recipe)) 305 306 dep_recipe_ref = oe.spdx.SPDXExternalDocumentRef() 307 dep_recipe_ref.externalDocumentId = "DocumentRef-dependency-" + spdx_dep_doc.name 308 dep_recipe_ref.spdxDocument = spdx_dep_doc.documentNamespace 309 dep_recipe_ref.checksum.algorithm = "SHA1" 310 dep_recipe_ref.checksum.checksumValue = spdx_dep_sha1 311 312 doc.externalDocumentRefs.append(dep_recipe_ref) 313 314 doc.add_relationship( 315 "%s:%s" % (dep_recipe_ref.externalDocumentId, spdx_dep_recipe.SPDXID), 316 "BUILD_DEPENDENCY_OF", 317 spdx_recipe 318 ) 319 320 return dep_recipes 321 322collect_dep_recipes[vardepsexclude] = "SPDX_MULTILIB_SSTATE_ARCHS" 323 324def collect_dep_sources(d, dep_recipes): 325 import oe.sbom 326 327 sources = {} 328 for dep in dep_recipes: 329 # Don't collect sources from native recipes as they 330 # match non-native sources also. 331 if recipe_spdx_is_native(d, dep.recipe): 332 continue 333 recipe_files = set(dep.recipe.hasFiles) 334 335 for spdx_file in dep.doc.files: 336 if spdx_file.SPDXID not in recipe_files: 337 continue 338 339 if "SOURCE" in spdx_file.fileTypes: 340 for checksum in spdx_file.checksums: 341 if checksum.algorithm == "SHA256": 342 sources[checksum.checksumValue] = oe.sbom.DepSource(dep.doc, dep.doc_sha1, dep.recipe, spdx_file) 343 break 344 345 return sources 346 347def add_download_packages(d, doc, recipe): 348 import os.path 349 from bb.fetch2 import decodeurl, CHECKSUM_LIST 350 import bb.process 351 import oe.spdx 352 import oe.sbom 353 354 for download_idx, src_uri in enumerate(d.getVar('SRC_URI').split()): 355 f = bb.fetch2.FetchData(src_uri, d) 356 357 package = oe.spdx.SPDXPackage() 358 package.name = "%s-source-%d" % (d.getVar("PN"), download_idx + 1) 359 package.SPDXID = oe.sbom.get_download_spdxid(d, download_idx + 1) 360 361 if f.type == "file": 362 continue 363 364 if f.method.supports_checksum(f): 365 for checksum_id in CHECKSUM_LIST: 366 if checksum_id.upper() not in oe.spdx.SPDXPackage.ALLOWED_CHECKSUMS: 367 continue 368 369 expected_checksum = getattr(f, "%s_expected" % checksum_id) 370 if expected_checksum is None: 371 continue 372 373 c = oe.spdx.SPDXChecksum() 374 c.algorithm = checksum_id.upper() 375 c.checksumValue = expected_checksum 376 package.checksums.append(c) 377 378 package.downloadLocation = oe.spdx_common.fetch_data_to_uri(f, f.name) 379 doc.packages.append(package) 380 doc.add_relationship(doc, "DESCRIBES", package) 381 # In the future, we might be able to do more fancy dependencies, 382 # but this should be sufficient for now 383 doc.add_relationship(package, "BUILD_DEPENDENCY_OF", recipe) 384 385def get_license_list_version(license_data, d): 386 # Newer versions of the SPDX license list are SemVer ("MAJOR.MINOR.MICRO"), 387 # but SPDX 2 only uses "MAJOR.MINOR". 388 return ".".join(license_data["licenseListVersion"].split(".")[:2]) 389 390 391python do_create_spdx() { 392 from datetime import datetime, timezone 393 import oe.sbom 394 import oe.spdx 395 import oe.spdx_common 396 import uuid 397 from pathlib import Path 398 from contextlib import contextmanager 399 import oe.cve_check 400 401 license_data = oe.spdx_common.load_spdx_license_data(d) 402 403 @contextmanager 404 def optional_tarfile(name, guard, mode="w"): 405 import tarfile 406 import bb.compress.zstd 407 408 num_threads = int(d.getVar("BB_NUMBER_THREADS")) 409 410 if guard: 411 name.parent.mkdir(parents=True, exist_ok=True) 412 with bb.compress.zstd.open(name, mode=mode + "b", num_threads=num_threads) as f: 413 with tarfile.open(fileobj=f, mode=mode + "|") as tf: 414 yield tf 415 else: 416 yield None 417 418 419 deploy_dir_spdx = Path(d.getVar("DEPLOY_DIR_SPDX")) 420 spdx_workdir = Path(d.getVar("SPDXWORK")) 421 include_sources = d.getVar("SPDX_INCLUDE_SOURCES") == "1" 422 archive_sources = d.getVar("SPDX_ARCHIVE_SOURCES") == "1" 423 archive_packaged = d.getVar("SPDX_ARCHIVE_PACKAGED") == "1" 424 pkg_arch = d.getVar("SSTATE_PKGARCH") 425 426 creation_time = datetime.now(tz=timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ") 427 428 doc = oe.spdx.SPDXDocument() 429 430 doc.name = "recipe-" + d.getVar("PN") 431 doc.documentNamespace = get_namespace(d, doc.name) 432 doc.creationInfo.created = creation_time 433 doc.creationInfo.comment = "This document was created by analyzing recipe files during the build." 434 doc.creationInfo.licenseListVersion = get_license_list_version(license_data, d) 435 doc.creationInfo.creators.append("Tool: OpenEmbedded Core create-spdx.bbclass") 436 doc.creationInfo.creators.append("Organization: %s" % d.getVar("SPDX_ORG")) 437 doc.creationInfo.creators.append("Person: N/A ()") 438 439 recipe = oe.spdx.SPDXPackage() 440 recipe.name = d.getVar("PN") 441 recipe.versionInfo = d.getVar("PV") 442 recipe.SPDXID = oe.sbom.get_recipe_spdxid(d) 443 recipe.supplier = d.getVar("SPDX_SUPPLIER") 444 if bb.data.inherits_class("native", d) or bb.data.inherits_class("cross", d): 445 recipe.annotations.append(create_annotation(d, "isNative")) 446 447 homepage = d.getVar("HOMEPAGE") 448 if homepage: 449 recipe.homepage = homepage 450 451 license = d.getVar("LICENSE") 452 if license: 453 recipe.licenseDeclared = convert_license_to_spdx(license, license_data, doc, d) 454 455 summary = d.getVar("SUMMARY") 456 if summary: 457 recipe.summary = summary 458 459 description = d.getVar("DESCRIPTION") 460 if description: 461 recipe.description = description 462 463 if d.getVar("SPDX_CUSTOM_ANNOTATION_VARS"): 464 for var in d.getVar('SPDX_CUSTOM_ANNOTATION_VARS').split(): 465 recipe.annotations.append(create_annotation(d, var + "=" + d.getVar(var))) 466 467 # Some CVEs may be patched during the build process without incrementing the version number, 468 # so querying for CVEs based on the CPE id can lead to false positives. To account for this, 469 # save the CVEs fixed by patches to source information field in the SPDX. 470 patched_cves = oe.cve_check.get_patched_cves(d) 471 patched_cves = list(patched_cves) 472 patched_cves = ' '.join(patched_cves) 473 if patched_cves: 474 recipe.sourceInfo = "CVEs fixed: " + patched_cves 475 476 cpe_ids = oe.cve_check.get_cpe_ids(d.getVar("CVE_PRODUCT"), d.getVar("CVE_VERSION")) 477 if cpe_ids: 478 for cpe_id in cpe_ids: 479 cpe = oe.spdx.SPDXExternalReference() 480 cpe.referenceCategory = "SECURITY" 481 cpe.referenceType = "http://spdx.org/rdf/references/cpe23Type" 482 cpe.referenceLocator = cpe_id 483 recipe.externalRefs.append(cpe) 484 485 doc.packages.append(recipe) 486 doc.add_relationship(doc, "DESCRIBES", recipe) 487 488 add_download_packages(d, doc, recipe) 489 490 if oe.spdx_common.process_sources(d) and include_sources: 491 recipe_archive = deploy_dir_spdx / "recipes" / (doc.name + ".tar.zst") 492 with optional_tarfile(recipe_archive, archive_sources) as archive: 493 oe.spdx_common.get_patched_src(d) 494 495 add_package_files( 496 d, 497 doc, 498 recipe, 499 spdx_workdir, 500 lambda file_counter: "SPDXRef-SourceFile-%s-%d" % (d.getVar("PN"), file_counter), 501 lambda filepath: ["SOURCE"], 502 ignore_dirs=[".git"], 503 ignore_top_level_dirs=["temp"], 504 archive=archive, 505 ) 506 507 if archive is not None: 508 recipe.packageFileName = str(recipe_archive.name) 509 510 dep_recipes = collect_dep_recipes(d, doc, recipe) 511 512 doc_sha1 = oe.sbom.write_doc(d, doc, pkg_arch, "recipes", indent=get_json_indent(d)) 513 dep_recipes.append(oe.sbom.DepRecipe(doc, doc_sha1, recipe)) 514 515 recipe_ref = oe.spdx.SPDXExternalDocumentRef() 516 recipe_ref.externalDocumentId = "DocumentRef-recipe-" + recipe.name 517 recipe_ref.spdxDocument = doc.documentNamespace 518 recipe_ref.checksum.algorithm = "SHA1" 519 recipe_ref.checksum.checksumValue = doc_sha1 520 521 sources = collect_dep_sources(d, dep_recipes) 522 found_licenses = {license.name:recipe_ref.externalDocumentId + ":" + license.licenseId for license in doc.hasExtractedLicensingInfos} 523 524 if not recipe_spdx_is_native(d, recipe): 525 bb.build.exec_func("read_subpackage_metadata", d) 526 527 pkgdest = Path(d.getVar("PKGDEST")) 528 for package in d.getVar("PACKAGES").split(): 529 if not oe.packagedata.packaged(package, d): 530 continue 531 532 package_doc = oe.spdx.SPDXDocument() 533 pkg_name = d.getVar("PKG:%s" % package) or package 534 package_doc.name = pkg_name 535 package_doc.documentNamespace = get_namespace(d, package_doc.name) 536 package_doc.creationInfo.created = creation_time 537 package_doc.creationInfo.comment = "This document was created by analyzing packages created during the build." 538 package_doc.creationInfo.licenseListVersion = get_license_list_version(license_data, d) 539 package_doc.creationInfo.creators.append("Tool: OpenEmbedded Core create-spdx.bbclass") 540 package_doc.creationInfo.creators.append("Organization: %s" % d.getVar("SPDX_ORG")) 541 package_doc.creationInfo.creators.append("Person: N/A ()") 542 package_doc.externalDocumentRefs.append(recipe_ref) 543 544 package_license = d.getVar("LICENSE:%s" % package) or d.getVar("LICENSE") 545 546 spdx_package = oe.spdx.SPDXPackage() 547 548 spdx_package.SPDXID = oe.sbom.get_package_spdxid(pkg_name) 549 spdx_package.name = pkg_name 550 spdx_package.versionInfo = d.getVar("PV") 551 spdx_package.licenseDeclared = convert_license_to_spdx(package_license, license_data, package_doc, d, found_licenses) 552 spdx_package.supplier = d.getVar("SPDX_SUPPLIER") 553 554 package_doc.packages.append(spdx_package) 555 556 package_doc.add_relationship(spdx_package, "GENERATED_FROM", "%s:%s" % (recipe_ref.externalDocumentId, recipe.SPDXID)) 557 package_doc.add_relationship(package_doc, "DESCRIBES", spdx_package) 558 559 package_archive = deploy_dir_spdx / "packages" / (package_doc.name + ".tar.zst") 560 with optional_tarfile(package_archive, archive_packaged) as archive: 561 package_files = add_package_files( 562 d, 563 package_doc, 564 spdx_package, 565 pkgdest / package, 566 lambda file_counter: oe.sbom.get_packaged_file_spdxid(pkg_name, file_counter), 567 lambda filepath: ["BINARY"], 568 ignore_top_level_dirs=['CONTROL', 'DEBIAN'], 569 archive=archive, 570 ) 571 572 if archive is not None: 573 spdx_package.packageFileName = str(package_archive.name) 574 575 add_package_sources_from_debug(d, package_doc, spdx_package, package, package_files, sources) 576 577 oe.sbom.write_doc(d, package_doc, pkg_arch, "packages", indent=get_json_indent(d)) 578} 579do_create_spdx[vardepsexclude] += "BB_NUMBER_THREADS" 580# NOTE: depending on do_unpack is a hack that is necessary to get it's dependencies for archive the source 581addtask do_create_spdx after do_package do_packagedata do_unpack do_collect_spdx_deps before do_populate_sdk do_build do_rm_work 582 583SSTATETASKS += "do_create_spdx" 584do_create_spdx[sstate-inputdirs] = "${SPDXDEPLOY}" 585do_create_spdx[sstate-outputdirs] = "${DEPLOY_DIR_SPDX}" 586 587python do_create_spdx_setscene () { 588 sstate_setscene(d) 589} 590addtask do_create_spdx_setscene 591 592do_create_spdx[dirs] = "${SPDXWORK}" 593do_create_spdx[cleandirs] = "${SPDXDEPLOY} ${SPDXWORK}" 594do_create_spdx[depends] += " \ 595 ${PATCHDEPENDENCY} \ 596 ${@create_spdx_source_deps(d)} \ 597" 598 599python do_create_runtime_spdx() { 600 from datetime import datetime, timezone 601 import oe.sbom 602 import oe.spdx 603 import oe.spdx_common 604 import oe.packagedata 605 from pathlib import Path 606 607 deploy_dir_spdx = Path(d.getVar("DEPLOY_DIR_SPDX")) 608 spdx_deploy = Path(d.getVar("SPDXRUNTIMEDEPLOY")) 609 is_native = bb.data.inherits_class("native", d) or bb.data.inherits_class("cross", d) 610 611 creation_time = datetime.now(tz=timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ") 612 613 license_data = oe.spdx_common.load_spdx_license_data(d) 614 615 providers = oe.spdx_common.collect_package_providers(d) 616 pkg_arch = d.getVar("SSTATE_PKGARCH") 617 package_archs = d.getVar("SPDX_MULTILIB_SSTATE_ARCHS").split() 618 package_archs.reverse() 619 620 if not is_native: 621 bb.build.exec_func("read_subpackage_metadata", d) 622 623 dep_package_cache = {} 624 625 pkgdest = Path(d.getVar("PKGDEST")) 626 for package in d.getVar("PACKAGES").split(): 627 localdata = bb.data.createCopy(d) 628 pkg_name = d.getVar("PKG:%s" % package) or package 629 localdata.setVar("PKG", pkg_name) 630 localdata.setVar('OVERRIDES', d.getVar("OVERRIDES", False) + ":" + package) 631 632 if not oe.packagedata.packaged(package, localdata): 633 continue 634 635 pkg_spdx_path = oe.sbom.doc_path(deploy_dir_spdx, pkg_name, pkg_arch, "packages") 636 637 package_doc, package_doc_sha1 = oe.sbom.read_doc(pkg_spdx_path) 638 639 for p in package_doc.packages: 640 if p.name == pkg_name: 641 spdx_package = p 642 break 643 else: 644 bb.fatal("Package '%s' not found in %s" % (pkg_name, pkg_spdx_path)) 645 646 runtime_doc = oe.spdx.SPDXDocument() 647 runtime_doc.name = "runtime-" + pkg_name 648 runtime_doc.documentNamespace = get_namespace(localdata, runtime_doc.name) 649 runtime_doc.creationInfo.created = creation_time 650 runtime_doc.creationInfo.comment = "This document was created by analyzing package runtime dependencies." 651 runtime_doc.creationInfo.licenseListVersion = get_license_list_version(license_data, d) 652 runtime_doc.creationInfo.creators.append("Tool: OpenEmbedded Core create-spdx.bbclass") 653 runtime_doc.creationInfo.creators.append("Organization: %s" % d.getVar("SPDX_ORG")) 654 runtime_doc.creationInfo.creators.append("Person: N/A ()") 655 656 package_ref = oe.spdx.SPDXExternalDocumentRef() 657 package_ref.externalDocumentId = "DocumentRef-package-" + package 658 package_ref.spdxDocument = package_doc.documentNamespace 659 package_ref.checksum.algorithm = "SHA1" 660 package_ref.checksum.checksumValue = package_doc_sha1 661 662 runtime_doc.externalDocumentRefs.append(package_ref) 663 664 runtime_doc.add_relationship( 665 runtime_doc.SPDXID, 666 "AMENDS", 667 "%s:%s" % (package_ref.externalDocumentId, package_doc.SPDXID) 668 ) 669 670 deps = bb.utils.explode_dep_versions2(localdata.getVar("RDEPENDS") or "") 671 seen_deps = set() 672 for dep, _ in deps.items(): 673 if dep in seen_deps: 674 continue 675 676 if dep not in providers: 677 continue 678 679 (dep, dep_hashfn) = providers[dep] 680 681 if not oe.packagedata.packaged(dep, localdata): 682 continue 683 684 dep_pkg_data = oe.packagedata.read_subpkgdata_dict(dep, d) 685 dep_pkg = dep_pkg_data["PKG"] 686 687 if dep in dep_package_cache: 688 (dep_spdx_package, dep_package_ref) = dep_package_cache[dep] 689 else: 690 dep_path = oe.sbom.doc_find_by_hashfn(deploy_dir_spdx, package_archs, dep_pkg, dep_hashfn) 691 if not dep_path: 692 bb.fatal("No SPDX file found for package %s, %s" % (dep_pkg, dep_hashfn)) 693 694 spdx_dep_doc, spdx_dep_sha1 = oe.sbom.read_doc(dep_path) 695 696 for pkg in spdx_dep_doc.packages: 697 if pkg.name == dep_pkg: 698 dep_spdx_package = pkg 699 break 700 else: 701 bb.fatal("Package '%s' not found in %s" % (dep_pkg, dep_path)) 702 703 dep_package_ref = oe.spdx.SPDXExternalDocumentRef() 704 dep_package_ref.externalDocumentId = "DocumentRef-runtime-dependency-" + spdx_dep_doc.name 705 dep_package_ref.spdxDocument = spdx_dep_doc.documentNamespace 706 dep_package_ref.checksum.algorithm = "SHA1" 707 dep_package_ref.checksum.checksumValue = spdx_dep_sha1 708 709 dep_package_cache[dep] = (dep_spdx_package, dep_package_ref) 710 711 runtime_doc.externalDocumentRefs.append(dep_package_ref) 712 713 runtime_doc.add_relationship( 714 "%s:%s" % (dep_package_ref.externalDocumentId, dep_spdx_package.SPDXID), 715 "RUNTIME_DEPENDENCY_OF", 716 "%s:%s" % (package_ref.externalDocumentId, spdx_package.SPDXID) 717 ) 718 seen_deps.add(dep) 719 720 oe.sbom.write_doc(d, runtime_doc, pkg_arch, "runtime", spdx_deploy, indent=get_json_indent(d)) 721} 722 723do_create_runtime_spdx[vardepsexclude] += "OVERRIDES SPDX_MULTILIB_SSTATE_ARCHS" 724 725addtask do_create_runtime_spdx after do_create_spdx before do_build do_rm_work 726SSTATETASKS += "do_create_runtime_spdx" 727do_create_runtime_spdx[sstate-inputdirs] = "${SPDXRUNTIMEDEPLOY}" 728do_create_runtime_spdx[sstate-outputdirs] = "${DEPLOY_DIR_SPDX}" 729 730python do_create_runtime_spdx_setscene () { 731 sstate_setscene(d) 732} 733addtask do_create_runtime_spdx_setscene 734 735do_create_runtime_spdx[dirs] = "${SPDXRUNTIMEDEPLOY}" 736do_create_runtime_spdx[cleandirs] = "${SPDXRUNTIMEDEPLOY}" 737do_create_runtime_spdx[rdeptask] = "do_create_spdx" 738 739do_rootfs[recrdeptask] += "do_create_spdx do_create_runtime_spdx" 740do_rootfs[cleandirs] += "${SPDXIMAGEWORK}" 741 742ROOTFS_POSTUNINSTALL_COMMAND =+ "image_combine_spdx" 743 744do_populate_sdk[recrdeptask] += "do_create_spdx do_create_runtime_spdx" 745do_populate_sdk[cleandirs] += "${SPDXSDKWORK}" 746POPULATE_SDK_POST_HOST_COMMAND:append:task-populate-sdk = " sdk_host_combine_spdx" 747POPULATE_SDK_POST_TARGET_COMMAND:append:task-populate-sdk = " sdk_target_combine_spdx" 748 749python image_combine_spdx() { 750 import os 751 import oe.sbom 752 from pathlib import Path 753 from oe.rootfs import image_list_installed_packages 754 755 image_name = d.getVar("IMAGE_NAME") 756 image_link_name = d.getVar("IMAGE_LINK_NAME") 757 imgdeploydir = Path(d.getVar("IMGDEPLOYDIR")) 758 img_spdxid = oe.sbom.get_image_spdxid(image_name) 759 packages = image_list_installed_packages(d) 760 761 combine_spdx(d, image_name, imgdeploydir, img_spdxid, packages, Path(d.getVar("SPDXIMAGEWORK"))) 762 763 def make_image_link(target_path, suffix): 764 if image_link_name: 765 link = imgdeploydir / (image_link_name + suffix) 766 if link != target_path: 767 link.symlink_to(os.path.relpath(target_path, link.parent)) 768 769 spdx_tar_path = imgdeploydir / (image_name + ".spdx.tar.zst") 770 make_image_link(spdx_tar_path, ".spdx.tar.zst") 771} 772 773python sdk_host_combine_spdx() { 774 sdk_combine_spdx(d, "host") 775} 776 777python sdk_target_combine_spdx() { 778 sdk_combine_spdx(d, "target") 779} 780 781def sdk_combine_spdx(d, sdk_type): 782 import oe.sbom 783 from pathlib import Path 784 from oe.sdk import sdk_list_installed_packages 785 786 sdk_name = d.getVar("TOOLCHAIN_OUTPUTNAME") + "-" + sdk_type 787 sdk_deploydir = Path(d.getVar("SDKDEPLOYDIR")) 788 sdk_spdxid = oe.sbom.get_sdk_spdxid(sdk_name) 789 sdk_packages = sdk_list_installed_packages(d, sdk_type == "target") 790 combine_spdx(d, sdk_name, sdk_deploydir, sdk_spdxid, sdk_packages, Path(d.getVar('SPDXSDKWORK'))) 791 792def combine_spdx(d, rootfs_name, rootfs_deploydir, rootfs_spdxid, packages, spdx_workdir): 793 import os 794 import oe.spdx 795 import oe.sbom 796 import oe.spdx_common 797 import io 798 import json 799 from datetime import timezone, datetime 800 from pathlib import Path 801 import tarfile 802 import bb.compress.zstd 803 804 license_data = oe.spdx_common.load_spdx_license_data(d) 805 806 providers = oe.spdx_common.collect_package_providers(d) 807 package_archs = d.getVar("SPDX_MULTILIB_SSTATE_ARCHS").split() 808 package_archs.reverse() 809 810 creation_time = datetime.now(tz=timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ") 811 deploy_dir_spdx = Path(d.getVar("DEPLOY_DIR_SPDX")) 812 source_date_epoch = d.getVar("SOURCE_DATE_EPOCH") 813 814 doc = oe.spdx.SPDXDocument() 815 doc.name = rootfs_name 816 doc.documentNamespace = get_namespace(d, doc.name) 817 doc.creationInfo.created = creation_time 818 doc.creationInfo.comment = "This document was created by analyzing the source of the Yocto recipe during the build." 819 doc.creationInfo.licenseListVersion = get_license_list_version(license_data, d) 820 doc.creationInfo.creators.append("Tool: OpenEmbedded Core create-spdx.bbclass") 821 doc.creationInfo.creators.append("Organization: %s" % d.getVar("SPDX_ORG")) 822 doc.creationInfo.creators.append("Person: N/A ()") 823 824 image = oe.spdx.SPDXPackage() 825 image.name = d.getVar("PN") 826 image.versionInfo = d.getVar("PV") 827 image.SPDXID = rootfs_spdxid 828 image.supplier = d.getVar("SPDX_SUPPLIER") 829 830 doc.packages.append(image) 831 832 if packages: 833 for name in sorted(packages.keys()): 834 if name not in providers: 835 bb.fatal("Unable to find SPDX provider for '%s'" % name) 836 837 pkg_name, pkg_hashfn = providers[name] 838 839 pkg_spdx_path = oe.sbom.doc_find_by_hashfn(deploy_dir_spdx, package_archs, pkg_name, pkg_hashfn) 840 if not pkg_spdx_path: 841 bb.fatal("No SPDX file found for package %s, %s" % (pkg_name, pkg_hashfn)) 842 843 pkg_doc, pkg_doc_sha1 = oe.sbom.read_doc(pkg_spdx_path) 844 845 for p in pkg_doc.packages: 846 if p.name == name: 847 pkg_ref = oe.spdx.SPDXExternalDocumentRef() 848 pkg_ref.externalDocumentId = "DocumentRef-%s" % pkg_doc.name 849 pkg_ref.spdxDocument = pkg_doc.documentNamespace 850 pkg_ref.checksum.algorithm = "SHA1" 851 pkg_ref.checksum.checksumValue = pkg_doc_sha1 852 853 doc.externalDocumentRefs.append(pkg_ref) 854 doc.add_relationship(image, "CONTAINS", "%s:%s" % (pkg_ref.externalDocumentId, p.SPDXID)) 855 break 856 else: 857 bb.fatal("Unable to find package with name '%s' in SPDX file %s" % (name, pkg_spdx_path)) 858 859 runtime_spdx_path = oe.sbom.doc_find_by_hashfn(deploy_dir_spdx, package_archs, "runtime-" + name, pkg_hashfn) 860 if not runtime_spdx_path: 861 bb.fatal("No runtime SPDX document found for %s, %s" % (name, pkg_hashfn)) 862 863 runtime_doc, runtime_doc_sha1 = oe.sbom.read_doc(runtime_spdx_path) 864 865 runtime_ref = oe.spdx.SPDXExternalDocumentRef() 866 runtime_ref.externalDocumentId = "DocumentRef-%s" % runtime_doc.name 867 runtime_ref.spdxDocument = runtime_doc.documentNamespace 868 runtime_ref.checksum.algorithm = "SHA1" 869 runtime_ref.checksum.checksumValue = runtime_doc_sha1 870 871 # "OTHER" isn't ideal here, but I can't find a relationship that makes sense 872 doc.externalDocumentRefs.append(runtime_ref) 873 doc.add_relationship( 874 image, 875 "OTHER", 876 "%s:%s" % (runtime_ref.externalDocumentId, runtime_doc.SPDXID), 877 comment="Runtime dependencies for %s" % name 878 ) 879 bb.utils.mkdirhier(spdx_workdir) 880 image_spdx_path = spdx_workdir / (rootfs_name + ".spdx.json") 881 882 with image_spdx_path.open("wb") as f: 883 doc.to_json(f, sort_keys=True, indent=get_json_indent(d)) 884 885 num_threads = int(d.getVar("BB_NUMBER_THREADS")) 886 887 visited_docs = set() 888 889 index = {"documents": []} 890 891 spdx_tar_path = rootfs_deploydir / (rootfs_name + ".spdx.tar.zst") 892 with bb.compress.zstd.open(spdx_tar_path, "w", num_threads=num_threads) as f: 893 with tarfile.open(fileobj=f, mode="w|") as tar: 894 def collect_spdx_document(path): 895 nonlocal tar 896 nonlocal deploy_dir_spdx 897 nonlocal source_date_epoch 898 nonlocal index 899 900 if path in visited_docs: 901 return 902 903 visited_docs.add(path) 904 905 with path.open("rb") as f: 906 doc, sha1 = oe.sbom.read_doc(f) 907 f.seek(0) 908 909 if doc.documentNamespace in visited_docs: 910 return 911 912 bb.note("Adding SPDX document %s" % path) 913 visited_docs.add(doc.documentNamespace) 914 info = tar.gettarinfo(fileobj=f) 915 916 info.name = doc.name + ".spdx.json" 917 info.uid = 0 918 info.gid = 0 919 info.uname = "root" 920 info.gname = "root" 921 922 if source_date_epoch is not None and info.mtime > int(source_date_epoch): 923 info.mtime = int(source_date_epoch) 924 925 tar.addfile(info, f) 926 927 index["documents"].append({ 928 "filename": info.name, 929 "documentNamespace": doc.documentNamespace, 930 "sha1": sha1, 931 }) 932 933 for ref in doc.externalDocumentRefs: 934 ref_path = oe.sbom.doc_find_by_namespace(deploy_dir_spdx, package_archs, ref.spdxDocument) 935 if not ref_path: 936 bb.fatal("Cannot find any SPDX file for document %s" % ref.spdxDocument) 937 collect_spdx_document(ref_path) 938 939 collect_spdx_document(image_spdx_path) 940 941 index["documents"].sort(key=lambda x: x["filename"]) 942 943 index_str = io.BytesIO(json.dumps( 944 index, 945 sort_keys=True, 946 indent=get_json_indent(d), 947 ).encode("utf-8")) 948 949 info = tarfile.TarInfo() 950 info.name = "index.json" 951 info.size = len(index_str.getvalue()) 952 info.uid = 0 953 info.gid = 0 954 info.uname = "root" 955 info.gname = "root" 956 957 tar.addfile(info, fileobj=index_str) 958 959combine_spdx[vardepsexclude] += "BB_NUMBER_THREADS SPDX_MULTILIB_SSTATE_ARCHS" 960