1# 2# Copyright OpenEmbedded Contributors 3# 4# SPDX-License-Identifier: GPL-2.0-only 5# 6 7inherit spdx-common 8 9SPDX_VERSION = "2.2" 10 11SPDX_ORG ??= "OpenEmbedded ()" 12SPDX_SUPPLIER ??= "Organization: ${SPDX_ORG}" 13SPDX_SUPPLIER[doc] = "The SPDX PackageSupplier field for SPDX packages created from \ 14 this recipe. For SPDX documents create using this class during the build, this \ 15 is the contact information for the person or organization who is doing the \ 16 build." 17 18 19def get_namespace(d, name): 20 import uuid 21 namespace_uuid = uuid.uuid5(uuid.NAMESPACE_DNS, d.getVar("SPDX_UUID_NAMESPACE")) 22 return "%s/%s-%s" % (d.getVar("SPDX_NAMESPACE_PREFIX"), name, str(uuid.uuid5(namespace_uuid, name))) 23 24 25def create_annotation(d, comment): 26 from datetime import datetime, timezone 27 28 creation_time = datetime.now(tz=timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ") 29 annotation = oe.spdx.SPDXAnnotation() 30 annotation.annotationDate = creation_time 31 annotation.annotationType = "OTHER" 32 annotation.annotator = "Tool: %s - %s" % (d.getVar("SPDX_TOOL_NAME"), d.getVar("SPDX_TOOL_VERSION")) 33 annotation.comment = comment 34 return annotation 35 36def recipe_spdx_is_native(d, recipe): 37 return any(a.annotationType == "OTHER" and 38 a.annotator == "Tool: %s - %s" % (d.getVar("SPDX_TOOL_NAME"), d.getVar("SPDX_TOOL_VERSION")) and 39 a.comment == "isNative" for a in recipe.annotations) 40 41def get_json_indent(d): 42 if d.getVar("SPDX_PRETTY") == "1": 43 return 2 44 return None 45 46 47def convert_license_to_spdx(lic, license_data, document, d, existing={}): 48 from pathlib import Path 49 import oe.spdx 50 51 extracted = {} 52 53 def add_extracted_license(ident, name): 54 nonlocal document 55 56 if name in extracted: 57 return 58 59 extracted_info = oe.spdx.SPDXExtractedLicensingInfo() 60 extracted_info.name = name 61 extracted_info.licenseId = ident 62 extracted_info.extractedText = None 63 64 if name == "PD": 65 # Special-case this. 66 extracted_info.extractedText = "Software released to the public domain" 67 else: 68 # Seach for the license in COMMON_LICENSE_DIR and LICENSE_PATH 69 for directory in [d.getVar('COMMON_LICENSE_DIR')] + (d.getVar('LICENSE_PATH') or '').split(): 70 try: 71 with (Path(directory) / name).open(errors="replace") as f: 72 extracted_info.extractedText = f.read() 73 break 74 except FileNotFoundError: 75 pass 76 if extracted_info.extractedText is None: 77 # If it's not SPDX or PD, then NO_GENERIC_LICENSE must be set 78 filename = d.getVarFlag('NO_GENERIC_LICENSE', name) 79 if filename: 80 filename = d.expand("${S}/" + filename) 81 with open(filename, errors="replace") as f: 82 extracted_info.extractedText = f.read() 83 else: 84 bb.fatal("Cannot find any text for license %s" % name) 85 86 extracted[name] = extracted_info 87 document.hasExtractedLicensingInfos.append(extracted_info) 88 89 def convert(l): 90 if l == "(" or l == ")": 91 return l 92 93 if l == "&": 94 return "AND" 95 96 if l == "|": 97 return "OR" 98 99 if l == "CLOSED": 100 return "NONE" 101 102 spdx_license = d.getVarFlag("SPDXLICENSEMAP", l) or l 103 if spdx_license in license_data["licenses"]: 104 return spdx_license 105 106 try: 107 spdx_license = existing[l] 108 except KeyError: 109 spdx_license = "LicenseRef-" + l 110 add_extracted_license(spdx_license, l) 111 112 return spdx_license 113 114 lic_split = lic.replace("(", " ( ").replace(")", " ) ").replace("|", " | ").replace("&", " & ").split() 115 116 return ' '.join(convert(l) for l in lic_split) 117 118def add_package_files(d, doc, spdx_pkg, topdir, get_spdxid, get_types, *, archive=None, ignore_dirs=[], ignore_top_level_dirs=[]): 119 from pathlib import Path 120 import oe.spdx 121 import oe.spdx_common 122 import hashlib 123 124 source_date_epoch = d.getVar("SOURCE_DATE_EPOCH") 125 if source_date_epoch: 126 source_date_epoch = int(source_date_epoch) 127 128 sha1s = [] 129 spdx_files = [] 130 131 file_counter = 1 132 for subdir, dirs, files in os.walk(topdir): 133 dirs[:] = [d for d in dirs if d not in ignore_dirs] 134 if subdir == str(topdir): 135 dirs[:] = [d for d in dirs if d not in ignore_top_level_dirs] 136 137 for file in files: 138 filepath = Path(subdir) / file 139 filename = str(filepath.relative_to(topdir)) 140 141 if not filepath.is_symlink() and filepath.is_file(): 142 spdx_file = oe.spdx.SPDXFile() 143 spdx_file.SPDXID = get_spdxid(file_counter) 144 for t in get_types(filepath): 145 spdx_file.fileTypes.append(t) 146 spdx_file.fileName = filename 147 148 if archive is not None: 149 with filepath.open("rb") as f: 150 info = archive.gettarinfo(fileobj=f) 151 info.name = filename 152 info.uid = 0 153 info.gid = 0 154 info.uname = "root" 155 info.gname = "root" 156 157 if source_date_epoch is not None and info.mtime > source_date_epoch: 158 info.mtime = source_date_epoch 159 160 archive.addfile(info, f) 161 162 sha1 = bb.utils.sha1_file(filepath) 163 sha1s.append(sha1) 164 spdx_file.checksums.append(oe.spdx.SPDXChecksum( 165 algorithm="SHA1", 166 checksumValue=sha1, 167 )) 168 spdx_file.checksums.append(oe.spdx.SPDXChecksum( 169 algorithm="SHA256", 170 checksumValue=bb.utils.sha256_file(filepath), 171 )) 172 173 if "SOURCE" in spdx_file.fileTypes: 174 extracted_lics = oe.spdx_common.extract_licenses(filepath) 175 if extracted_lics: 176 spdx_file.licenseInfoInFiles = extracted_lics 177 178 doc.files.append(spdx_file) 179 doc.add_relationship(spdx_pkg, "CONTAINS", spdx_file) 180 spdx_pkg.hasFiles.append(spdx_file.SPDXID) 181 182 spdx_files.append(spdx_file) 183 184 file_counter += 1 185 186 sha1s.sort() 187 verifier = hashlib.sha1() 188 for v in sha1s: 189 verifier.update(v.encode("utf-8")) 190 spdx_pkg.packageVerificationCode.packageVerificationCodeValue = verifier.hexdigest() 191 192 return spdx_files 193 194 195def add_package_sources_from_debug(d, package_doc, spdx_package, package, package_files, sources): 196 from pathlib import Path 197 import hashlib 198 import oe.packagedata 199 import oe.spdx 200 201 debug_search_paths = [ 202 Path(d.getVar('PKGD')), 203 Path(d.getVar('STAGING_DIR_TARGET')), 204 Path(d.getVar('STAGING_DIR_NATIVE')), 205 Path(d.getVar('STAGING_KERNEL_DIR')), 206 ] 207 208 pkg_data = oe.packagedata.read_subpkgdata_extended(package, d) 209 210 if pkg_data is None: 211 return 212 213 for file_path, file_data in pkg_data["files_info"].items(): 214 if not "debugsrc" in file_data: 215 continue 216 217 for pkg_file in package_files: 218 if file_path.lstrip("/") == pkg_file.fileName.lstrip("/"): 219 break 220 else: 221 bb.fatal("No package file found for %s in %s; SPDX found: %s" % (str(file_path), package, 222 " ".join(p.fileName for p in package_files))) 223 continue 224 225 for debugsrc in file_data["debugsrc"]: 226 ref_id = "NOASSERTION" 227 for search in debug_search_paths: 228 if debugsrc.startswith("/usr/src/kernel"): 229 debugsrc_path = search / debugsrc.replace('/usr/src/kernel/', '') 230 else: 231 debugsrc_path = search / debugsrc.lstrip("/") 232 # We can only hash files below, skip directories, links, etc. 233 if not os.path.isfile(debugsrc_path): 234 continue 235 236 file_sha256 = bb.utils.sha256_file(debugsrc_path) 237 238 if file_sha256 in sources: 239 source_file = sources[file_sha256] 240 241 doc_ref = package_doc.find_external_document_ref(source_file.doc.documentNamespace) 242 if doc_ref is None: 243 doc_ref = oe.spdx.SPDXExternalDocumentRef() 244 doc_ref.externalDocumentId = "DocumentRef-dependency-" + source_file.doc.name 245 doc_ref.spdxDocument = source_file.doc.documentNamespace 246 doc_ref.checksum.algorithm = "SHA1" 247 doc_ref.checksum.checksumValue = source_file.doc_sha1 248 package_doc.externalDocumentRefs.append(doc_ref) 249 250 ref_id = "%s:%s" % (doc_ref.externalDocumentId, source_file.file.SPDXID) 251 else: 252 bb.debug(1, "Debug source %s with SHA256 %s not found in any dependency" % (str(debugsrc_path), file_sha256)) 253 break 254 else: 255 bb.debug(1, "Debug source %s not found" % debugsrc) 256 257 package_doc.add_relationship(pkg_file, "GENERATED_FROM", ref_id, comment=debugsrc) 258 259add_package_sources_from_debug[vardepsexclude] += "STAGING_KERNEL_DIR" 260 261def collect_dep_recipes(d, doc, spdx_recipe): 262 import json 263 from pathlib import Path 264 import oe.sbom 265 import oe.spdx 266 import oe.spdx_common 267 268 deploy_dir_spdx = Path(d.getVar("DEPLOY_DIR_SPDX")) 269 package_archs = d.getVar("SPDX_MULTILIB_SSTATE_ARCHS").split() 270 package_archs.reverse() 271 272 dep_recipes = [] 273 274 deps = oe.spdx_common.get_spdx_deps(d) 275 276 for dep_pn, dep_hashfn, in_taskhash in deps: 277 # If this dependency is not calculated in the taskhash skip it. 278 # Otherwise, it can result in broken links since this task won't 279 # rebuild and see the new SPDX ID if the dependency changes 280 if not in_taskhash: 281 continue 282 283 dep_recipe_path = oe.sbom.doc_find_by_hashfn(deploy_dir_spdx, package_archs, "recipe-" + dep_pn, dep_hashfn) 284 if not dep_recipe_path: 285 bb.fatal("Cannot find any SPDX file for recipe %s, %s" % (dep_pn, dep_hashfn)) 286 287 spdx_dep_doc, spdx_dep_sha1 = oe.sbom.read_doc(dep_recipe_path) 288 289 for pkg in spdx_dep_doc.packages: 290 if pkg.name == dep_pn: 291 spdx_dep_recipe = pkg 292 break 293 else: 294 continue 295 296 dep_recipes.append(oe.sbom.DepRecipe(spdx_dep_doc, spdx_dep_sha1, spdx_dep_recipe)) 297 298 dep_recipe_ref = oe.spdx.SPDXExternalDocumentRef() 299 dep_recipe_ref.externalDocumentId = "DocumentRef-dependency-" + spdx_dep_doc.name 300 dep_recipe_ref.spdxDocument = spdx_dep_doc.documentNamespace 301 dep_recipe_ref.checksum.algorithm = "SHA1" 302 dep_recipe_ref.checksum.checksumValue = spdx_dep_sha1 303 304 doc.externalDocumentRefs.append(dep_recipe_ref) 305 306 doc.add_relationship( 307 "%s:%s" % (dep_recipe_ref.externalDocumentId, spdx_dep_recipe.SPDXID), 308 "BUILD_DEPENDENCY_OF", 309 spdx_recipe 310 ) 311 312 return dep_recipes 313 314collect_dep_recipes[vardepsexclude] = "SPDX_MULTILIB_SSTATE_ARCHS" 315 316def collect_dep_sources(d, dep_recipes): 317 import oe.sbom 318 319 sources = {} 320 for dep in dep_recipes: 321 # Don't collect sources from native recipes as they 322 # match non-native sources also. 323 if recipe_spdx_is_native(d, dep.recipe): 324 continue 325 recipe_files = set(dep.recipe.hasFiles) 326 327 for spdx_file in dep.doc.files: 328 if spdx_file.SPDXID not in recipe_files: 329 continue 330 331 if "SOURCE" in spdx_file.fileTypes: 332 for checksum in spdx_file.checksums: 333 if checksum.algorithm == "SHA256": 334 sources[checksum.checksumValue] = oe.sbom.DepSource(dep.doc, dep.doc_sha1, dep.recipe, spdx_file) 335 break 336 337 return sources 338 339def add_download_packages(d, doc, recipe): 340 import os.path 341 from bb.fetch2 import decodeurl, CHECKSUM_LIST 342 import bb.process 343 import oe.spdx 344 import oe.sbom 345 346 for download_idx, src_uri in enumerate(d.getVar('SRC_URI').split()): 347 f = bb.fetch2.FetchData(src_uri, d) 348 349 for name in f.names: 350 package = oe.spdx.SPDXPackage() 351 package.name = "%s-source-%d" % (d.getVar("PN"), download_idx + 1) 352 package.SPDXID = oe.sbom.get_download_spdxid(d, download_idx + 1) 353 354 if f.type == "file": 355 continue 356 357 if f.method.supports_checksum(f): 358 for checksum_id in CHECKSUM_LIST: 359 if checksum_id.upper() not in oe.spdx.SPDXPackage.ALLOWED_CHECKSUMS: 360 continue 361 362 expected_checksum = getattr(f, "%s_expected" % checksum_id) 363 if expected_checksum is None: 364 continue 365 366 c = oe.spdx.SPDXChecksum() 367 c.algorithm = checksum_id.upper() 368 c.checksumValue = expected_checksum 369 package.checksums.append(c) 370 371 package.downloadLocation = oe.spdx_common.fetch_data_to_uri(f, name) 372 doc.packages.append(package) 373 doc.add_relationship(doc, "DESCRIBES", package) 374 # In the future, we might be able to do more fancy dependencies, 375 # but this should be sufficient for now 376 doc.add_relationship(package, "BUILD_DEPENDENCY_OF", recipe) 377 378def get_license_list_version(license_data, d): 379 # Newer versions of the SPDX license list are SemVer ("MAJOR.MINOR.MICRO"), 380 # but SPDX 2 only uses "MAJOR.MINOR". 381 return ".".join(license_data["licenseListVersion"].split(".")[:2]) 382 383 384python do_create_spdx() { 385 from datetime import datetime, timezone 386 import oe.sbom 387 import oe.spdx 388 import oe.spdx_common 389 import uuid 390 from pathlib import Path 391 from contextlib import contextmanager 392 import oe.cve_check 393 394 license_data = oe.spdx_common.load_spdx_license_data(d) 395 396 @contextmanager 397 def optional_tarfile(name, guard, mode="w"): 398 import tarfile 399 import bb.compress.zstd 400 401 num_threads = int(d.getVar("BB_NUMBER_THREADS")) 402 403 if guard: 404 name.parent.mkdir(parents=True, exist_ok=True) 405 with bb.compress.zstd.open(name, mode=mode + "b", num_threads=num_threads) as f: 406 with tarfile.open(fileobj=f, mode=mode + "|") as tf: 407 yield tf 408 else: 409 yield None 410 411 412 deploy_dir_spdx = Path(d.getVar("DEPLOY_DIR_SPDX")) 413 spdx_workdir = Path(d.getVar("SPDXWORK")) 414 include_sources = d.getVar("SPDX_INCLUDE_SOURCES") == "1" 415 archive_sources = d.getVar("SPDX_ARCHIVE_SOURCES") == "1" 416 archive_packaged = d.getVar("SPDX_ARCHIVE_PACKAGED") == "1" 417 pkg_arch = d.getVar("SSTATE_PKGARCH") 418 419 creation_time = datetime.now(tz=timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ") 420 421 doc = oe.spdx.SPDXDocument() 422 423 doc.name = "recipe-" + d.getVar("PN") 424 doc.documentNamespace = get_namespace(d, doc.name) 425 doc.creationInfo.created = creation_time 426 doc.creationInfo.comment = "This document was created by analyzing recipe files during the build." 427 doc.creationInfo.licenseListVersion = get_license_list_version(license_data, d) 428 doc.creationInfo.creators.append("Tool: OpenEmbedded Core create-spdx.bbclass") 429 doc.creationInfo.creators.append("Organization: %s" % d.getVar("SPDX_ORG")) 430 doc.creationInfo.creators.append("Person: N/A ()") 431 432 recipe = oe.spdx.SPDXPackage() 433 recipe.name = d.getVar("PN") 434 recipe.versionInfo = d.getVar("PV") 435 recipe.SPDXID = oe.sbom.get_recipe_spdxid(d) 436 recipe.supplier = d.getVar("SPDX_SUPPLIER") 437 if bb.data.inherits_class("native", d) or bb.data.inherits_class("cross", d): 438 recipe.annotations.append(create_annotation(d, "isNative")) 439 440 homepage = d.getVar("HOMEPAGE") 441 if homepage: 442 recipe.homepage = homepage 443 444 license = d.getVar("LICENSE") 445 if license: 446 recipe.licenseDeclared = convert_license_to_spdx(license, license_data, doc, d) 447 448 summary = d.getVar("SUMMARY") 449 if summary: 450 recipe.summary = summary 451 452 description = d.getVar("DESCRIPTION") 453 if description: 454 recipe.description = description 455 456 if d.getVar("SPDX_CUSTOM_ANNOTATION_VARS"): 457 for var in d.getVar('SPDX_CUSTOM_ANNOTATION_VARS').split(): 458 recipe.annotations.append(create_annotation(d, var + "=" + d.getVar(var))) 459 460 # Some CVEs may be patched during the build process without incrementing the version number, 461 # so querying for CVEs based on the CPE id can lead to false positives. To account for this, 462 # save the CVEs fixed by patches to source information field in the SPDX. 463 patched_cves = oe.cve_check.get_patched_cves(d) 464 patched_cves = list(patched_cves) 465 patched_cves = ' '.join(patched_cves) 466 if patched_cves: 467 recipe.sourceInfo = "CVEs fixed: " + patched_cves 468 469 cpe_ids = oe.cve_check.get_cpe_ids(d.getVar("CVE_PRODUCT"), d.getVar("CVE_VERSION")) 470 if cpe_ids: 471 for cpe_id in cpe_ids: 472 cpe = oe.spdx.SPDXExternalReference() 473 cpe.referenceCategory = "SECURITY" 474 cpe.referenceType = "http://spdx.org/rdf/references/cpe23Type" 475 cpe.referenceLocator = cpe_id 476 recipe.externalRefs.append(cpe) 477 478 doc.packages.append(recipe) 479 doc.add_relationship(doc, "DESCRIBES", recipe) 480 481 add_download_packages(d, doc, recipe) 482 483 if oe.spdx_common.process_sources(d) and include_sources: 484 recipe_archive = deploy_dir_spdx / "recipes" / (doc.name + ".tar.zst") 485 with optional_tarfile(recipe_archive, archive_sources) as archive: 486 oe.spdx_common.get_patched_src(d) 487 488 add_package_files( 489 d, 490 doc, 491 recipe, 492 spdx_workdir, 493 lambda file_counter: "SPDXRef-SourceFile-%s-%d" % (d.getVar("PN"), file_counter), 494 lambda filepath: ["SOURCE"], 495 ignore_dirs=[".git"], 496 ignore_top_level_dirs=["temp"], 497 archive=archive, 498 ) 499 500 if archive is not None: 501 recipe.packageFileName = str(recipe_archive.name) 502 503 dep_recipes = collect_dep_recipes(d, doc, recipe) 504 505 doc_sha1 = oe.sbom.write_doc(d, doc, pkg_arch, "recipes", indent=get_json_indent(d)) 506 dep_recipes.append(oe.sbom.DepRecipe(doc, doc_sha1, recipe)) 507 508 recipe_ref = oe.spdx.SPDXExternalDocumentRef() 509 recipe_ref.externalDocumentId = "DocumentRef-recipe-" + recipe.name 510 recipe_ref.spdxDocument = doc.documentNamespace 511 recipe_ref.checksum.algorithm = "SHA1" 512 recipe_ref.checksum.checksumValue = doc_sha1 513 514 sources = collect_dep_sources(d, dep_recipes) 515 found_licenses = {license.name:recipe_ref.externalDocumentId + ":" + license.licenseId for license in doc.hasExtractedLicensingInfos} 516 517 if not recipe_spdx_is_native(d, recipe): 518 bb.build.exec_func("read_subpackage_metadata", d) 519 520 pkgdest = Path(d.getVar("PKGDEST")) 521 for package in d.getVar("PACKAGES").split(): 522 if not oe.packagedata.packaged(package, d): 523 continue 524 525 package_doc = oe.spdx.SPDXDocument() 526 pkg_name = d.getVar("PKG:%s" % package) or package 527 package_doc.name = pkg_name 528 package_doc.documentNamespace = get_namespace(d, package_doc.name) 529 package_doc.creationInfo.created = creation_time 530 package_doc.creationInfo.comment = "This document was created by analyzing packages created during the build." 531 package_doc.creationInfo.licenseListVersion = get_license_list_version(license_data, d) 532 package_doc.creationInfo.creators.append("Tool: OpenEmbedded Core create-spdx.bbclass") 533 package_doc.creationInfo.creators.append("Organization: %s" % d.getVar("SPDX_ORG")) 534 package_doc.creationInfo.creators.append("Person: N/A ()") 535 package_doc.externalDocumentRefs.append(recipe_ref) 536 537 package_license = d.getVar("LICENSE:%s" % package) or d.getVar("LICENSE") 538 539 spdx_package = oe.spdx.SPDXPackage() 540 541 spdx_package.SPDXID = oe.sbom.get_package_spdxid(pkg_name) 542 spdx_package.name = pkg_name 543 spdx_package.versionInfo = d.getVar("PV") 544 spdx_package.licenseDeclared = convert_license_to_spdx(package_license, license_data, package_doc, d, found_licenses) 545 spdx_package.supplier = d.getVar("SPDX_SUPPLIER") 546 547 package_doc.packages.append(spdx_package) 548 549 package_doc.add_relationship(spdx_package, "GENERATED_FROM", "%s:%s" % (recipe_ref.externalDocumentId, recipe.SPDXID)) 550 package_doc.add_relationship(package_doc, "DESCRIBES", spdx_package) 551 552 package_archive = deploy_dir_spdx / "packages" / (package_doc.name + ".tar.zst") 553 with optional_tarfile(package_archive, archive_packaged) as archive: 554 package_files = add_package_files( 555 d, 556 package_doc, 557 spdx_package, 558 pkgdest / package, 559 lambda file_counter: oe.sbom.get_packaged_file_spdxid(pkg_name, file_counter), 560 lambda filepath: ["BINARY"], 561 ignore_top_level_dirs=['CONTROL', 'DEBIAN'], 562 archive=archive, 563 ) 564 565 if archive is not None: 566 spdx_package.packageFileName = str(package_archive.name) 567 568 add_package_sources_from_debug(d, package_doc, spdx_package, package, package_files, sources) 569 570 oe.sbom.write_doc(d, package_doc, pkg_arch, "packages", indent=get_json_indent(d)) 571} 572do_create_spdx[vardepsexclude] += "BB_NUMBER_THREADS" 573# NOTE: depending on do_unpack is a hack that is necessary to get it's dependencies for archive the source 574addtask do_create_spdx after do_package do_packagedata do_unpack do_collect_spdx_deps before do_populate_sdk do_build do_rm_work 575 576SSTATETASKS += "do_create_spdx" 577do_create_spdx[sstate-inputdirs] = "${SPDXDEPLOY}" 578do_create_spdx[sstate-outputdirs] = "${DEPLOY_DIR_SPDX}" 579 580python do_create_spdx_setscene () { 581 sstate_setscene(d) 582} 583addtask do_create_spdx_setscene 584 585do_create_spdx[dirs] = "${SPDXWORK}" 586do_create_spdx[cleandirs] = "${SPDXDEPLOY} ${SPDXWORK}" 587do_create_spdx[depends] += " \ 588 ${PATCHDEPENDENCY} \ 589 ${@create_spdx_source_deps(d)} \ 590" 591 592python do_create_runtime_spdx() { 593 from datetime import datetime, timezone 594 import oe.sbom 595 import oe.spdx 596 import oe.spdx_common 597 import oe.packagedata 598 from pathlib import Path 599 600 deploy_dir_spdx = Path(d.getVar("DEPLOY_DIR_SPDX")) 601 spdx_deploy = Path(d.getVar("SPDXRUNTIMEDEPLOY")) 602 is_native = bb.data.inherits_class("native", d) or bb.data.inherits_class("cross", d) 603 604 creation_time = datetime.now(tz=timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ") 605 606 license_data = oe.spdx_common.load_spdx_license_data(d) 607 608 providers = oe.spdx_common.collect_package_providers(d) 609 pkg_arch = d.getVar("SSTATE_PKGARCH") 610 package_archs = d.getVar("SPDX_MULTILIB_SSTATE_ARCHS").split() 611 package_archs.reverse() 612 613 if not is_native: 614 bb.build.exec_func("read_subpackage_metadata", d) 615 616 dep_package_cache = {} 617 618 pkgdest = Path(d.getVar("PKGDEST")) 619 for package in d.getVar("PACKAGES").split(): 620 localdata = bb.data.createCopy(d) 621 pkg_name = d.getVar("PKG:%s" % package) or package 622 localdata.setVar("PKG", pkg_name) 623 localdata.setVar('OVERRIDES', d.getVar("OVERRIDES", False) + ":" + package) 624 625 if not oe.packagedata.packaged(package, localdata): 626 continue 627 628 pkg_spdx_path = oe.sbom.doc_path(deploy_dir_spdx, pkg_name, pkg_arch, "packages") 629 630 package_doc, package_doc_sha1 = oe.sbom.read_doc(pkg_spdx_path) 631 632 for p in package_doc.packages: 633 if p.name == pkg_name: 634 spdx_package = p 635 break 636 else: 637 bb.fatal("Package '%s' not found in %s" % (pkg_name, pkg_spdx_path)) 638 639 runtime_doc = oe.spdx.SPDXDocument() 640 runtime_doc.name = "runtime-" + pkg_name 641 runtime_doc.documentNamespace = get_namespace(localdata, runtime_doc.name) 642 runtime_doc.creationInfo.created = creation_time 643 runtime_doc.creationInfo.comment = "This document was created by analyzing package runtime dependencies." 644 runtime_doc.creationInfo.licenseListVersion = get_license_list_version(license_data, d) 645 runtime_doc.creationInfo.creators.append("Tool: OpenEmbedded Core create-spdx.bbclass") 646 runtime_doc.creationInfo.creators.append("Organization: %s" % d.getVar("SPDX_ORG")) 647 runtime_doc.creationInfo.creators.append("Person: N/A ()") 648 649 package_ref = oe.spdx.SPDXExternalDocumentRef() 650 package_ref.externalDocumentId = "DocumentRef-package-" + package 651 package_ref.spdxDocument = package_doc.documentNamespace 652 package_ref.checksum.algorithm = "SHA1" 653 package_ref.checksum.checksumValue = package_doc_sha1 654 655 runtime_doc.externalDocumentRefs.append(package_ref) 656 657 runtime_doc.add_relationship( 658 runtime_doc.SPDXID, 659 "AMENDS", 660 "%s:%s" % (package_ref.externalDocumentId, package_doc.SPDXID) 661 ) 662 663 deps = bb.utils.explode_dep_versions2(localdata.getVar("RDEPENDS") or "") 664 seen_deps = set() 665 for dep, _ in deps.items(): 666 if dep in seen_deps: 667 continue 668 669 if dep not in providers: 670 continue 671 672 (dep, dep_hashfn) = providers[dep] 673 674 if not oe.packagedata.packaged(dep, localdata): 675 continue 676 677 dep_pkg_data = oe.packagedata.read_subpkgdata_dict(dep, d) 678 dep_pkg = dep_pkg_data["PKG"] 679 680 if dep in dep_package_cache: 681 (dep_spdx_package, dep_package_ref) = dep_package_cache[dep] 682 else: 683 dep_path = oe.sbom.doc_find_by_hashfn(deploy_dir_spdx, package_archs, dep_pkg, dep_hashfn) 684 if not dep_path: 685 bb.fatal("No SPDX file found for package %s, %s" % (dep_pkg, dep_hashfn)) 686 687 spdx_dep_doc, spdx_dep_sha1 = oe.sbom.read_doc(dep_path) 688 689 for pkg in spdx_dep_doc.packages: 690 if pkg.name == dep_pkg: 691 dep_spdx_package = pkg 692 break 693 else: 694 bb.fatal("Package '%s' not found in %s" % (dep_pkg, dep_path)) 695 696 dep_package_ref = oe.spdx.SPDXExternalDocumentRef() 697 dep_package_ref.externalDocumentId = "DocumentRef-runtime-dependency-" + spdx_dep_doc.name 698 dep_package_ref.spdxDocument = spdx_dep_doc.documentNamespace 699 dep_package_ref.checksum.algorithm = "SHA1" 700 dep_package_ref.checksum.checksumValue = spdx_dep_sha1 701 702 dep_package_cache[dep] = (dep_spdx_package, dep_package_ref) 703 704 runtime_doc.externalDocumentRefs.append(dep_package_ref) 705 706 runtime_doc.add_relationship( 707 "%s:%s" % (dep_package_ref.externalDocumentId, dep_spdx_package.SPDXID), 708 "RUNTIME_DEPENDENCY_OF", 709 "%s:%s" % (package_ref.externalDocumentId, spdx_package.SPDXID) 710 ) 711 seen_deps.add(dep) 712 713 oe.sbom.write_doc(d, runtime_doc, pkg_arch, "runtime", spdx_deploy, indent=get_json_indent(d)) 714} 715 716do_create_runtime_spdx[vardepsexclude] += "OVERRIDES SPDX_MULTILIB_SSTATE_ARCHS" 717 718addtask do_create_runtime_spdx after do_create_spdx before do_build do_rm_work 719SSTATETASKS += "do_create_runtime_spdx" 720do_create_runtime_spdx[sstate-inputdirs] = "${SPDXRUNTIMEDEPLOY}" 721do_create_runtime_spdx[sstate-outputdirs] = "${DEPLOY_DIR_SPDX}" 722 723python do_create_runtime_spdx_setscene () { 724 sstate_setscene(d) 725} 726addtask do_create_runtime_spdx_setscene 727 728do_create_runtime_spdx[dirs] = "${SPDXRUNTIMEDEPLOY}" 729do_create_runtime_spdx[cleandirs] = "${SPDXRUNTIMEDEPLOY}" 730do_create_runtime_spdx[rdeptask] = "do_create_spdx" 731 732do_rootfs[recrdeptask] += "do_create_spdx do_create_runtime_spdx" 733do_rootfs[cleandirs] += "${SPDXIMAGEWORK}" 734 735ROOTFS_POSTUNINSTALL_COMMAND =+ "image_combine_spdx" 736 737do_populate_sdk[recrdeptask] += "do_create_spdx do_create_runtime_spdx" 738do_populate_sdk[cleandirs] += "${SPDXSDKWORK}" 739POPULATE_SDK_POST_HOST_COMMAND:append:task-populate-sdk = " sdk_host_combine_spdx" 740POPULATE_SDK_POST_TARGET_COMMAND:append:task-populate-sdk = " sdk_target_combine_spdx" 741 742python image_combine_spdx() { 743 import os 744 import oe.sbom 745 from pathlib import Path 746 from oe.rootfs import image_list_installed_packages 747 748 image_name = d.getVar("IMAGE_NAME") 749 image_link_name = d.getVar("IMAGE_LINK_NAME") 750 imgdeploydir = Path(d.getVar("IMGDEPLOYDIR")) 751 img_spdxid = oe.sbom.get_image_spdxid(image_name) 752 packages = image_list_installed_packages(d) 753 754 combine_spdx(d, image_name, imgdeploydir, img_spdxid, packages, Path(d.getVar("SPDXIMAGEWORK"))) 755 756 def make_image_link(target_path, suffix): 757 if image_link_name: 758 link = imgdeploydir / (image_link_name + suffix) 759 if link != target_path: 760 link.symlink_to(os.path.relpath(target_path, link.parent)) 761 762 spdx_tar_path = imgdeploydir / (image_name + ".spdx.tar.zst") 763 make_image_link(spdx_tar_path, ".spdx.tar.zst") 764} 765 766python sdk_host_combine_spdx() { 767 sdk_combine_spdx(d, "host") 768} 769 770python sdk_target_combine_spdx() { 771 sdk_combine_spdx(d, "target") 772} 773 774def sdk_combine_spdx(d, sdk_type): 775 import oe.sbom 776 from pathlib import Path 777 from oe.sdk import sdk_list_installed_packages 778 779 sdk_name = d.getVar("TOOLCHAIN_OUTPUTNAME") + "-" + sdk_type 780 sdk_deploydir = Path(d.getVar("SDKDEPLOYDIR")) 781 sdk_spdxid = oe.sbom.get_sdk_spdxid(sdk_name) 782 sdk_packages = sdk_list_installed_packages(d, sdk_type == "target") 783 combine_spdx(d, sdk_name, sdk_deploydir, sdk_spdxid, sdk_packages, Path(d.getVar('SPDXSDKWORK'))) 784 785def combine_spdx(d, rootfs_name, rootfs_deploydir, rootfs_spdxid, packages, spdx_workdir): 786 import os 787 import oe.spdx 788 import oe.sbom 789 import oe.spdx_common 790 import io 791 import json 792 from datetime import timezone, datetime 793 from pathlib import Path 794 import tarfile 795 import bb.compress.zstd 796 797 license_data = oe.spdx_common.load_spdx_license_data(d) 798 799 providers = oe.spdx_common.collect_package_providers(d) 800 package_archs = d.getVar("SPDX_MULTILIB_SSTATE_ARCHS").split() 801 package_archs.reverse() 802 803 creation_time = datetime.now(tz=timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ") 804 deploy_dir_spdx = Path(d.getVar("DEPLOY_DIR_SPDX")) 805 source_date_epoch = d.getVar("SOURCE_DATE_EPOCH") 806 807 doc = oe.spdx.SPDXDocument() 808 doc.name = rootfs_name 809 doc.documentNamespace = get_namespace(d, doc.name) 810 doc.creationInfo.created = creation_time 811 doc.creationInfo.comment = "This document was created by analyzing the source of the Yocto recipe during the build." 812 doc.creationInfo.licenseListVersion = get_license_list_version(license_data, d) 813 doc.creationInfo.creators.append("Tool: OpenEmbedded Core create-spdx.bbclass") 814 doc.creationInfo.creators.append("Organization: %s" % d.getVar("SPDX_ORG")) 815 doc.creationInfo.creators.append("Person: N/A ()") 816 817 image = oe.spdx.SPDXPackage() 818 image.name = d.getVar("PN") 819 image.versionInfo = d.getVar("PV") 820 image.SPDXID = rootfs_spdxid 821 image.supplier = d.getVar("SPDX_SUPPLIER") 822 823 doc.packages.append(image) 824 825 if packages: 826 for name in sorted(packages.keys()): 827 if name not in providers: 828 bb.fatal("Unable to find SPDX provider for '%s'" % name) 829 830 pkg_name, pkg_hashfn = providers[name] 831 832 pkg_spdx_path = oe.sbom.doc_find_by_hashfn(deploy_dir_spdx, package_archs, pkg_name, pkg_hashfn) 833 if not pkg_spdx_path: 834 bb.fatal("No SPDX file found for package %s, %s" % (pkg_name, pkg_hashfn)) 835 836 pkg_doc, pkg_doc_sha1 = oe.sbom.read_doc(pkg_spdx_path) 837 838 for p in pkg_doc.packages: 839 if p.name == name: 840 pkg_ref = oe.spdx.SPDXExternalDocumentRef() 841 pkg_ref.externalDocumentId = "DocumentRef-%s" % pkg_doc.name 842 pkg_ref.spdxDocument = pkg_doc.documentNamespace 843 pkg_ref.checksum.algorithm = "SHA1" 844 pkg_ref.checksum.checksumValue = pkg_doc_sha1 845 846 doc.externalDocumentRefs.append(pkg_ref) 847 doc.add_relationship(image, "CONTAINS", "%s:%s" % (pkg_ref.externalDocumentId, p.SPDXID)) 848 break 849 else: 850 bb.fatal("Unable to find package with name '%s' in SPDX file %s" % (name, pkg_spdx_path)) 851 852 runtime_spdx_path = oe.sbom.doc_find_by_hashfn(deploy_dir_spdx, package_archs, "runtime-" + name, pkg_hashfn) 853 if not runtime_spdx_path: 854 bb.fatal("No runtime SPDX document found for %s, %s" % (name, pkg_hashfn)) 855 856 runtime_doc, runtime_doc_sha1 = oe.sbom.read_doc(runtime_spdx_path) 857 858 runtime_ref = oe.spdx.SPDXExternalDocumentRef() 859 runtime_ref.externalDocumentId = "DocumentRef-%s" % runtime_doc.name 860 runtime_ref.spdxDocument = runtime_doc.documentNamespace 861 runtime_ref.checksum.algorithm = "SHA1" 862 runtime_ref.checksum.checksumValue = runtime_doc_sha1 863 864 # "OTHER" isn't ideal here, but I can't find a relationship that makes sense 865 doc.externalDocumentRefs.append(runtime_ref) 866 doc.add_relationship( 867 image, 868 "OTHER", 869 "%s:%s" % (runtime_ref.externalDocumentId, runtime_doc.SPDXID), 870 comment="Runtime dependencies for %s" % name 871 ) 872 bb.utils.mkdirhier(spdx_workdir) 873 image_spdx_path = spdx_workdir / (rootfs_name + ".spdx.json") 874 875 with image_spdx_path.open("wb") as f: 876 doc.to_json(f, sort_keys=True, indent=get_json_indent(d)) 877 878 num_threads = int(d.getVar("BB_NUMBER_THREADS")) 879 880 visited_docs = set() 881 882 index = {"documents": []} 883 884 spdx_tar_path = rootfs_deploydir / (rootfs_name + ".spdx.tar.zst") 885 with bb.compress.zstd.open(spdx_tar_path, "w", num_threads=num_threads) as f: 886 with tarfile.open(fileobj=f, mode="w|") as tar: 887 def collect_spdx_document(path): 888 nonlocal tar 889 nonlocal deploy_dir_spdx 890 nonlocal source_date_epoch 891 nonlocal index 892 893 if path in visited_docs: 894 return 895 896 visited_docs.add(path) 897 898 with path.open("rb") as f: 899 doc, sha1 = oe.sbom.read_doc(f) 900 f.seek(0) 901 902 if doc.documentNamespace in visited_docs: 903 return 904 905 bb.note("Adding SPDX document %s" % path) 906 visited_docs.add(doc.documentNamespace) 907 info = tar.gettarinfo(fileobj=f) 908 909 info.name = doc.name + ".spdx.json" 910 info.uid = 0 911 info.gid = 0 912 info.uname = "root" 913 info.gname = "root" 914 915 if source_date_epoch is not None and info.mtime > int(source_date_epoch): 916 info.mtime = int(source_date_epoch) 917 918 tar.addfile(info, f) 919 920 index["documents"].append({ 921 "filename": info.name, 922 "documentNamespace": doc.documentNamespace, 923 "sha1": sha1, 924 }) 925 926 for ref in doc.externalDocumentRefs: 927 ref_path = oe.sbom.doc_find_by_namespace(deploy_dir_spdx, package_archs, ref.spdxDocument) 928 if not ref_path: 929 bb.fatal("Cannot find any SPDX file for document %s" % ref.spdxDocument) 930 collect_spdx_document(ref_path) 931 932 collect_spdx_document(image_spdx_path) 933 934 index["documents"].sort(key=lambda x: x["filename"]) 935 936 index_str = io.BytesIO(json.dumps( 937 index, 938 sort_keys=True, 939 indent=get_json_indent(d), 940 ).encode("utf-8")) 941 942 info = tarfile.TarInfo() 943 info.name = "index.json" 944 info.size = len(index_str.getvalue()) 945 info.uid = 0 946 info.gid = 0 947 info.uname = "root" 948 info.gname = "root" 949 950 tar.addfile(info, fileobj=index_str) 951 952combine_spdx[vardepsexclude] += "BB_NUMBER_THREADS SPDX_MULTILIB_SSTATE_ARCHS" 953