1# 2# Copyright OpenEmbedded Contributors 3# 4# SPDX-License-Identifier: GPL-2.0-only 5# 6 7inherit spdx-common 8 9SPDX_VERSION = "2.2" 10 11SPDX_ORG ??= "OpenEmbedded ()" 12SPDX_SUPPLIER ??= "Organization: ${SPDX_ORG}" 13SPDX_SUPPLIER[doc] = "The SPDX PackageSupplier field for SPDX packages created from \ 14 this recipe. For SPDX documents create using this class during the build, this \ 15 is the contact information for the person or organization who is doing the \ 16 build." 17 18 19def get_namespace(d, name): 20 import uuid 21 namespace_uuid = uuid.uuid5(uuid.NAMESPACE_DNS, d.getVar("SPDX_UUID_NAMESPACE")) 22 return "%s/%s-%s" % (d.getVar("SPDX_NAMESPACE_PREFIX"), name, str(uuid.uuid5(namespace_uuid, name))) 23 24 25def create_annotation(d, comment): 26 from datetime import datetime, timezone 27 28 creation_time = datetime.now(tz=timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ") 29 annotation = oe.spdx.SPDXAnnotation() 30 annotation.annotationDate = creation_time 31 annotation.annotationType = "OTHER" 32 annotation.annotator = "Tool: %s - %s" % (d.getVar("SPDX_TOOL_NAME"), d.getVar("SPDX_TOOL_VERSION")) 33 annotation.comment = comment 34 return annotation 35 36def recipe_spdx_is_native(d, recipe): 37 return any(a.annotationType == "OTHER" and 38 a.annotator == "Tool: %s - %s" % (d.getVar("SPDX_TOOL_NAME"), d.getVar("SPDX_TOOL_VERSION")) and 39 a.comment == "isNative" for a in recipe.annotations) 40 41def get_json_indent(d): 42 if d.getVar("SPDX_PRETTY") == "1": 43 return 2 44 return None 45 46 47def convert_license_to_spdx(lic, license_data, document, d, existing={}): 48 from pathlib import Path 49 import oe.spdx 50 51 extracted = {} 52 53 def add_extracted_license(ident, name): 54 nonlocal document 55 56 if name in extracted: 57 return 58 59 extracted_info = oe.spdx.SPDXExtractedLicensingInfo() 60 extracted_info.name = name 61 extracted_info.licenseId = ident 62 extracted_info.extractedText = None 63 64 if name == "PD": 65 # Special-case this. 66 extracted_info.extractedText = "Software released to the public domain" 67 else: 68 # Seach for the license in COMMON_LICENSE_DIR and LICENSE_PATH 69 for directory in [d.getVar('COMMON_LICENSE_DIR')] + (d.getVar('LICENSE_PATH') or '').split(): 70 try: 71 with (Path(directory) / name).open(errors="replace") as f: 72 extracted_info.extractedText = f.read() 73 break 74 except FileNotFoundError: 75 pass 76 if extracted_info.extractedText is None: 77 # If it's not SPDX or PD, then NO_GENERIC_LICENSE must be set 78 entry = d.getVarFlag('NO_GENERIC_LICENSE', name).split(';') 79 filename = entry[0] 80 params = {i.split('=')[0]: i.split('=')[1] for i in entry[1:] if '=' in i} 81 beginline = int(params.get('beginline', 1)) 82 endline = params.get('endline', None) 83 if endline: 84 endline = int(endline) 85 if filename: 86 filename = d.expand("${S}/" + filename) 87 with open(filename, errors="replace") as f: 88 extracted_info.extractedText = "".join(line for idx, line in enumerate(f, 1) if beginline <= idx and idx <= (endline or idx)) 89 else: 90 bb.fatal("Cannot find any text for license %s" % name) 91 92 extracted[name] = extracted_info 93 document.hasExtractedLicensingInfos.append(extracted_info) 94 95 def convert(l): 96 if l == "(" or l == ")": 97 return l 98 99 if l == "&": 100 return "AND" 101 102 if l == "|": 103 return "OR" 104 105 if l == "CLOSED": 106 return "NONE" 107 108 spdx_license = d.getVarFlag("SPDXLICENSEMAP", l) or l 109 if spdx_license in license_data["licenses"]: 110 return spdx_license 111 112 try: 113 spdx_license = existing[l] 114 except KeyError: 115 spdx_license = "LicenseRef-" + l 116 add_extracted_license(spdx_license, l) 117 118 return spdx_license 119 120 lic_split = lic.replace("(", " ( ").replace(")", " ) ").replace("|", " | ").replace("&", " & ").split() 121 122 return ' '.join(convert(l) for l in lic_split) 123 124def add_package_files(d, doc, spdx_pkg, topdir, get_spdxid, get_types, *, archive=None, ignore_dirs=[], ignore_top_level_dirs=[]): 125 from pathlib import Path 126 import oe.spdx 127 import oe.spdx_common 128 import hashlib 129 130 source_date_epoch = d.getVar("SOURCE_DATE_EPOCH") 131 if source_date_epoch: 132 source_date_epoch = int(source_date_epoch) 133 134 sha1s = [] 135 spdx_files = [] 136 137 file_counter = 1 138 for subdir, dirs, files in os.walk(topdir): 139 dirs[:] = [d for d in dirs if d not in ignore_dirs] 140 if subdir == str(topdir): 141 dirs[:] = [d for d in dirs if d not in ignore_top_level_dirs] 142 143 for file in files: 144 filepath = Path(subdir) / file 145 filename = str(filepath.relative_to(topdir)) 146 147 if not filepath.is_symlink() and filepath.is_file(): 148 spdx_file = oe.spdx.SPDXFile() 149 spdx_file.SPDXID = get_spdxid(file_counter) 150 for t in get_types(filepath): 151 spdx_file.fileTypes.append(t) 152 spdx_file.fileName = filename 153 154 if archive is not None: 155 with filepath.open("rb") as f: 156 info = archive.gettarinfo(fileobj=f) 157 info.name = filename 158 info.uid = 0 159 info.gid = 0 160 info.uname = "root" 161 info.gname = "root" 162 163 if source_date_epoch is not None and info.mtime > source_date_epoch: 164 info.mtime = source_date_epoch 165 166 archive.addfile(info, f) 167 168 sha1 = bb.utils.sha1_file(filepath) 169 sha1s.append(sha1) 170 spdx_file.checksums.append(oe.spdx.SPDXChecksum( 171 algorithm="SHA1", 172 checksumValue=sha1, 173 )) 174 spdx_file.checksums.append(oe.spdx.SPDXChecksum( 175 algorithm="SHA256", 176 checksumValue=bb.utils.sha256_file(filepath), 177 )) 178 179 if "SOURCE" in spdx_file.fileTypes: 180 extracted_lics = oe.spdx_common.extract_licenses(filepath) 181 if extracted_lics: 182 spdx_file.licenseInfoInFiles = extracted_lics 183 184 doc.files.append(spdx_file) 185 doc.add_relationship(spdx_pkg, "CONTAINS", spdx_file) 186 spdx_pkg.hasFiles.append(spdx_file.SPDXID) 187 188 spdx_files.append(spdx_file) 189 190 file_counter += 1 191 192 sha1s.sort() 193 verifier = hashlib.sha1() 194 for v in sha1s: 195 verifier.update(v.encode("utf-8")) 196 spdx_pkg.packageVerificationCode.packageVerificationCodeValue = verifier.hexdigest() 197 198 return spdx_files 199 200 201def add_package_sources_from_debug(d, package_doc, spdx_package, package, package_files, sources): 202 from pathlib import Path 203 import hashlib 204 import oe.packagedata 205 import oe.spdx 206 207 debug_search_paths = [ 208 Path(d.getVar('PKGD')), 209 Path(d.getVar('STAGING_DIR_TARGET')), 210 Path(d.getVar('STAGING_DIR_NATIVE')), 211 Path(d.getVar('STAGING_KERNEL_DIR')), 212 ] 213 214 pkg_data = oe.packagedata.read_subpkgdata_extended(package, d) 215 216 if pkg_data is None: 217 return 218 219 for file_path, file_data in pkg_data["files_info"].items(): 220 if not "debugsrc" in file_data: 221 continue 222 223 for pkg_file in package_files: 224 if file_path.lstrip("/") == pkg_file.fileName.lstrip("/"): 225 break 226 else: 227 bb.fatal("No package file found for %s in %s; SPDX found: %s" % (str(file_path), package, 228 " ".join(p.fileName for p in package_files))) 229 continue 230 231 for debugsrc in file_data["debugsrc"]: 232 ref_id = "NOASSERTION" 233 for search in debug_search_paths: 234 if debugsrc.startswith("/usr/src/kernel"): 235 debugsrc_path = search / debugsrc.replace('/usr/src/kernel/', '') 236 else: 237 debugsrc_path = search / debugsrc.lstrip("/") 238 # We can only hash files below, skip directories, links, etc. 239 if not os.path.isfile(debugsrc_path): 240 continue 241 242 file_sha256 = bb.utils.sha256_file(debugsrc_path) 243 244 if file_sha256 in sources: 245 source_file = sources[file_sha256] 246 247 doc_ref = package_doc.find_external_document_ref(source_file.doc.documentNamespace) 248 if doc_ref is None: 249 doc_ref = oe.spdx.SPDXExternalDocumentRef() 250 doc_ref.externalDocumentId = "DocumentRef-dependency-" + source_file.doc.name 251 doc_ref.spdxDocument = source_file.doc.documentNamespace 252 doc_ref.checksum.algorithm = "SHA1" 253 doc_ref.checksum.checksumValue = source_file.doc_sha1 254 package_doc.externalDocumentRefs.append(doc_ref) 255 256 ref_id = "%s:%s" % (doc_ref.externalDocumentId, source_file.file.SPDXID) 257 else: 258 bb.debug(1, "Debug source %s with SHA256 %s not found in any dependency" % (str(debugsrc_path), file_sha256)) 259 break 260 else: 261 bb.debug(1, "Debug source %s not found" % debugsrc) 262 263 package_doc.add_relationship(pkg_file, "GENERATED_FROM", ref_id, comment=debugsrc) 264 265add_package_sources_from_debug[vardepsexclude] += "STAGING_KERNEL_DIR" 266 267def collect_dep_recipes(d, doc, spdx_recipe): 268 import json 269 from pathlib import Path 270 import oe.sbom 271 import oe.spdx 272 import oe.spdx_common 273 274 deploy_dir_spdx = Path(d.getVar("DEPLOY_DIR_SPDX")) 275 package_archs = d.getVar("SPDX_MULTILIB_SSTATE_ARCHS").split() 276 package_archs.reverse() 277 278 dep_recipes = [] 279 280 deps = oe.spdx_common.get_spdx_deps(d) 281 282 for dep_pn, dep_hashfn, in_taskhash in deps: 283 # If this dependency is not calculated in the taskhash skip it. 284 # Otherwise, it can result in broken links since this task won't 285 # rebuild and see the new SPDX ID if the dependency changes 286 if not in_taskhash: 287 continue 288 289 dep_recipe_path = oe.sbom.doc_find_by_hashfn(deploy_dir_spdx, package_archs, "recipe-" + dep_pn, dep_hashfn) 290 if not dep_recipe_path: 291 bb.fatal("Cannot find any SPDX file for recipe %s, %s" % (dep_pn, dep_hashfn)) 292 293 spdx_dep_doc, spdx_dep_sha1 = oe.sbom.read_doc(dep_recipe_path) 294 295 for pkg in spdx_dep_doc.packages: 296 if pkg.name == dep_pn: 297 spdx_dep_recipe = pkg 298 break 299 else: 300 continue 301 302 dep_recipes.append(oe.sbom.DepRecipe(spdx_dep_doc, spdx_dep_sha1, spdx_dep_recipe)) 303 304 dep_recipe_ref = oe.spdx.SPDXExternalDocumentRef() 305 dep_recipe_ref.externalDocumentId = "DocumentRef-dependency-" + spdx_dep_doc.name 306 dep_recipe_ref.spdxDocument = spdx_dep_doc.documentNamespace 307 dep_recipe_ref.checksum.algorithm = "SHA1" 308 dep_recipe_ref.checksum.checksumValue = spdx_dep_sha1 309 310 doc.externalDocumentRefs.append(dep_recipe_ref) 311 312 doc.add_relationship( 313 "%s:%s" % (dep_recipe_ref.externalDocumentId, spdx_dep_recipe.SPDXID), 314 "BUILD_DEPENDENCY_OF", 315 spdx_recipe 316 ) 317 318 return dep_recipes 319 320collect_dep_recipes[vardepsexclude] = "SPDX_MULTILIB_SSTATE_ARCHS" 321 322def collect_dep_sources(d, dep_recipes): 323 import oe.sbom 324 325 sources = {} 326 for dep in dep_recipes: 327 # Don't collect sources from native recipes as they 328 # match non-native sources also. 329 if recipe_spdx_is_native(d, dep.recipe): 330 continue 331 recipe_files = set(dep.recipe.hasFiles) 332 333 for spdx_file in dep.doc.files: 334 if spdx_file.SPDXID not in recipe_files: 335 continue 336 337 if "SOURCE" in spdx_file.fileTypes: 338 for checksum in spdx_file.checksums: 339 if checksum.algorithm == "SHA256": 340 sources[checksum.checksumValue] = oe.sbom.DepSource(dep.doc, dep.doc_sha1, dep.recipe, spdx_file) 341 break 342 343 return sources 344 345def add_download_packages(d, doc, recipe): 346 import os.path 347 from bb.fetch2 import decodeurl, CHECKSUM_LIST 348 import bb.process 349 import oe.spdx 350 import oe.sbom 351 352 for download_idx, src_uri in enumerate(d.getVar('SRC_URI').split()): 353 f = bb.fetch2.FetchData(src_uri, d) 354 355 for name in f.names: 356 package = oe.spdx.SPDXPackage() 357 package.name = "%s-source-%d" % (d.getVar("PN"), download_idx + 1) 358 package.SPDXID = oe.sbom.get_download_spdxid(d, download_idx + 1) 359 360 if f.type == "file": 361 continue 362 363 if f.method.supports_checksum(f): 364 for checksum_id in CHECKSUM_LIST: 365 if checksum_id.upper() not in oe.spdx.SPDXPackage.ALLOWED_CHECKSUMS: 366 continue 367 368 expected_checksum = getattr(f, "%s_expected" % checksum_id) 369 if expected_checksum is None: 370 continue 371 372 c = oe.spdx.SPDXChecksum() 373 c.algorithm = checksum_id.upper() 374 c.checksumValue = expected_checksum 375 package.checksums.append(c) 376 377 package.downloadLocation = oe.spdx_common.fetch_data_to_uri(f, name) 378 doc.packages.append(package) 379 doc.add_relationship(doc, "DESCRIBES", package) 380 # In the future, we might be able to do more fancy dependencies, 381 # but this should be sufficient for now 382 doc.add_relationship(package, "BUILD_DEPENDENCY_OF", recipe) 383 384def get_license_list_version(license_data, d): 385 # Newer versions of the SPDX license list are SemVer ("MAJOR.MINOR.MICRO"), 386 # but SPDX 2 only uses "MAJOR.MINOR". 387 return ".".join(license_data["licenseListVersion"].split(".")[:2]) 388 389 390python do_create_spdx() { 391 from datetime import datetime, timezone 392 import oe.sbom 393 import oe.spdx 394 import oe.spdx_common 395 import uuid 396 from pathlib import Path 397 from contextlib import contextmanager 398 import oe.cve_check 399 400 license_data = oe.spdx_common.load_spdx_license_data(d) 401 402 @contextmanager 403 def optional_tarfile(name, guard, mode="w"): 404 import tarfile 405 import bb.compress.zstd 406 407 num_threads = int(d.getVar("BB_NUMBER_THREADS")) 408 409 if guard: 410 name.parent.mkdir(parents=True, exist_ok=True) 411 with bb.compress.zstd.open(name, mode=mode + "b", num_threads=num_threads) as f: 412 with tarfile.open(fileobj=f, mode=mode + "|") as tf: 413 yield tf 414 else: 415 yield None 416 417 418 deploy_dir_spdx = Path(d.getVar("DEPLOY_DIR_SPDX")) 419 spdx_workdir = Path(d.getVar("SPDXWORK")) 420 include_sources = d.getVar("SPDX_INCLUDE_SOURCES") == "1" 421 archive_sources = d.getVar("SPDX_ARCHIVE_SOURCES") == "1" 422 archive_packaged = d.getVar("SPDX_ARCHIVE_PACKAGED") == "1" 423 pkg_arch = d.getVar("SSTATE_PKGARCH") 424 425 creation_time = datetime.now(tz=timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ") 426 427 doc = oe.spdx.SPDXDocument() 428 429 doc.name = "recipe-" + d.getVar("PN") 430 doc.documentNamespace = get_namespace(d, doc.name) 431 doc.creationInfo.created = creation_time 432 doc.creationInfo.comment = "This document was created by analyzing recipe files during the build." 433 doc.creationInfo.licenseListVersion = get_license_list_version(license_data, d) 434 doc.creationInfo.creators.append("Tool: OpenEmbedded Core create-spdx.bbclass") 435 doc.creationInfo.creators.append("Organization: %s" % d.getVar("SPDX_ORG")) 436 doc.creationInfo.creators.append("Person: N/A ()") 437 438 recipe = oe.spdx.SPDXPackage() 439 recipe.name = d.getVar("PN") 440 recipe.versionInfo = d.getVar("PV") 441 recipe.SPDXID = oe.sbom.get_recipe_spdxid(d) 442 recipe.supplier = d.getVar("SPDX_SUPPLIER") 443 if bb.data.inherits_class("native", d) or bb.data.inherits_class("cross", d): 444 recipe.annotations.append(create_annotation(d, "isNative")) 445 446 homepage = d.getVar("HOMEPAGE") 447 if homepage: 448 recipe.homepage = homepage 449 450 license = d.getVar("LICENSE") 451 if license: 452 recipe.licenseDeclared = convert_license_to_spdx(license, license_data, doc, d) 453 454 summary = d.getVar("SUMMARY") 455 if summary: 456 recipe.summary = summary 457 458 description = d.getVar("DESCRIPTION") 459 if description: 460 recipe.description = description 461 462 if d.getVar("SPDX_CUSTOM_ANNOTATION_VARS"): 463 for var in d.getVar('SPDX_CUSTOM_ANNOTATION_VARS').split(): 464 recipe.annotations.append(create_annotation(d, var + "=" + d.getVar(var))) 465 466 # Some CVEs may be patched during the build process without incrementing the version number, 467 # so querying for CVEs based on the CPE id can lead to false positives. To account for this, 468 # save the CVEs fixed by patches to source information field in the SPDX. 469 patched_cves = oe.cve_check.get_patched_cves(d) 470 patched_cves = list(patched_cves) 471 patched_cves = ' '.join(patched_cves) 472 if patched_cves: 473 recipe.sourceInfo = "CVEs fixed: " + patched_cves 474 475 cpe_ids = oe.cve_check.get_cpe_ids(d.getVar("CVE_PRODUCT"), d.getVar("CVE_VERSION")) 476 if cpe_ids: 477 for cpe_id in cpe_ids: 478 cpe = oe.spdx.SPDXExternalReference() 479 cpe.referenceCategory = "SECURITY" 480 cpe.referenceType = "http://spdx.org/rdf/references/cpe23Type" 481 cpe.referenceLocator = cpe_id 482 recipe.externalRefs.append(cpe) 483 484 doc.packages.append(recipe) 485 doc.add_relationship(doc, "DESCRIBES", recipe) 486 487 add_download_packages(d, doc, recipe) 488 489 if oe.spdx_common.process_sources(d) and include_sources: 490 recipe_archive = deploy_dir_spdx / "recipes" / (doc.name + ".tar.zst") 491 with optional_tarfile(recipe_archive, archive_sources) as archive: 492 oe.spdx_common.get_patched_src(d) 493 494 add_package_files( 495 d, 496 doc, 497 recipe, 498 spdx_workdir, 499 lambda file_counter: "SPDXRef-SourceFile-%s-%d" % (d.getVar("PN"), file_counter), 500 lambda filepath: ["SOURCE"], 501 ignore_dirs=[".git"], 502 ignore_top_level_dirs=["temp"], 503 archive=archive, 504 ) 505 506 if archive is not None: 507 recipe.packageFileName = str(recipe_archive.name) 508 509 dep_recipes = collect_dep_recipes(d, doc, recipe) 510 511 doc_sha1 = oe.sbom.write_doc(d, doc, pkg_arch, "recipes", indent=get_json_indent(d)) 512 dep_recipes.append(oe.sbom.DepRecipe(doc, doc_sha1, recipe)) 513 514 recipe_ref = oe.spdx.SPDXExternalDocumentRef() 515 recipe_ref.externalDocumentId = "DocumentRef-recipe-" + recipe.name 516 recipe_ref.spdxDocument = doc.documentNamespace 517 recipe_ref.checksum.algorithm = "SHA1" 518 recipe_ref.checksum.checksumValue = doc_sha1 519 520 sources = collect_dep_sources(d, dep_recipes) 521 found_licenses = {license.name:recipe_ref.externalDocumentId + ":" + license.licenseId for license in doc.hasExtractedLicensingInfos} 522 523 if not recipe_spdx_is_native(d, recipe): 524 bb.build.exec_func("read_subpackage_metadata", d) 525 526 pkgdest = Path(d.getVar("PKGDEST")) 527 for package in d.getVar("PACKAGES").split(): 528 if not oe.packagedata.packaged(package, d): 529 continue 530 531 package_doc = oe.spdx.SPDXDocument() 532 pkg_name = d.getVar("PKG:%s" % package) or package 533 package_doc.name = pkg_name 534 package_doc.documentNamespace = get_namespace(d, package_doc.name) 535 package_doc.creationInfo.created = creation_time 536 package_doc.creationInfo.comment = "This document was created by analyzing packages created during the build." 537 package_doc.creationInfo.licenseListVersion = get_license_list_version(license_data, d) 538 package_doc.creationInfo.creators.append("Tool: OpenEmbedded Core create-spdx.bbclass") 539 package_doc.creationInfo.creators.append("Organization: %s" % d.getVar("SPDX_ORG")) 540 package_doc.creationInfo.creators.append("Person: N/A ()") 541 package_doc.externalDocumentRefs.append(recipe_ref) 542 543 package_license = d.getVar("LICENSE:%s" % package) or d.getVar("LICENSE") 544 545 spdx_package = oe.spdx.SPDXPackage() 546 547 spdx_package.SPDXID = oe.sbom.get_package_spdxid(pkg_name) 548 spdx_package.name = pkg_name 549 spdx_package.versionInfo = d.getVar("PV") 550 spdx_package.licenseDeclared = convert_license_to_spdx(package_license, license_data, package_doc, d, found_licenses) 551 spdx_package.supplier = d.getVar("SPDX_SUPPLIER") 552 553 package_doc.packages.append(spdx_package) 554 555 package_doc.add_relationship(spdx_package, "GENERATED_FROM", "%s:%s" % (recipe_ref.externalDocumentId, recipe.SPDXID)) 556 package_doc.add_relationship(package_doc, "DESCRIBES", spdx_package) 557 558 package_archive = deploy_dir_spdx / "packages" / (package_doc.name + ".tar.zst") 559 with optional_tarfile(package_archive, archive_packaged) as archive: 560 package_files = add_package_files( 561 d, 562 package_doc, 563 spdx_package, 564 pkgdest / package, 565 lambda file_counter: oe.sbom.get_packaged_file_spdxid(pkg_name, file_counter), 566 lambda filepath: ["BINARY"], 567 ignore_top_level_dirs=['CONTROL', 'DEBIAN'], 568 archive=archive, 569 ) 570 571 if archive is not None: 572 spdx_package.packageFileName = str(package_archive.name) 573 574 add_package_sources_from_debug(d, package_doc, spdx_package, package, package_files, sources) 575 576 oe.sbom.write_doc(d, package_doc, pkg_arch, "packages", indent=get_json_indent(d)) 577} 578do_create_spdx[vardepsexclude] += "BB_NUMBER_THREADS" 579# NOTE: depending on do_unpack is a hack that is necessary to get it's dependencies for archive the source 580addtask do_create_spdx after do_package do_packagedata do_unpack do_collect_spdx_deps before do_populate_sdk do_build do_rm_work 581 582SSTATETASKS += "do_create_spdx" 583do_create_spdx[sstate-inputdirs] = "${SPDXDEPLOY}" 584do_create_spdx[sstate-outputdirs] = "${DEPLOY_DIR_SPDX}" 585 586python do_create_spdx_setscene () { 587 sstate_setscene(d) 588} 589addtask do_create_spdx_setscene 590 591do_create_spdx[dirs] = "${SPDXWORK}" 592do_create_spdx[cleandirs] = "${SPDXDEPLOY} ${SPDXWORK}" 593do_create_spdx[depends] += " \ 594 ${PATCHDEPENDENCY} \ 595 ${@create_spdx_source_deps(d)} \ 596" 597 598python do_create_runtime_spdx() { 599 from datetime import datetime, timezone 600 import oe.sbom 601 import oe.spdx 602 import oe.spdx_common 603 import oe.packagedata 604 from pathlib import Path 605 606 deploy_dir_spdx = Path(d.getVar("DEPLOY_DIR_SPDX")) 607 spdx_deploy = Path(d.getVar("SPDXRUNTIMEDEPLOY")) 608 is_native = bb.data.inherits_class("native", d) or bb.data.inherits_class("cross", d) 609 610 creation_time = datetime.now(tz=timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ") 611 612 license_data = oe.spdx_common.load_spdx_license_data(d) 613 614 providers = oe.spdx_common.collect_package_providers(d) 615 pkg_arch = d.getVar("SSTATE_PKGARCH") 616 package_archs = d.getVar("SPDX_MULTILIB_SSTATE_ARCHS").split() 617 package_archs.reverse() 618 619 if not is_native: 620 bb.build.exec_func("read_subpackage_metadata", d) 621 622 dep_package_cache = {} 623 624 pkgdest = Path(d.getVar("PKGDEST")) 625 for package in d.getVar("PACKAGES").split(): 626 localdata = bb.data.createCopy(d) 627 pkg_name = d.getVar("PKG:%s" % package) or package 628 localdata.setVar("PKG", pkg_name) 629 localdata.setVar('OVERRIDES', d.getVar("OVERRIDES", False) + ":" + package) 630 631 if not oe.packagedata.packaged(package, localdata): 632 continue 633 634 pkg_spdx_path = oe.sbom.doc_path(deploy_dir_spdx, pkg_name, pkg_arch, "packages") 635 636 package_doc, package_doc_sha1 = oe.sbom.read_doc(pkg_spdx_path) 637 638 for p in package_doc.packages: 639 if p.name == pkg_name: 640 spdx_package = p 641 break 642 else: 643 bb.fatal("Package '%s' not found in %s" % (pkg_name, pkg_spdx_path)) 644 645 runtime_doc = oe.spdx.SPDXDocument() 646 runtime_doc.name = "runtime-" + pkg_name 647 runtime_doc.documentNamespace = get_namespace(localdata, runtime_doc.name) 648 runtime_doc.creationInfo.created = creation_time 649 runtime_doc.creationInfo.comment = "This document was created by analyzing package runtime dependencies." 650 runtime_doc.creationInfo.licenseListVersion = get_license_list_version(license_data, d) 651 runtime_doc.creationInfo.creators.append("Tool: OpenEmbedded Core create-spdx.bbclass") 652 runtime_doc.creationInfo.creators.append("Organization: %s" % d.getVar("SPDX_ORG")) 653 runtime_doc.creationInfo.creators.append("Person: N/A ()") 654 655 package_ref = oe.spdx.SPDXExternalDocumentRef() 656 package_ref.externalDocumentId = "DocumentRef-package-" + package 657 package_ref.spdxDocument = package_doc.documentNamespace 658 package_ref.checksum.algorithm = "SHA1" 659 package_ref.checksum.checksumValue = package_doc_sha1 660 661 runtime_doc.externalDocumentRefs.append(package_ref) 662 663 runtime_doc.add_relationship( 664 runtime_doc.SPDXID, 665 "AMENDS", 666 "%s:%s" % (package_ref.externalDocumentId, package_doc.SPDXID) 667 ) 668 669 deps = bb.utils.explode_dep_versions2(localdata.getVar("RDEPENDS") or "") 670 seen_deps = set() 671 for dep, _ in deps.items(): 672 if dep in seen_deps: 673 continue 674 675 if dep not in providers: 676 continue 677 678 (dep, dep_hashfn) = providers[dep] 679 680 if not oe.packagedata.packaged(dep, localdata): 681 continue 682 683 dep_pkg_data = oe.packagedata.read_subpkgdata_dict(dep, d) 684 dep_pkg = dep_pkg_data["PKG"] 685 686 if dep in dep_package_cache: 687 (dep_spdx_package, dep_package_ref) = dep_package_cache[dep] 688 else: 689 dep_path = oe.sbom.doc_find_by_hashfn(deploy_dir_spdx, package_archs, dep_pkg, dep_hashfn) 690 if not dep_path: 691 bb.fatal("No SPDX file found for package %s, %s" % (dep_pkg, dep_hashfn)) 692 693 spdx_dep_doc, spdx_dep_sha1 = oe.sbom.read_doc(dep_path) 694 695 for pkg in spdx_dep_doc.packages: 696 if pkg.name == dep_pkg: 697 dep_spdx_package = pkg 698 break 699 else: 700 bb.fatal("Package '%s' not found in %s" % (dep_pkg, dep_path)) 701 702 dep_package_ref = oe.spdx.SPDXExternalDocumentRef() 703 dep_package_ref.externalDocumentId = "DocumentRef-runtime-dependency-" + spdx_dep_doc.name 704 dep_package_ref.spdxDocument = spdx_dep_doc.documentNamespace 705 dep_package_ref.checksum.algorithm = "SHA1" 706 dep_package_ref.checksum.checksumValue = spdx_dep_sha1 707 708 dep_package_cache[dep] = (dep_spdx_package, dep_package_ref) 709 710 runtime_doc.externalDocumentRefs.append(dep_package_ref) 711 712 runtime_doc.add_relationship( 713 "%s:%s" % (dep_package_ref.externalDocumentId, dep_spdx_package.SPDXID), 714 "RUNTIME_DEPENDENCY_OF", 715 "%s:%s" % (package_ref.externalDocumentId, spdx_package.SPDXID) 716 ) 717 seen_deps.add(dep) 718 719 oe.sbom.write_doc(d, runtime_doc, pkg_arch, "runtime", spdx_deploy, indent=get_json_indent(d)) 720} 721 722do_create_runtime_spdx[vardepsexclude] += "OVERRIDES SPDX_MULTILIB_SSTATE_ARCHS" 723 724addtask do_create_runtime_spdx after do_create_spdx before do_build do_rm_work 725SSTATETASKS += "do_create_runtime_spdx" 726do_create_runtime_spdx[sstate-inputdirs] = "${SPDXRUNTIMEDEPLOY}" 727do_create_runtime_spdx[sstate-outputdirs] = "${DEPLOY_DIR_SPDX}" 728 729python do_create_runtime_spdx_setscene () { 730 sstate_setscene(d) 731} 732addtask do_create_runtime_spdx_setscene 733 734do_create_runtime_spdx[dirs] = "${SPDXRUNTIMEDEPLOY}" 735do_create_runtime_spdx[cleandirs] = "${SPDXRUNTIMEDEPLOY}" 736do_create_runtime_spdx[rdeptask] = "do_create_spdx" 737 738do_rootfs[recrdeptask] += "do_create_spdx do_create_runtime_spdx" 739do_rootfs[cleandirs] += "${SPDXIMAGEWORK}" 740 741ROOTFS_POSTUNINSTALL_COMMAND =+ "image_combine_spdx" 742 743do_populate_sdk[recrdeptask] += "do_create_spdx do_create_runtime_spdx" 744do_populate_sdk[cleandirs] += "${SPDXSDKWORK}" 745POPULATE_SDK_POST_HOST_COMMAND:append:task-populate-sdk = " sdk_host_combine_spdx" 746POPULATE_SDK_POST_TARGET_COMMAND:append:task-populate-sdk = " sdk_target_combine_spdx" 747 748python image_combine_spdx() { 749 import os 750 import oe.sbom 751 from pathlib import Path 752 from oe.rootfs import image_list_installed_packages 753 754 image_name = d.getVar("IMAGE_NAME") 755 image_link_name = d.getVar("IMAGE_LINK_NAME") 756 imgdeploydir = Path(d.getVar("IMGDEPLOYDIR")) 757 img_spdxid = oe.sbom.get_image_spdxid(image_name) 758 packages = image_list_installed_packages(d) 759 760 combine_spdx(d, image_name, imgdeploydir, img_spdxid, packages, Path(d.getVar("SPDXIMAGEWORK"))) 761 762 def make_image_link(target_path, suffix): 763 if image_link_name: 764 link = imgdeploydir / (image_link_name + suffix) 765 if link != target_path: 766 link.symlink_to(os.path.relpath(target_path, link.parent)) 767 768 spdx_tar_path = imgdeploydir / (image_name + ".spdx.tar.zst") 769 make_image_link(spdx_tar_path, ".spdx.tar.zst") 770} 771 772python sdk_host_combine_spdx() { 773 sdk_combine_spdx(d, "host") 774} 775 776python sdk_target_combine_spdx() { 777 sdk_combine_spdx(d, "target") 778} 779 780def sdk_combine_spdx(d, sdk_type): 781 import oe.sbom 782 from pathlib import Path 783 from oe.sdk import sdk_list_installed_packages 784 785 sdk_name = d.getVar("TOOLCHAIN_OUTPUTNAME") + "-" + sdk_type 786 sdk_deploydir = Path(d.getVar("SDKDEPLOYDIR")) 787 sdk_spdxid = oe.sbom.get_sdk_spdxid(sdk_name) 788 sdk_packages = sdk_list_installed_packages(d, sdk_type == "target") 789 combine_spdx(d, sdk_name, sdk_deploydir, sdk_spdxid, sdk_packages, Path(d.getVar('SPDXSDKWORK'))) 790 791def combine_spdx(d, rootfs_name, rootfs_deploydir, rootfs_spdxid, packages, spdx_workdir): 792 import os 793 import oe.spdx 794 import oe.sbom 795 import oe.spdx_common 796 import io 797 import json 798 from datetime import timezone, datetime 799 from pathlib import Path 800 import tarfile 801 import bb.compress.zstd 802 803 license_data = oe.spdx_common.load_spdx_license_data(d) 804 805 providers = oe.spdx_common.collect_package_providers(d) 806 package_archs = d.getVar("SPDX_MULTILIB_SSTATE_ARCHS").split() 807 package_archs.reverse() 808 809 creation_time = datetime.now(tz=timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ") 810 deploy_dir_spdx = Path(d.getVar("DEPLOY_DIR_SPDX")) 811 source_date_epoch = d.getVar("SOURCE_DATE_EPOCH") 812 813 doc = oe.spdx.SPDXDocument() 814 doc.name = rootfs_name 815 doc.documentNamespace = get_namespace(d, doc.name) 816 doc.creationInfo.created = creation_time 817 doc.creationInfo.comment = "This document was created by analyzing the source of the Yocto recipe during the build." 818 doc.creationInfo.licenseListVersion = get_license_list_version(license_data, d) 819 doc.creationInfo.creators.append("Tool: OpenEmbedded Core create-spdx.bbclass") 820 doc.creationInfo.creators.append("Organization: %s" % d.getVar("SPDX_ORG")) 821 doc.creationInfo.creators.append("Person: N/A ()") 822 823 image = oe.spdx.SPDXPackage() 824 image.name = d.getVar("PN") 825 image.versionInfo = d.getVar("PV") 826 image.SPDXID = rootfs_spdxid 827 image.supplier = d.getVar("SPDX_SUPPLIER") 828 829 doc.packages.append(image) 830 831 if packages: 832 for name in sorted(packages.keys()): 833 if name not in providers: 834 bb.fatal("Unable to find SPDX provider for '%s'" % name) 835 836 pkg_name, pkg_hashfn = providers[name] 837 838 pkg_spdx_path = oe.sbom.doc_find_by_hashfn(deploy_dir_spdx, package_archs, pkg_name, pkg_hashfn) 839 if not pkg_spdx_path: 840 bb.fatal("No SPDX file found for package %s, %s" % (pkg_name, pkg_hashfn)) 841 842 pkg_doc, pkg_doc_sha1 = oe.sbom.read_doc(pkg_spdx_path) 843 844 for p in pkg_doc.packages: 845 if p.name == name: 846 pkg_ref = oe.spdx.SPDXExternalDocumentRef() 847 pkg_ref.externalDocumentId = "DocumentRef-%s" % pkg_doc.name 848 pkg_ref.spdxDocument = pkg_doc.documentNamespace 849 pkg_ref.checksum.algorithm = "SHA1" 850 pkg_ref.checksum.checksumValue = pkg_doc_sha1 851 852 doc.externalDocumentRefs.append(pkg_ref) 853 doc.add_relationship(image, "CONTAINS", "%s:%s" % (pkg_ref.externalDocumentId, p.SPDXID)) 854 break 855 else: 856 bb.fatal("Unable to find package with name '%s' in SPDX file %s" % (name, pkg_spdx_path)) 857 858 runtime_spdx_path = oe.sbom.doc_find_by_hashfn(deploy_dir_spdx, package_archs, "runtime-" + name, pkg_hashfn) 859 if not runtime_spdx_path: 860 bb.fatal("No runtime SPDX document found for %s, %s" % (name, pkg_hashfn)) 861 862 runtime_doc, runtime_doc_sha1 = oe.sbom.read_doc(runtime_spdx_path) 863 864 runtime_ref = oe.spdx.SPDXExternalDocumentRef() 865 runtime_ref.externalDocumentId = "DocumentRef-%s" % runtime_doc.name 866 runtime_ref.spdxDocument = runtime_doc.documentNamespace 867 runtime_ref.checksum.algorithm = "SHA1" 868 runtime_ref.checksum.checksumValue = runtime_doc_sha1 869 870 # "OTHER" isn't ideal here, but I can't find a relationship that makes sense 871 doc.externalDocumentRefs.append(runtime_ref) 872 doc.add_relationship( 873 image, 874 "OTHER", 875 "%s:%s" % (runtime_ref.externalDocumentId, runtime_doc.SPDXID), 876 comment="Runtime dependencies for %s" % name 877 ) 878 bb.utils.mkdirhier(spdx_workdir) 879 image_spdx_path = spdx_workdir / (rootfs_name + ".spdx.json") 880 881 with image_spdx_path.open("wb") as f: 882 doc.to_json(f, sort_keys=True, indent=get_json_indent(d)) 883 884 num_threads = int(d.getVar("BB_NUMBER_THREADS")) 885 886 visited_docs = set() 887 888 index = {"documents": []} 889 890 spdx_tar_path = rootfs_deploydir / (rootfs_name + ".spdx.tar.zst") 891 with bb.compress.zstd.open(spdx_tar_path, "w", num_threads=num_threads) as f: 892 with tarfile.open(fileobj=f, mode="w|") as tar: 893 def collect_spdx_document(path): 894 nonlocal tar 895 nonlocal deploy_dir_spdx 896 nonlocal source_date_epoch 897 nonlocal index 898 899 if path in visited_docs: 900 return 901 902 visited_docs.add(path) 903 904 with path.open("rb") as f: 905 doc, sha1 = oe.sbom.read_doc(f) 906 f.seek(0) 907 908 if doc.documentNamespace in visited_docs: 909 return 910 911 bb.note("Adding SPDX document %s" % path) 912 visited_docs.add(doc.documentNamespace) 913 info = tar.gettarinfo(fileobj=f) 914 915 info.name = doc.name + ".spdx.json" 916 info.uid = 0 917 info.gid = 0 918 info.uname = "root" 919 info.gname = "root" 920 921 if source_date_epoch is not None and info.mtime > int(source_date_epoch): 922 info.mtime = int(source_date_epoch) 923 924 tar.addfile(info, f) 925 926 index["documents"].append({ 927 "filename": info.name, 928 "documentNamespace": doc.documentNamespace, 929 "sha1": sha1, 930 }) 931 932 for ref in doc.externalDocumentRefs: 933 ref_path = oe.sbom.doc_find_by_namespace(deploy_dir_spdx, package_archs, ref.spdxDocument) 934 if not ref_path: 935 bb.fatal("Cannot find any SPDX file for document %s" % ref.spdxDocument) 936 collect_spdx_document(ref_path) 937 938 collect_spdx_document(image_spdx_path) 939 940 index["documents"].sort(key=lambda x: x["filename"]) 941 942 index_str = io.BytesIO(json.dumps( 943 index, 944 sort_keys=True, 945 indent=get_json_indent(d), 946 ).encode("utf-8")) 947 948 info = tarfile.TarInfo() 949 info.name = "index.json" 950 info.size = len(index_str.getvalue()) 951 info.uid = 0 952 info.gid = 0 953 info.uname = "root" 954 info.gname = "root" 955 956 tar.addfile(info, fileobj=index_str) 957 958combine_spdx[vardepsexclude] += "BB_NUMBER_THREADS SPDX_MULTILIB_SSTATE_ARCHS" 959