Skip to content

gh-112844: Add SBOM for external dependencies #115789

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 3 commits into from
Feb 29, 2024
Merged

gh-112844: Add SBOM for external dependencies #115789

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
7c891ed
Add SBOM for external dependencies
sethmlarson Feb 21, 2024
9dee611
Add new externals.spdx.json file to CODEOWNERS
sethmlarson Feb 27, 2024
afe2517
Change parsing of git refs to handle OpenSSL-like versions
sethmlarson Feb 28, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions .github/CODEOWNERS
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -247,5 +247,6 @@ Lib/test/test_interpreters/ @ericsnowcurrently
/Tools/wasm/ @brettcannon

# SBOM
/Misc/externals.spdx.json @sethmlarson
/Misc/sbom.spdx.json @sethmlarson
/Tools/build/generate_sbom.py @sethmlarson
174 changes: 174 additions & 0 deletions Misc/externals.spdx.json
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,174 @@
{
"SPDXID": "SPDXRef-DOCUMENT",
"packages": [
{
"SPDXID": "SPDXRef-PACKAGE-bzip2",
"checksums": [
{
"algorithm": "SHA256",
"checksumValue": "ab8d1b0cc087c20d4c32c0e4fcf7d0c733a95da12cedc6d63b3f0a9af07427e2"
}
],
"downloadLocation": "https://github.com/python/cpython-source-deps/archive/refs/tags/bzip2-1.0.8.tar.gz",
"externalRefs": [
{
"referenceCategory": "SECURITY",
"referenceLocator": "cpe:2.3:a:bzip:bzip2:1.0.8:*:*:*:*:*:*:*",
"referenceType": "cpe23Type"
}
],
"licenseConcluded": "NOASSERTION",
"name": "bzip2",
"primaryPackagePurpose": "SOURCE",
"versionInfo": "1.0.8"
},
{
"SPDXID": "SPDXRef-PACKAGE-libffi",
"checksums": [
{
"algorithm": "SHA256",
"checksumValue": "9d802681adfea27d84cae0487a785fb9caa925bdad44c401b364c59ab2b8edda"
}
],
"downloadLocation": "https://github.com/python/cpython-source-deps/archive/refs/tags/libffi-3.4.4.tar.gz",
"externalRefs": [
{
"referenceCategory": "SECURITY",
"referenceLocator": "cpe:2.3:a:libffi_project:libffi:3.4.4:*:*:*:*:*:*:*",
"referenceType": "cpe23Type"
}
],
"licenseConcluded": "NOASSERTION",
"name": "libffi",
"primaryPackagePurpose": "SOURCE",
"versionInfo": "3.4.4"
},
{
"SPDXID": "SPDXRef-PACKAGE-openssl",
"checksums": [
{
"algorithm": "SHA256",
"checksumValue": "e6a77c273ebb284fedd8ea19b081fce74a9455936ffd47215f7c24713e2614b2"
}
],
"downloadLocation": "https://github.com/python/cpython-source-deps/archive/refs/tags/openssl-3.0.13.tar.gz",
"externalRefs": [
{
"referenceCategory": "SECURITY",
"referenceLocator": "cpe:2.3:a:openssl:openssl:3.0.13:*:*:*:*:*:*:*",
"referenceType": "cpe23Type"
}
],
"licenseConcluded": "NOASSERTION",
"name": "openssl",
"primaryPackagePurpose": "SOURCE",
"versionInfo": "3.0.13"
},
{
"SPDXID": "SPDXRef-PACKAGE-sqlite",
"checksums": [
{
"algorithm": "SHA256",
"checksumValue": "6f0364a27375435a34137b138ca4fedef8d23eec6493ca1dfff33bfc0c34fda4"
}
],
"downloadLocation": "https://github.com/python/cpython-source-deps/archive/refs/tags/sqlite-3.45.1.0.tar.gz",
"externalRefs": [
{
"referenceCategory": "SECURITY",
"referenceLocator": "cpe:2.3:a:sqlite:sqlite:3.45.1.0:*:*:*:*:*:*:*",
"referenceType": "cpe23Type"
}
],
"licenseConcluded": "NOASSERTION",
"name": "sqlite",
"primaryPackagePurpose": "SOURCE",
"versionInfo": "3.45.1.0"
},
{
"SPDXID": "SPDXRef-PACKAGE-tcl-core",
"checksums": [
{
"algorithm": "SHA256",
"checksumValue": "1d3f2015e49e269cf681373d433cd54d88d5ef7443fe87f5f50f5fcfe9003e73"
}
],
"downloadLocation": "https://github.com/python/cpython-source-deps/archive/refs/tags/tcl-core-8.6.13.1.tar.gz",
"externalRefs": [
{
"referenceCategory": "SECURITY",
"referenceLocator": "cpe:2.3:a:tcl_tk:tcl_tk:8.6.13.1:*:*:*:*:*:*:*",
"referenceType": "cpe23Type"
}
],
"licenseConcluded": "NOASSERTION",
"name": "tcl-core",
"primaryPackagePurpose": "SOURCE",
"versionInfo": "8.6.13.1"
},
{
"SPDXID": "SPDXRef-PACKAGE-tk",
"checksums": [
{
"algorithm": "SHA256",
"checksumValue": "6056203b8a6aaf6ea89d90a7b55dc7f407e55c093f731a98fd830a712a3c81d3"
}
],
"downloadLocation": "https://github.com/python/cpython-source-deps/archive/refs/tags/tk-8.6.13.1.tar.gz",
"externalRefs": [
{
"referenceCategory": "SECURITY",
"referenceLocator": "cpe:2.3:a:tcl_tk:tcl_tk:8.6.13.1:*:*:*:*:*:*:*",
"referenceType": "cpe23Type"
}
],
"licenseConcluded": "NOASSERTION",
"name": "tk",
"primaryPackagePurpose": "SOURCE",
"versionInfo": "8.6.13.1"
},
{
"SPDXID": "SPDXRef-PACKAGE-xz",
"checksums": [
{
"algorithm": "SHA256",
"checksumValue": "a15c168e39e87d750c3dc766edc7f19bdda57dacf01e509678467eace91ad282"
}
],
"downloadLocation": "https://github.com/python/cpython-source-deps/archive/refs/tags/xz-5.2.5.tar.gz",
"externalRefs": [
{
"referenceCategory": "SECURITY",
"referenceLocator": "cpe:2.3:a:xz_project:xz:5.2.5:*:*:*:*:*:*:*",
"referenceType": "cpe23Type"
}
],
"licenseConcluded": "NOASSERTION",
"name": "xz",
"primaryPackagePurpose": "SOURCE",
"versionInfo": "5.2.5"
},
{
"SPDXID": "SPDXRef-PACKAGE-zlib",
"checksums": [
{
"algorithm": "SHA256",
"checksumValue": "e3f3fb32564952006eb18b091ca8464740e5eca29d328cfb0b2da22768e0b638"
}
],
"downloadLocation": "https://github.com/python/cpython-source-deps/archive/refs/tags/zlib-1.3.1.tar.gz",
"externalRefs": [
{
"referenceCategory": "SECURITY",
"referenceLocator": "cpe:2.3:a:zlib:zlib:1.3.1:*:*:*:*:*:*:*",
"referenceType": "cpe23Type"
}
],
"licenseConcluded": "NOASSERTION",
"name": "zlib",
"primaryPackagePurpose": "SOURCE",
"versionInfo": "1.3.1"
}
],
"spdxVersion": "SPDX-2.3"
}
108 changes: 91 additions & 17 deletions Tools/build/generate_sbom.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,9 +7,8 @@
import pathlib
import subprocess
import sys
import urllib.request
import typing
import zipfile
from urllib.request import urlopen

CPYTHON_ROOT_DIR = pathlib.Path(__file__).parent.parent.parent

Expand Down Expand Up @@ -125,30 +124,41 @@ def filter_gitignored_paths(paths: list[str]) -> list[str]:
return sorted([line.split()[-1] for line in git_check_ignore_lines if line.startswith("::")])


def main() -> None:
sbom_path = CPYTHON_ROOT_DIR / "Misc/sbom.spdx.json"
sbom_data = json.loads(sbom_path.read_bytes())
def get_externals() -> list[str]:
"""
Parses 'PCbuild/get_externals.bat' for external libraries.
Returns a list of (git tag, name, version) tuples.
"""
get_externals_bat_path = CPYTHON_ROOT_DIR / "PCbuild/get_externals.bat"
externals = re.findall(
r"set\s+libraries\s*=\s*%libraries%\s+([a-zA-Z0-9.-]+)\s",
get_externals_bat_path.read_text()
)
return externals

# We regenerate all of this information. Package information
# should be preserved though since that is edited by humans.
sbom_data["files"] = []
sbom_data["relationships"] = []

# Ensure all packages in this tool are represented also in the SBOM file.
actual_names = {package["name"] for package in sbom_data["packages"]}
expected_names = set(PACKAGE_TO_FILES)
error_if(
actual_names != expected_names,
f"Packages defined in SBOM tool don't match those defined in SBOM file: {actual_names}, {expected_names}",
)
def check_sbom_packages(sbom_data: dict[str, typing.Any]) -> None:
"""Make a bunch of assertions about the SBOM package data to ensure it's consistent."""

# Make a bunch of assertions about the SBOM data to ensure it's consistent.
for package in sbom_data["packages"]:
# Properties and ID must be properly formed.
error_if(
"name" not in package,
"Package is missing the 'name' field"
)

# Verify that the checksum matches the expected value
# and that the download URL is valid.
if "checksums" not in package or "CI" in os.environ:
download_location = package["downloadLocation"]
resp = urllib.request.urlopen(download_location)
error_if(resp.status != 200, f"Couldn't access URL: {download_location}'")

package["checksums"] = [{
"algorithm": "SHA256",
"checksumValue": hashlib.sha256(resp.read()).hexdigest()
}]

missing_required_keys = REQUIRED_PROPERTIES_PACKAGE - set(package.keys())
error_if(
bool(missing_required_keys),
Expand Down Expand Up @@ -180,6 +190,26 @@ def main() -> None:
f"License identifier must be 'NOASSERTION'"
)


def create_source_sbom() -> None:
sbom_path = CPYTHON_ROOT_DIR / "Misc/sbom.spdx.json"
sbom_data = json.loads(sbom_path.read_bytes())

# We regenerate all of this information. Package information
# should be preserved though since that is edited by humans.
sbom_data["files"] = []
sbom_data["relationships"] = []

# Ensure all packages in this tool are represented also in the SBOM file.
actual_names = {package["name"] for package in sbom_data["packages"]}
expected_names = set(PACKAGE_TO_FILES)
error_if(
actual_names != expected_names,
f"Packages defined in SBOM tool don't match those defined in SBOM file: {actual_names}, {expected_names}",
)

check_sbom_packages(sbom_data)

# We call 'sorted()' here a lot to avoid filesystem scan order issues.
for name, files in sorted(PACKAGE_TO_FILES.items()):
package_spdx_id = spdx_id(f"SPDXRef-PACKAGE-{name}")
Expand Down Expand Up @@ -224,5 +254,49 @@ def main() -> None:
sbom_path.write_text(json.dumps(sbom_data, indent=2, sort_keys=True))


def create_externals_sbom() -> None:
sbom_path = CPYTHON_ROOT_DIR / "Misc/externals.spdx.json"
sbom_data = json.loads(sbom_path.read_bytes())

externals = get_externals()
externals_name_to_version = {}
externals_name_to_git_tag = {}
for git_tag in externals:
name, _, version = git_tag.rpartition("-")
externals_name_to_version[name] = version
externals_name_to_git_tag[name] = git_tag

# Ensure all packages in this tool are represented also in the SBOM file.
actual_names = {package["name"] for package in sbom_data["packages"]}
expected_names = set(externals_name_to_version)
error_if(
actual_names != expected_names,
f"Packages defined in SBOM tool don't match those defined in SBOM file: {actual_names}, {expected_names}",
)

# Set the versionInfo and downloadLocation fields for all packages.
for package in sbom_data["packages"]:
package["versionInfo"] = externals_name_to_version[package["name"]]
download_location = (
f"https://github.com/python/cpython-source-deps/archive/refs/tags/{externals_name_to_git_tag[package['name']]}.tar.gz"
)
download_location_changed = download_location != package["downloadLocation"]
package["downloadLocation"] = download_location

# If the download URL has changed we want one to get recalulated.
if download_location_changed:
package.pop("checksums", None)

check_sbom_packages(sbom_data)

# Update the SBOM on disk
sbom_path.write_text(json.dumps(sbom_data, indent=2, sort_keys=True))


def main() -> None:
create_source_sbom()
create_externals_sbom()


if __name__ == "__main__":
main()