From c73b15e4f44ae3a2149102a53cddd857ec604578 Mon Sep 17 00:00:00 2001 From: Frank Scholter Peres Date: Thu, 6 Aug 2026 15:13:08 +0000 Subject: [PATCH 1/4] add python and java collector and two modes --- README.md | 73 ++++++++- defs.bzl | 11 ++ internal/generator/sbom_generator.py | 39 ++++- internal/rules.bzl | 43 +++++- scripts/BUILD.bazel | 12 ++ scripts/generate_python_metadata_cache.py | 147 +++++++++++++++++++ tests/BUILD | 6 + tests/test_generate_python_metadata_cache.py | 53 +++++++ tests/test_sbom_generator.py | 22 +++ 9 files changed, 401 insertions(+), 5 deletions(-) create mode 100644 scripts/generate_python_metadata_cache.py create mode 100644 tests/test_generate_python_metadata_cache.py diff --git a/README.md b/README.md index d37f775..445d0a3 100644 --- a/README.md +++ b/README.md @@ -37,11 +37,76 @@ sbom( "@score_crates//:MODULE.bazel.lock", ":MODULE.bazel.lock", ], + python_lockfiles = ["//path/to:requirements.txt.lock"], auto_crates_cache = True, auto_cdxgen = True, ) ``` +## SBOM modes + +The rule supports two complementary SBOM modes. The modes are selected by the +consumer's `BUILD` file; they are not separate Bazel rules. + +### Product mode + +Product mode describes software delivered by the project. Pass the product or +runtime binaries and libraries in `targets`, and enable the collectors needed +for their shipped dependencies. Product SBOMs commonly emit both SPDX and +CycloneDX: + +```starlark +sbom( + name = "product_sbom", + targets = ["//app:runtime_binary"], + component_name = "my_product", + output_formats = ["spdx", "cyclonedx"], + module_lockfiles = [":MODULE.bazel.lock"], + auto_cdxgen = True, + auto_crates_cache = True, +) +``` + +### Tool qualification mode + +Tool mode describes software used to build, test, or generate documentation. +It is kept separate from product mode because these components are not product +runtime dependencies. Pass build-tool and documentation targets, use SPDX for +the qualification inventory when required, and add the lockfiles and files +used by those tools: + +```starlark +sbom( + name = "build_tools_sbom", + targets = [ + "//:docs", + "//tools:build_tool", + "@docs_as_code//:plantuml", + ], + component_name = "my_product_build_tools", + output_formats = ["spdx"], + auto_cdxgen = False, + auto_crates_cache = False, + python_lockfiles = [ + "//tools:requirements.txt.lock", + "@docs_as_code//:requirements_lock", + ], + java_files = ["@docs_as_code//:plantuml.jar"], + exclude_patterns = ["rules_python++pip+"], +) +``` + +The Python collector reads pinned pip-compile lockfiles and emits PyPI +components with versions and SHA-256 hashes. The Java collector inventories +declared `.jar` or other Java files as file-level components, including the +file name, size, and SHA-256 checksum. This is useful for tools such as +PlantUML, where the Python integration and the Java JAR are separate artifacts. + +The host Java runtime itself (for example, the `java` executable installed by +the operating system) is not a Bazel file input and therefore is not collected +by `java_files`; it must be supplied through a separate host-tool inventory if +the qualification scope requires the JDK installation as well. + ### Parameters | Parameter | Default | Description | @@ -51,6 +116,9 @@ sbom( | `component_name` | rule `name` | Name of the root component written into the SBOM; defaults to the rule name if omitted. | | `component_version` | `None` | Version string for the root component; auto-detected from the module graph when omitted. | | `module_lockfiles` | `[]` | One or more `MODULE.bazel.lock` files used to extract dependency versions and SHA-256 checksums; C++ projects need only the workspace lockfile (`:MODULE.bazel.lock`), Rust projects should also pass `@score_crates//:MODULE.bazel.lock` to cover crate versions and checksums. | +| `python_lockfiles` | `[]` | One or more pip-compile lockfiles (`requirements.txt.lock`) used to add pinned PyPI packages, SHA-256 hashes, and license expressions from DASH. Packages that DASH cannot verify retain `NOASSERTION`; descriptions are not enriched yet and remain `Missing`. | +| `auto_python_cache` | `True` | Generates Python package metadata from `python_lockfiles`; set to `False` to disable it. | +| `java_files` | `[]` | Java or JAR files to inventory as file-level components. Each file gets its name, size, and SHA-256 checksum; this does not inventory the host Java runtime. | | `auto_crates_cache` | `True` | Runs `generate_crates_metadata_cache` at build time (requires network) to fetch Rust crate license and supplier data from dash-license-scan and crates.io; set to `False` only as a workaround for air-gapped or offline build environments — doing so produces a non-compliant SBOM where all Rust crates show `NOASSERTION` for license, supplier, and description. Has no effect when no lockfiles are provided (pure C++ projects). | | `cargo_lockfile` | `None` | Path to a `Cargo.lock` file for crate enumeration; not needed when `module_lockfiles` is provided, as a synthetic `Cargo.lock` is generated from it automatically. **Deprecated — will be removed in a future release.** | | `cdxgen_sbom` | `None` | Label to a pre-generated cdxgen CycloneDX JSON file; alternative to `auto_cdxgen` for C++ projects where cdxgen cannot run inside the Bazel build (e.g. CI environment without npm). Run cdxgen manually and pass its output here. Ignored for pure Rust projects. | @@ -137,13 +205,13 @@ Generated in `bazel-bin/`: **Data sources:** - **Bazel module graph** — version, PURL, and registry info for `bazel_dep` modules - **Bazel aspect** — transitive dependency graph and external repo dependency edges -- **dash-license-scan** — licenses data +- **dash-license-scan** — Rust and Python license data from the Eclipse Foundation and ClearlyDefined services - **crates.io API** — description and supplier for Rust crates - **cdxgen** — C++ dependency licenses, descriptions, and suppliers ### Automated Metadata Sources -All license, hash, supplier, and description values are derived from automated sources: `MODULE.bazel.lock`, `http_archive` rules, dash-license-scan (Rust), crates.io API (Rust), and cdxgen (C++). Cache files such as `cpp_metadata.json` must never be hand-edited. +All license, hash, supplier, and description values are derived from automated sources: `MODULE.bazel.lock`, `http_archive` rules, dash-license-scan (Rust and Python), crates.io API (Rust), and cdxgen (C++). Cache files such as `cpp_metadata.json` must never be hand-edited. CPE, aliases, and pedigree are the only fields that may be set manually via `sbom_ext.license()`, as they represent identity and provenance annotations that cannot be auto-deduced. @@ -177,6 +245,7 @@ Only transitive dependencies of the declared build targets are included. Build-t ### License Data by Language - **Rust**: Licenses via dash-license-scan (Eclipse Foundation + ClearlyDefined); descriptions and suppliers from crates.io API. Crates with platform-specific suffixes (e.g. `iceoryx2-bb-lock-free-qnx8`) fall back to the base crate name for lookup. +- **Python**: Licenses via dash-license-scan (Eclipse Foundation + ClearlyDefined), using `pypi/pypi/-//` identifiers generated from pip-compile lockfiles. Descriptions and suppliers are not enriched yet. - **C++**: Licenses, descriptions, and suppliers via cdxgen source tree scan. There is no dash-license-scan integration for C++ — it does not support `pkg:generic/...` PURLs used by BCR modules. If cdxgen cannot resolve a component, its description is set to `"Missing"` and its license field is empty. ### Output Format Versions diff --git a/defs.bzl b/defs.bzl index c614682..c495d25 100644 --- a/defs.bzl +++ b/defs.bzl @@ -40,6 +40,10 @@ def sbom( auto_cdxgen = False, cargo_lockfile = None, module_lockfiles = None, + python_lockfiles = None, + java_files = None, + testonly = False, + auto_python_cache = True, auto_crates_cache = True, output_formats = ["spdx", "cyclonedx"], producer_name = "Eclipse Foundation", @@ -79,6 +83,9 @@ def sbom( auto_cdxgen: Run cdxgen automatically when no cdxgen_sbom is provided cargo_lockfile: Optional Cargo.lock for crates metadata cache generation module_lockfiles: MODULE.bazel.lock files for crate metadata extraction (e.g., from score_crates and workspace) + python_lockfiles: pip-compile requirements lockfiles for Python package metadata. + java_files: Java/JAR files to inventory as file-level components. + auto_python_cache: Run Python metadata collection when python_lockfiles are provided auto_crates_cache: Run crates metadata cache generation when cargo_lockfile or module_lockfiles is provided output_formats: List of formats to generate ("spdx", "cyclonedx") producer_name: SBOM producer organization name @@ -138,6 +145,10 @@ def sbom( auto_cdxgen = auto_cdxgen, cargo_lockfile = cargo_lockfile, module_lockfiles = module_lockfiles if module_lockfiles else [], + python_lockfiles = python_lockfiles if python_lockfiles else [], + java_files = java_files if java_files else [], + testonly = testonly, + auto_python_cache = auto_python_cache, auto_crates_cache = auto_crates_cache, output_formats = output_formats, producer_name = producer_name, diff --git a/internal/generator/sbom_generator.py b/internal/generator/sbom_generator.py index 1f39c75..e34b36c 100644 --- a/internal/generator/sbom_generator.py +++ b/internal/generator/sbom_generator.py @@ -20,6 +20,7 @@ """ import argparse +import hashlib import json import re import sys @@ -157,6 +158,28 @@ def load_crates_cache(cache_path: str | None = None) -> dict[str, Any]: """ if not cache_path: return {} + + +def collect_java_file_components(file_paths: list[str]) -> list[dict[str, Any]]: + """Create file-level components for declared Java and JAR artifacts.""" + components = [] + for file_path in file_paths: + path = Path(file_path) + try: + digest = hashlib.sha256(path.read_bytes()).hexdigest() + size = path.stat().st_size + except OSError: + continue + components.append({ + "name": path.name, + "version": "file", + "type": "file", + "source": "java", + "url": "NOASSERTION", + "checksum": digest, + "description": f"Java artifact {path.name} ({size} bytes)", + }) + return components try: with open(cache_path, encoding="utf-8") as f: return json.load(f) @@ -410,6 +433,7 @@ def main() -> int: parser.add_argument("--spdx-output", help="SPDX 2.3 JSON output file") parser.add_argument("--cyclonedx-output", help="CycloneDX 1.6 output file") parser.add_argument("--crates-cache", help="Path to crates_metadata.json override") + parser.add_argument("--python-cache", help="Path to Python package metadata cache") parser.add_argument( "--cdxgen-sbom", help="Path to cdxgen-generated CycloneDX JSON for C++ enrichment", @@ -451,6 +475,14 @@ def main() -> int: # Load crates metadata cache (licenses + checksums + versions) crates_cache = load_crates_cache(args.crates_cache) + python_cache = {} + if args.python_cache: + try: + with open(args.python_cache, encoding="utf-8") as f: + python_cache = json.load(f) + except (OSError, json.JSONDecodeError): + python_cache = {} + # Add crates cache to metadata if crates_cache: if "crates" not in metadata: @@ -458,6 +490,9 @@ def main() -> int: for name, cache_data in crates_cache.items(): metadata["crates"].setdefault(name, cache_data) + components = list(python_cache.values()) + components.extend(collect_java_file_components(data.get("java_files", []))) + # Apply BCR known licenses and user overrides to modules apply_known_licenses(metadata) @@ -472,7 +507,7 @@ def main() -> int: filtered_repos = filter_repos(external_repos, exclude_patterns) # Build component list with metadata - components = [] + components.extend([]) for repo in filtered_repos: component = resolve_component(repo, metadata) @@ -502,7 +537,7 @@ def main() -> int: # (from an edge destination) are not treated as distinct components — both # would otherwise produce the same sanitised bom-ref, creating duplicates. existing_names = {c.get("name", "").rstrip("+") for c in components} - for dst in sorted(edge_dst_repos): + for dst in filter_repos(sorted(edge_dst_repos), exclude_patterns): if dst.rstrip("+") not in existing_names: component = resolve_component(dst, metadata) if component: diff --git a/internal/rules.bzl b/internal/rules.bzl index 57b3624..8e3a5ef 100644 --- a/internal/rules.bzl +++ b/internal/rules.bzl @@ -76,6 +76,8 @@ def _sbom_impl(ctx): # Collect MODULE.bazel files from dependency modules for version extraction dep_module_paths = [f.path for f in ctx.files.dep_module_files] module_lock_paths = [f.path for f in ctx.files.module_lockfiles] + python_lock_paths = [f.path for f in ctx.files.python_lockfiles] + java_file_paths = [f.path for f in ctx.files.java_files] deps_data = { "external_repos": all_external_repos.to_list(), @@ -85,6 +87,8 @@ def _sbom_impl(ctx): "exclude_patterns": exclude_patterns, "dep_module_files": dep_module_paths, "module_lockfiles": module_lock_paths, + "python_lockfiles": python_lock_paths, + "java_files": java_file_paths, "config": { "producer_name": ctx.attr.producer_name, "producer_url": ctx.attr.producer_url, @@ -119,7 +123,7 @@ def _sbom_impl(ctx): args.add("--cyclonedx-output", cdx_out) # Build inputs list - generator_inputs = [deps_json, metadata_file] + ctx.files.dep_module_files + ctx.files.module_lockfiles + generator_inputs = [deps_json, metadata_file] + ctx.files.dep_module_files + ctx.files.module_lockfiles + ctx.files.python_lockfiles + ctx.files.java_files # Auto-generate crates metadata cache if enabled and a lockfile is provided crates_cache = None @@ -180,6 +184,24 @@ def _sbom_impl(ctx): args.add("--crates-cache", crates_cache) generator_inputs.append(crates_cache) + python_cache = None + if ctx.files.python_lockfiles and ctx.attr.auto_python_cache: + python_cache = ctx.actions.declare_file(ctx.attr.name + "_python_metadata.json") + ctx.actions.run( + inputs = ctx.files.python_lockfiles, + outputs = [python_cache], + executable = ctx.executable._python_cache, + arguments = [python_cache.path] + [f.path for f in ctx.files.python_lockfiles], + mnemonic = "PythonMetadataGenerate", + progress_message = "Generating Python metadata cache for %s" % ctx.attr.name, + execution_requirements = {"requires-network": ""}, + use_default_shell_env = True, + ) + + if python_cache: + args.add("--python-cache", python_cache) + generator_inputs.append(python_cache) + # Run Python generator ctx.actions.run( inputs = generator_inputs, @@ -268,6 +290,20 @@ sbom_rule = rule( allow_files = True, doc = "MODULE.bazel.lock files for crate metadata extraction (e.g., from score_crates and workspace)", ), + "python_lockfiles": attr.label_list( + allow_files = True, + default = [], + doc = "pip-compile requirements lockfiles for Python metadata extraction", + ), + "java_files": attr.label_list( + allow_files = True, + default = [], + doc = "Java/JAR files to inventory as file-level components", + ), + "auto_python_cache": attr.bool( + default = True, + doc = "Automatically collect Python package metadata from lockfiles", + ), "cdxgen_sbom": attr.label( allow_single_file = [".json"], doc = "Optional CycloneDX JSON from cdxgen for C++ dependency enrichment", @@ -289,6 +325,11 @@ sbom_rule = rule( default = "//scripts:generate_crates_metadata_cache.py", allow_single_file = True, ), + "_python_cache": attr.label( + default = "//scripts:generate_python_metadata_cache", + executable = True, + cfg = "exec", + ), "_generator": attr.label( default = "//internal/generator:sbom_generator", executable = True, diff --git a/scripts/BUILD.bazel b/scripts/BUILD.bazel index c899fd4..75345d9 100644 --- a/scripts/BUILD.bazel +++ b/scripts/BUILD.bazel @@ -17,6 +17,7 @@ package(default_visibility = ["//visibility:public"]) exports_files([ "generate_crates_metadata_cache.py", + "generate_python_metadata_cache.py", ]) py_library( @@ -24,6 +25,17 @@ py_library( srcs = ["generate_crates_metadata_cache.py"], ) +py_binary( + name = "generate_python_metadata_cache", + srcs = ["generate_python_metadata_cache.py"], + main = "generate_python_metadata_cache.py", +) + +py_library( + name = "generate_python_metadata_cache_lib", + srcs = ["generate_python_metadata_cache.py"], +) + py_library( name = "generate_cpp_metadata_cache", srcs = ["generate_cpp_metadata_cache.py"], diff --git a/scripts/generate_python_metadata_cache.py b/scripts/generate_python_metadata_cache.py new file mode 100644 index 0000000..28f22a9 --- /dev/null +++ b/scripts/generate_python_metadata_cache.py @@ -0,0 +1,147 @@ +#!/usr/bin/env python3 + +"""Collect Python package metadata from pip-compile lockfiles.""" + +import argparse +import json +import re +import shutil +import subprocess +import tempfile +from pathlib import Path + + +PACKAGE_RE = re.compile(r"^([A-Za-z0-9][A-Za-z0-9_.-]*)\[.*\]==([^\s\\]+)|^([A-Za-z0-9][A-Za-z0-9_.-]*)==([^\s\\]+)") +HASH_RE = re.compile(r"--hash=sha256[=:]([0-9a-fA-F]{64})") + + +def _find_uvx() -> str: + """Locate uvx in PATH or in the standard user installation directory.""" + return shutil.which("uvx") or str(Path.home() / ".local/bin/uvx") + + +def parse_requirements_lockfile(path: str) -> dict[str, dict[str, str]]: + """Parse pinned packages and hashes from a pip-compile lockfile.""" + packages: dict[str, dict[str, str]] = {} + pending_name = "" + pending_version = "" + pending_hashes: list[str] = [] + + def store() -> None: + if not pending_name or not pending_version: + return + entry = { + "name": pending_name, + "version": pending_version, + "purl": f"pkg:pypi/{pending_name.lower().replace('_', '-')}@{pending_version}", + "license": "NOASSERTION", + "description": "Missing", + "supplier": "", + "source": "PyPI", + } + if pending_hashes: + entry["checksum"] = pending_hashes[0].lower() + packages[pending_name.lower().replace("_", "-")] = entry + + for raw_line in Path(path).read_text(encoding="utf-8").splitlines(): + line = raw_line.strip() + match = PACKAGE_RE.match(line) + if match: + store() + pending_name = match.group(1) or match.group(3) + pending_version = match.group(2) or match.group(4) + pending_hashes = HASH_RE.findall(line) + elif pending_name: + pending_hashes.extend(HASH_RE.findall(line)) + if not line.endswith("\\"): + store() + pending_name = "" + pending_version = "" + pending_hashes = [] + + store() + return packages + + +def run_dash_license_scan(lockfiles: list[str], summary_path: str) -> bool: + """Run DASH license scanning for Python lockfiles. + + The dash-license-scan wrapper converts requirements files to DASH PURLs and + writes the checker summary as CSV. Restricted or unverified packages are + reported by DASH with a non-zero status, but the summary remains usable. + """ + command = [ + _find_uvx(), + "--from", + "dash-license-scan@git+https://github.com/eclipse-score/dash-license-scan", + "dash-license-scan", + "--summary", + summary_path, + *lockfiles, + ] + try: + result = subprocess.run(command, check=False, text=True, timeout=600) + except (OSError, subprocess.TimeoutExpired) as error: + print(f"WARNING: DASH Python license scan unavailable: {error}") + return False + if result.returncode < 0: + print(f"WARNING: DASH Python license scan was terminated: {result.returncode}") + return False + return Path(summary_path).is_file() + + +def parse_dash_summary(summary_path: str) -> dict[str, str]: + """Parse DASH summary rows into a normalized PyPI-name/license lookup.""" + licenses: dict[str, str] = {} + for raw_line in Path(summary_path).read_text(encoding="utf-8").splitlines(): + parts = [part.strip() for part in raw_line.split(",")] + if len(parts) < 2: + continue + identifier, license_expression = parts[0], parts[1] + identifier_parts = identifier.split("/") + if ( + len(identifier_parts) >= 5 + and identifier_parts[0:3] == ["pypi", "pypi", "-"] + and license_expression + ): + licenses[identifier_parts[3].lower().replace("_", "-")] = ( + license_expression + ) + return licenses + + +def enrich_python_licenses( + packages: dict[str, dict[str, str]], lockfiles: list[str] +) -> None: + """Enrich parsed Python packages with SPDX expressions returned by DASH.""" + with tempfile.TemporaryDirectory(prefix="python-dash-") as temp_dir: + summary_path = str(Path(temp_dir) / "summary.csv") + if not run_dash_license_scan(lockfiles, summary_path): + return + for package_name, license_expression in parse_dash_summary(summary_path).items(): + if package_name in packages: + packages[package_name]["license"] = license_expression + + +def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument("output") + parser.add_argument("lockfiles", nargs="+") + parser.add_argument( + "--skip-dash", + action="store_true", + help="Skip DASH license enrichment (for offline builds)", + ) + args = parser.parse_args() + + packages: dict[str, dict[str, str]] = {} + for lockfile in args.lockfiles: + packages.update(parse_requirements_lockfile(lockfile)) + if not args.skip_dash: + enrich_python_licenses(packages, args.lockfiles) + Path(args.output).write_text(json.dumps(packages, indent=2), encoding="utf-8") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) \ No newline at end of file diff --git a/tests/BUILD b/tests/BUILD index 4fbdd6c..899f41f 100644 --- a/tests/BUILD +++ b/tests/BUILD @@ -52,6 +52,12 @@ score_py_pytest( deps = ["//scripts:generate_crates_metadata_cache"], ) +score_py_pytest( + name = "test_generate_python_metadata_cache", + srcs = ["test_generate_python_metadata_cache.py"], + deps = ["//scripts:generate_python_metadata_cache_lib"], +) + score_py_pytest( name = "test_generate_cpp_metadata_cache", srcs = ["test_generate_cpp_metadata_cache.py"], diff --git a/tests/test_generate_python_metadata_cache.py b/tests/test_generate_python_metadata_cache.py new file mode 100644 index 0000000..d754941 --- /dev/null +++ b/tests/test_generate_python_metadata_cache.py @@ -0,0 +1,53 @@ +import os +import tempfile +import unittest + +from scripts.generate_python_metadata_cache import ( + parse_dash_summary, + parse_requirements_lockfile, +) + + +class TestParseRequirementsLockfile(unittest.TestCase): + def test_pinned_packages_and_hashes(self): + fd, path = tempfile.mkstemp(text=True) + try: + with os.fdopen(fd, "w") as lockfile: + lockfile.write( + "requests==2.32.3 \\\n" + " --hash=sha256=" + "a" * 64 + "\n" + ) + lockfile.write("typing_extensions==4.12.2\n") + result = parse_requirements_lockfile(path) + finally: + os.unlink(path) + + self.assertEqual(result["requests"]["version"], "2.32.3") + self.assertEqual(result["requests"]["checksum"], "a" * 64) + self.assertEqual(result["requests"]["purl"], "pkg:pypi/requests@2.32.3") + self.assertEqual(result["typing-extensions"]["license"], "NOASSERTION") + + def test_extras_are_normalized(self): + fd, path = tempfile.mkstemp(text=True) + try: + with os.fdopen(fd, "w") as lockfile: + lockfile.write("uv[standard]==0.8.9\n") + result = parse_requirements_lockfile(path) + finally: + os.unlink(path) + + self.assertIn("uv", result) + + +class TestParseDashSummary(unittest.TestCase): + def test_parses_python_purl_license_rows(self): + fd, path = tempfile.mkstemp(text=True) + try: + with os.fdopen(fd, "w") as summary: + summary.write("pypi/pypi/-/mdurl/0.1.2, MIT, approved, Eclipse\n") + summary.write("not-a-pypi-row, Apache-2.0, approved, Eclipse\n") + result = parse_dash_summary(path) + finally: + os.unlink(path) + + self.assertEqual(result, {"mdurl": "MIT"}) diff --git a/tests/test_sbom_generator.py b/tests/test_sbom_generator.py index d763298..1921ad6 100644 --- a/tests/test_sbom_generator.py +++ b/tests/test_sbom_generator.py @@ -97,6 +97,7 @@ import unittest.mock from internal.generator.sbom_generator import ( + collect_java_file_components, deduplicate_components, filter_repos, main, @@ -107,6 +108,27 @@ ) +class TestJavaFileComponents(unittest.TestCase): + """Java and JAR artifacts are represented with reproducible checksums.""" + + def test_collects_jar_name_size_and_sha256(self): + with tempfile.NamedTemporaryFile(suffix=".jar", delete=False) as jar: + jar.write(b"plantuml-test-artifact") + jar_path = jar.name + try: + components = collect_java_file_components([jar_path]) + finally: + os.unlink(jar_path) + + self.assertEqual(len(components), 1) + self.assertEqual(components[0]["name"], os.path.basename(jar_path)) + self.assertEqual(components[0]["type"], "file") + self.assertEqual( + components[0]["checksum"], + "b05f25a893299cf4ef571b8283b4158bd6c7320b6c9733dd6f44b51a4142557c", + ) + + # --------------------------------------------------------------------------- # filter_repos # --------------------------------------------------------------------------- From e8b85632729fa01a9ee24f1286751870311dff12 Mon Sep 17 00:00:00 2001 From: Frank Scholter Peres Date: Thu, 6 Aug 2026 15:31:34 +0000 Subject: [PATCH 2/4] fix formatting --- internal/generator/sbom_generator.py | 20 ++++++++------- scripts/generate_python_metadata_cache.py | 26 +++++++++++++++----- tests/test_generate_python_metadata_cache.py | 16 ++++++++++-- 3 files changed, 45 insertions(+), 17 deletions(-) diff --git a/internal/generator/sbom_generator.py b/internal/generator/sbom_generator.py index e34b36c..aa262d1 100644 --- a/internal/generator/sbom_generator.py +++ b/internal/generator/sbom_generator.py @@ -170,15 +170,17 @@ def collect_java_file_components(file_paths: list[str]) -> list[dict[str, Any]]: size = path.stat().st_size except OSError: continue - components.append({ - "name": path.name, - "version": "file", - "type": "file", - "source": "java", - "url": "NOASSERTION", - "checksum": digest, - "description": f"Java artifact {path.name} ({size} bytes)", - }) + components.append( + { + "name": path.name, + "version": "file", + "type": "file", + "source": "java", + "url": "NOASSERTION", + "checksum": digest, + "description": f"Java artifact {path.name} ({size} bytes)", + } + ) return components try: with open(cache_path, encoding="utf-8") as f: diff --git a/scripts/generate_python_metadata_cache.py b/scripts/generate_python_metadata_cache.py index 28f22a9..3c1e6ee 100644 --- a/scripts/generate_python_metadata_cache.py +++ b/scripts/generate_python_metadata_cache.py @@ -1,4 +1,16 @@ #!/usr/bin/env python3 +# ******************************************************************************* +# Copyright (c) 2026 Contributors to the Eclipse Foundation +# +# See the NOTICE file(s) distributed with this work for additional +# information regarding copyright ownership. +# +# This program and the accompanying materials are made available under the +# terms of the Apache License Version 2.0 which is available at +# https://www.apache.org/licenses/LICENSE-2.0 +# +# SPDX-License-Identifier: Apache-2.0 +# ******************************************************************************* """Collect Python package metadata from pip-compile lockfiles.""" @@ -11,7 +23,9 @@ from pathlib import Path -PACKAGE_RE = re.compile(r"^([A-Za-z0-9][A-Za-z0-9_.-]*)\[.*\]==([^\s\\]+)|^([A-Za-z0-9][A-Za-z0-9_.-]*)==([^\s\\]+)") +PACKAGE_RE = re.compile( + r"^([A-Za-z0-9][A-Za-z0-9_.-]*)\[.*\]==([^\s\\]+)|^([A-Za-z0-9][A-Za-z0-9_.-]*)==([^\s\\]+)" +) HASH_RE = re.compile(r"--hash=sha256[=:]([0-9a-fA-F]{64})") @@ -104,9 +118,7 @@ def parse_dash_summary(summary_path: str) -> dict[str, str]: and identifier_parts[0:3] == ["pypi", "pypi", "-"] and license_expression ): - licenses[identifier_parts[3].lower().replace("_", "-")] = ( - license_expression - ) + licenses[identifier_parts[3].lower().replace("_", "-")] = license_expression return licenses @@ -118,7 +130,9 @@ def enrich_python_licenses( summary_path = str(Path(temp_dir) / "summary.csv") if not run_dash_license_scan(lockfiles, summary_path): return - for package_name, license_expression in parse_dash_summary(summary_path).items(): + for package_name, license_expression in parse_dash_summary( + summary_path + ).items(): if package_name in packages: packages[package_name]["license"] = license_expression @@ -144,4 +158,4 @@ def main() -> int: if __name__ == "__main__": - raise SystemExit(main()) \ No newline at end of file + raise SystemExit(main()) diff --git a/tests/test_generate_python_metadata_cache.py b/tests/test_generate_python_metadata_cache.py index d754941..951c3eb 100644 --- a/tests/test_generate_python_metadata_cache.py +++ b/tests/test_generate_python_metadata_cache.py @@ -1,3 +1,16 @@ +# ******************************************************************************* +# Copyright (c) 2026 Contributors to the Eclipse Foundation +# +# See the NOTICE file(s) distributed with this work for additional +# information regarding copyright ownership. +# +# This program and the accompanying materials are made available under the +# terms of the Apache License Version 2.0 which is available at +# https://www.apache.org/licenses/LICENSE-2.0 +# +# SPDX-License-Identifier: Apache-2.0 +# ******************************************************************************* + import os import tempfile import unittest @@ -14,8 +27,7 @@ def test_pinned_packages_and_hashes(self): try: with os.fdopen(fd, "w") as lockfile: lockfile.write( - "requests==2.32.3 \\\n" - " --hash=sha256=" + "a" * 64 + "\n" + "requests==2.32.3 \\\n --hash=sha256=" + "a" * 64 + "\n" ) lockfile.write("typing_extensions==4.12.2\n") result = parse_requirements_lockfile(path) From 74020ed5d7a75ce36de1fc5cfb2f13832dc69991 Mon Sep 17 00:00:00 2001 From: Frank Scholter Peres Date: Thu, 6 Aug 2026 15:38:02 +0000 Subject: [PATCH 3/4] fix unit tests --- internal/generator/sbom_generator.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/internal/generator/sbom_generator.py b/internal/generator/sbom_generator.py index aa262d1..fc4ac9a 100644 --- a/internal/generator/sbom_generator.py +++ b/internal/generator/sbom_generator.py @@ -158,6 +158,11 @@ def load_crates_cache(cache_path: str | None = None) -> dict[str, Any]: """ if not cache_path: return {} + try: + with open(cache_path, encoding="utf-8") as f: + return json.load(f) + except (OSError, json.JSONDecodeError): + return {} def collect_java_file_components(file_paths: list[str]) -> list[dict[str, Any]]: @@ -182,11 +187,6 @@ def collect_java_file_components(file_paths: list[str]) -> list[dict[str, Any]]: } ) return components - try: - with open(cache_path, encoding="utf-8") as f: - return json.load(f) - except (OSError, json.JSONDecodeError): - return {} # Known licenses for Bazel Central Registry (BCR) C++ modules. From 41a4732eedc3429d81c2567a6392e6f703d49926 Mon Sep 17 00:00:00 2001 From: Frank Scholter Peres Date: Fri, 7 Aug 2026 08:59:56 +0000 Subject: [PATCH 4/4] Fix ci and comments --- internal/generator/sbom_generator.py | 8 ++++++-- internal/rules.bzl | 2 +- scripts/generate_python_metadata_cache.py | 18 +++++++++++++++++- tests/test_sbom_generator.py | 6 +++++- 4 files changed, 29 insertions(+), 5 deletions(-) diff --git a/internal/generator/sbom_generator.py b/internal/generator/sbom_generator.py index fc4ac9a..c12ef53 100644 --- a/internal/generator/sbom_generator.py +++ b/internal/generator/sbom_generator.py @@ -171,13 +171,17 @@ def collect_java_file_components(file_paths: list[str]) -> list[dict[str, Any]]: for file_path in file_paths: path = Path(file_path) try: - digest = hashlib.sha256(path.read_bytes()).hexdigest() size = path.stat().st_size + digest_hash = hashlib.sha256() + with path.open("rb") as artifact: + for chunk in iter(lambda: artifact.read(1024 * 1024), b""): + digest_hash.update(chunk) + digest = digest_hash.hexdigest() except OSError: continue components.append( { - "name": path.name, + "name": f"{path.name}-{digest[:12]}", "version": "file", "type": "file", "source": "java", diff --git a/internal/rules.bzl b/internal/rules.bzl index 8e3a5ef..ebddc89 100644 --- a/internal/rules.bzl +++ b/internal/rules.bzl @@ -123,7 +123,7 @@ def _sbom_impl(ctx): args.add("--cyclonedx-output", cdx_out) # Build inputs list - generator_inputs = [deps_json, metadata_file] + ctx.files.dep_module_files + ctx.files.module_lockfiles + ctx.files.python_lockfiles + ctx.files.java_files + generator_inputs = [deps_json, metadata_file] + ctx.files.dep_module_files + ctx.files.module_lockfiles + ctx.files.java_files # Auto-generate crates metadata cache if enabled and a lockfile is provided crates_cache = None diff --git a/scripts/generate_python_metadata_cache.py b/scripts/generate_python_metadata_cache.py index 3c1e6ee..09f01c3 100644 --- a/scripts/generate_python_metadata_cache.py +++ b/scripts/generate_python_metadata_cache.py @@ -16,6 +16,7 @@ import argparse import json +import os import re import shutil import subprocess @@ -84,6 +85,10 @@ def run_dash_license_scan(lockfiles: list[str], summary_path: str) -> bool: writes the checker summary as CSV. Restricted or unverified packages are reported by DASH with a non-zero status, but the summary remains usable. """ + cache_dir = tempfile.mkdtemp(prefix="dash-license-scan-") + env = os.environ.copy() + env["UV_CACHE_DIR"] = cache_dir + env["UV_TOOL_DIR"] = cache_dir command = [ _find_uvx(), "--from", @@ -94,13 +99,24 @@ def run_dash_license_scan(lockfiles: list[str], summary_path: str) -> bool: *lockfiles, ] try: - result = subprocess.run(command, check=False, text=True, timeout=600) + result = subprocess.run( + command, + check=False, + text=True, + capture_output=True, + timeout=600, + env=env, + ) except (OSError, subprocess.TimeoutExpired) as error: print(f"WARNING: DASH Python license scan unavailable: {error}") return False if result.returncode < 0: print(f"WARNING: DASH Python license scan was terminated: {result.returncode}") return False + if result.returncode != 0 and result.stderr: + print( + f"WARNING: DASH Python license scan reported errors: {result.stderr.strip()}" + ) return Path(summary_path).is_file() diff --git a/tests/test_sbom_generator.py b/tests/test_sbom_generator.py index 1921ad6..f19299f 100644 --- a/tests/test_sbom_generator.py +++ b/tests/test_sbom_generator.py @@ -91,6 +91,7 @@ import json import os +import re import shutil import tempfile import unittest @@ -121,7 +122,10 @@ def test_collects_jar_name_size_and_sha256(self): os.unlink(jar_path) self.assertEqual(len(components), 1) - self.assertEqual(components[0]["name"], os.path.basename(jar_path)) + self.assertRegex( + components[0]["name"], + rf"^{re.escape(os.path.basename(jar_path))}-[0-9a-f]{{12}}$", + ) self.assertEqual(components[0]["type"], "file") self.assertEqual( components[0]["checksum"],