diff --git a/.github/workflows/build-rapids-image.yml b/.github/workflows/build-rapids-image.yml index 02947e37..af03aad9 100644 --- a/.github/workflows/build-rapids-image.yml +++ b/.github/workflows/build-rapids-image.yml @@ -1,4 +1,4 @@ -# Copyright (c) 2025-2026, NVIDIA CORPORATION. +# Copyright (c) 2025-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. name: build-rapids on: @@ -25,6 +25,9 @@ on: RAPIDS_VER: required: true type: string + RAPIDS_NOTEBOOKS_REF: + required: true + type: string BASE_TAG: required: true type: string @@ -32,6 +35,10 @@ on: required: true type: string +permissions: + contents: read + id-token: write + env: # prevent buildx from creating arch-specific manifest lists... # we want single images, that are later referenced together in a multiarch manifest @@ -75,6 +82,18 @@ jobs: with: username: ${{ secrets.GPUCIBOT_DOCKERHUB_USER }} password: ${{ secrets.GPUCIBOT_DOCKERHUB_TOKEN }} + - name: Install Go + uses: actions/setup-go@4a3601121dd01d1626a1e23e37211e3254c1c06c # v6 + with: + go-version: '1.25.x' + cache: false + - name: Install ORAS + run: | + GOBIN="${RUNNER_TEMP}/oras-bin" go install oras.land/oras/cmd/oras@v1.3.0 + echo "${RUNNER_TEMP}/oras-bin" >> "${GITHUB_PATH}" + "${RUNNER_TEMP}/oras-bin/oras" version + - name: Set up Cosign + uses: sigstore/cosign-installer@f713795cb21599bc4e5c4b58cbad1da852d7eeb9 # v3 - name: Set up Docker Context for Buildx id: buildx-context run: | @@ -99,8 +118,10 @@ jobs: LINUX_DISTRO_VER: ${{ inputs.LINUX_DISTRO_VER }} LINUX_VER: ${{ inputs.LINUX_VER }} PYTHON_VER: ${{ inputs.PYTHON_VER }} + RAPIDS_NOTEBOOKS_REF: ${{ inputs.RAPIDS_NOTEBOOKS_REF }} RAPIDS_VER: ${{ inputs.RAPIDS_VER }} - name: Build base image + id: build-base uses: docker/build-push-action@10e90e3645eae34f1e60eeb005ba3a3d33f178e8 # v6 with: context: context @@ -113,7 +134,26 @@ jobs: tags: ${{ inputs.BASE_TAG }}-${{ matrix.ARCH }} # ensure only OCI mediatypes are used: https://docs.docker.com/build/exporters/#oci-media-types outputs: type=registry,oci-mediatypes=true + - name: Export base image package provenance + env: + DOCKER_BUILD_ARGS: ${{ steps.generate-build-args.outputs.DOCKER_BUILD_ARGS }} + PROVENANCE_OUTPUT_DIR: ${{ runner.temp }}/provenance-base-${{ matrix.ARCH }} + PROVENANCE_TARGET: provenance-base + run: ci/export-image-provenance.sh + - name: Publish base image provenance + env: + CUDA_VER: ${{ inputs.CUDA_VER }} + DOCKER_BUILD_ARGS: ${{ steps.generate-build-args.outputs.DOCKER_BUILD_ARGS }} + IMAGE_DIGEST: ${{ steps.build-base.outputs.digest }} + IMAGE_KIND: base + IMAGE_PLATFORM: linux/${{ matrix.ARCH }} + IMAGE_REFERENCE: ${{ inputs.BASE_TAG }}-${{ matrix.ARCH }} + PROVENANCE_OUTPUT_DIR: ${{ runner.temp }}/provenance-base-${{ matrix.ARCH }} + PYTHON_VER: ${{ inputs.PYTHON_VER }} + RAPIDS_VER: ${{ inputs.RAPIDS_VER }} + run: ci/publish-image-provenance.sh - name: Build notebooks image + id: build-notebooks uses: docker/build-push-action@10e90e3645eae34f1e60eeb005ba3a3d33f178e8 # v6 with: context: context @@ -126,3 +166,21 @@ jobs: tags: ${{ inputs.NOTEBOOKS_TAG }}-${{ matrix.ARCH }} # ensure only OCI mediatypes are used: https://docs.docker.com/build/exporters/#oci-media-types outputs: type=registry,oci-mediatypes=true + - name: Export notebooks image package provenance + env: + DOCKER_BUILD_ARGS: ${{ steps.generate-build-args.outputs.DOCKER_BUILD_ARGS }} + PROVENANCE_OUTPUT_DIR: ${{ runner.temp }}/provenance-notebooks-${{ matrix.ARCH }} + PROVENANCE_TARGET: provenance-notebooks + run: ci/export-image-provenance.sh + - name: Publish notebooks image provenance + env: + CUDA_VER: ${{ inputs.CUDA_VER }} + DOCKER_BUILD_ARGS: ${{ steps.generate-build-args.outputs.DOCKER_BUILD_ARGS }} + IMAGE_DIGEST: ${{ steps.build-notebooks.outputs.digest }} + IMAGE_KIND: notebooks + IMAGE_PLATFORM: linux/${{ matrix.ARCH }} + IMAGE_REFERENCE: ${{ inputs.NOTEBOOKS_TAG }}-${{ matrix.ARCH }} + PROVENANCE_OUTPUT_DIR: ${{ runner.temp }}/provenance-notebooks-${{ matrix.ARCH }} + PYTHON_VER: ${{ inputs.PYTHON_VER }} + RAPIDS_VER: ${{ inputs.RAPIDS_VER }} + run: ci/publish-image-provenance.sh diff --git a/.github/workflows/build-test-publish-images.yml b/.github/workflows/build-test-publish-images.yml index 1d3bf902..dc0ff7cb 100644 --- a/.github/workflows/build-test-publish-images.yml +++ b/.github/workflows/build-test-publish-images.yml @@ -55,6 +55,10 @@ jobs: - name: Run hadolint run: | ci/lint-dockerfiles.sh + - name: Test image provenance manifest generator + run: | + python3 -m pip install --disable-pip-version-check -r requirements-test.txt + python3 -m pytest -q tests compute-matrix: runs-on: ubuntu-latest outputs: @@ -68,6 +72,7 @@ jobs: NOTEBOOKS_TAG_PREFIX: ${{ steps.compute-tag-prefix.outputs.NOTEBOOKS_TAG_PREFIX }} CUVS_BENCH_TAG_PREFIX: ${{ steps.compute-tag-prefix.outputs.CUVS_BENCH_TAG_PREFIX }} CUVS_BENCH_CPU_TAG_PREFIX: ${{ steps.compute-tag-prefix.outputs.CUVS_BENCH_CPU_TAG_PREFIX }} + RAPIDS_NOTEBOOKS_REF: ${{ steps.compute-rapids-ver.outputs.RAPIDS_NOTEBOOKS_REF }} RAPIDS_VER: ${{ steps.compute-rapids-ver.outputs.RAPIDS_VER }} ALPHA_TAG: ${{ steps.compute-rapids-ver.outputs.ALPHA_TAG }} steps: @@ -119,6 +124,8 @@ jobs: echo "CUVS_BENCH_CPU_IMAGE_REPO=${cuvs_bench_cpu_repo}" | tee -a ${GITHUB_OUTPUT} - name: Compute RAPIDS_VER id: compute-rapids-ver + env: + BUILD_TYPE: ${{ inputs.build_type }} run: | GIT_DESCRIBE_TAG="$(git describe --tags --first-parent --abbrev=0)" GIT_DESCRIBE_TAG="${GIT_DESCRIBE_TAG:1}" # remove leading 'v' @@ -128,7 +135,20 @@ jobs: ALPHA_TAG="a" fi RAPIDS_VER="$(echo $GIT_DESCRIBE_TAG | awk 'BEGIN{FS=OFS="."} NF--')" # Convert full tag to YY.MM + RAPIDS_NOTEBOOKS_REF="main" + if [[ ! $GIT_DESCRIBE_TAG =~ [a-z] ]]; then + RAPIDS_NOTEBOOKS_REF="release/${RAPIDS_VER}" + fi + # The source branch advances before its packages are published. Keep + # pull-request images on a matching published package and source line. + if [[ $BUILD_TYPE == "pull-request" ]]; then + RAPIDS_VER="26.08" + RAPIDS_NOTEBOOKS_REF="release/26.08" + ALPHA_TAG="a" + fi + + echo "RAPIDS_NOTEBOOKS_REF=${RAPIDS_NOTEBOOKS_REF}" | tee -a ${GITHUB_OUTPUT} echo "RAPIDS_VER=${RAPIDS_VER}" | tee -a ${GITHUB_OUTPUT} echo "ALPHA_TAG=${ALPHA_TAG}" | tee -a ${GITHUB_OUTPUT} - name: Compute test matrix @@ -159,6 +179,7 @@ jobs: LINUX_DISTRO_VER: ${{ matrix.LINUX_DISTRO_VER }} LINUX_VER: ${{ matrix.LINUX_VER }} PYTHON_VER: ${{ matrix.PYTHON_VER }} + RAPIDS_NOTEBOOKS_REF: ${{ needs.compute-matrix.outputs.RAPIDS_NOTEBOOKS_REF }} RAPIDS_VER: ${{ needs.compute-matrix.outputs.RAPIDS_VER }} BASE_TAG: "rapidsai/${{ needs.compute-matrix.outputs.BASE_IMAGE_REPO }}:\ @@ -224,6 +245,18 @@ jobs: with: username: ${{ secrets.GPUCIBOT_DOCKERHUB_USER }} password: ${{ secrets.GPUCIBOT_DOCKERHUB_TOKEN }} + - name: Install Go + uses: actions/setup-go@4a3601121dd01d1626a1e23e37211e3254c1c06c # v6 + with: + go-version: '1.25.x' + cache: false + - name: Install ORAS + run: | + GOBIN="${RUNNER_TEMP}/oras-bin" go install oras.land/oras/cmd/oras@v1.3.0 + echo "${RUNNER_TEMP}/oras-bin" >> "${GITHUB_PATH}" + "${RUNNER_TEMP}/oras-bin/oras" version + - name: Set up Cosign + uses: sigstore/cosign-installer@f713795cb21599bc4e5c4b58cbad1da852d7eeb9 # v3 - name: Create multiarch manifest shell: bash env: @@ -239,6 +272,38 @@ jobs: GPUCIBOT_DOCKERHUB_TOKEN: ${{ secrets.GPUCIBOT_DOCKERHUB_TOKEN }} ARCHES: ${{ toJSON(matrix.ARCHES) }} run: ci/create-rapids-multiarch-manifest.sh + - name: Publish multiarch image provenance indexes + shell: bash + env: + BASE_IMAGE_REPO: ${{ needs.compute-matrix.outputs.BASE_IMAGE_REPO }} + BASE_TAG_PREFIX: ${{ needs.compute-matrix.outputs.BASE_TAG_PREFIX }} + RAPIDS_VER: ${{ needs.compute-matrix.outputs.RAPIDS_VER }} + ALPHA_TAG: ${{ needs.compute-matrix.outputs.ALPHA_TAG }} + CUDA_TAG: ${{ matrix.CUDA_TAG }} + PYTHON_VER: ${{ matrix.PYTHON_VER }} + NOTEBOOKS_IMAGE_REPO: ${{ needs.compute-matrix.outputs.NOTEBOOKS_IMAGE_REPO }} + NOTEBOOKS_TAG_PREFIX: ${{ needs.compute-matrix.outputs.NOTEBOOKS_TAG_PREFIX }} + ARCHES: ${{ toJSON(matrix.ARCHES) }} + run: | + base_reference="rapidsai/${BASE_IMAGE_REPO}:${BASE_TAG_PREFIX}${RAPIDS_VER}${ALPHA_TAG}-cuda${CUDA_TAG}-py${PYTHON_VER}" + notebooks_reference="rapidsai/${NOTEBOOKS_IMAGE_REPO}:${NOTEBOOKS_TAG_PREFIX}${RAPIDS_VER}${ALPHA_TAG}-cuda${CUDA_TAG}-py${PYTHON_VER}" + for image_kind in base notebooks; do + if [[ "$image_kind" == base ]]; then + image_reference="$base_reference" + else + image_reference="$notebooks_reference" + fi + platform_references="$({ + while IFS= read -r arch; do + printf 'linux/%s=%s-%s\n' "$arch" "$image_reference" "$arch" + done < <(jq -r '.[]' <<< "$ARCHES") + })" + IMAGE_KIND="$image_kind" \ + IMAGE_REFERENCE="$image_reference" \ + PLATFORM_REFERENCES="$platform_references" \ + CUDA_VER="$CUDA_TAG" \ + ci/publish-image-provenance-index.sh + done build-cuvs-multiarch-manifest: needs: [build-cuvs, compute-matrix] strategy: diff --git a/.gitignore b/.gitignore index f99d452a..2d5c5c06 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,6 @@ .vscode +__pycache__/ +*.py[cod] *.a *.bz2 diff --git a/Dockerfile b/Dockerfile index 756b38ad..5a232a5e 100644 --- a/Dockerfile +++ b/Dockerfile @@ -14,7 +14,7 @@ FROM python:${PYTHON_VER} AS dependencies ARG CPU_ARCH=notset ARG CUDA_VER=notset ARG PYTHON_VER=notset -ARG RAPIDS_BRANCH="main" +ARG RAPIDS_NOTEBOOKS_REF="main" ARG RAPIDS_VER=26.10 ARG YQ_VER=notset @@ -258,6 +258,7 @@ LABEL com.nvidia.workbench.os="linux" LABEL com.nvidia.workbench.package-manager-environment.target="/opt/conda" LABEL com.nvidia.workbench.package-manager-environment.type="conda" LABEL com.nvidia.workbench.package-manager.apt.binary="/usr/bin/apt" + LABEL com.nvidia.workbench.package-manager.apt.installed-packages="" LABEL com.nvidia.workbench.package-manager.conda3.binary="/opt/conda/bin/conda" LABEL com.nvidia.workbench.package-manager.conda3.installed-packages="rapids cudf cuml cugraph rmm pylibraft cucim xgboost jupyterlab" @@ -268,3 +269,15 @@ LABEL com.nvidia.workbench.schema-version="v2" LABEL com.nvidia.workbench.user.gid="1000" LABEL com.nvidia.workbench.user.uid="1001" LABEL com.nvidia.workbench.user.username="rapids" + +# Minimal BuildKit export targets used to publish image provenance without +# exporting or pulling the image filesystem. Each contains only conda package +# metadata copied from the corresponding final image target. +FROM scratch AS provenance-base +COPY --from=base /opt/conda/conda-meta /conda-meta + +FROM scratch AS provenance-notebooks +COPY --from=notebooks /opt/conda/conda-meta /conda-meta + +# Keep the default Docker build target runnable for local users. +FROM notebooks diff --git a/README.md b/README.md index d074ef1e..1f99d314 100644 --- a/README.md +++ b/README.md @@ -39,6 +39,29 @@ The following environment variables can be passed to the `docker run` commands f - `PIP_TIMEOUT` - how long (in seconds) the `pip` install should wait before exiting - `UNQUOTE` - Whether the command line args to `docker run` should be [executed with or without being quoted](./context/entrypoint.sh). Default to false and it is unlikely that you need to change this. +## Image Provenance Manifests + +Published `base` and `notebooks` images have an OCI referrer containing a +RAPIDS image provenance manifest. It is attached to the immutable image digest, +not to a mutable tag, and can be retrieved from the registry without downloading +image layers. The platform-specific manifest records the source commit and +workflow run plus the exact conda package name, version, build, channel, and +source URL taken from `conda-meta`. The multiarch image-index manifest links the +corresponding platform manifests. + +Conda package records intentionally leave `purls` empty until an upstream pURL +mapping has been verified. Conda is a distribution format, not an upstream pURL +type, so consumers must not derive `pkg:conda/...` identifiers from these +records. The manifest uses the OCI artifact type +`application/vnd.rapids.image.provenance.v1+json` for platform images and +`application/vnd.rapids.image.provenance.index.v1+json` for multiarch indexes. + +The build publishes these records with [ORAS](https://oras.land/), using an OCI +referrer to the image digest, and keylessly signs the attached artifact with +GitHub Actions OIDC. Scanner and triage tooling can discover the referrer, +retrieve its JSON, and compare the scanner-reported component version with the +build facts without pulling the image. + ## Bind Mounts Mounting files/folders to the locations specified below provide additional functionality for the images. diff --git a/ci/compute-build-args.sh b/ci/compute-build-args.sh index 38ffb60c..fd52fb55 100755 --- a/ci/compute-build-args.sh +++ b/ci/compute-build-args.sh @@ -12,6 +12,7 @@ LINUX_DISTRO: ${LINUX_DISTRO:-notset} LINUX_DISTRO_VER: ${LINUX_DISTRO_VER:-notset} LINUX_VER: ${LINUX_VER} PYTHON_VER: ${PYTHON_VER} +RAPIDS_NOTEBOOKS_REF: ${RAPIDS_NOTEBOOKS_REF:-main} RAPIDS_VER: ${RAPIDS_VER} " export ARGS @@ -30,7 +31,7 @@ if [ -n "${GITHUB_ACTIONS:-}" ]; then # cat < "${GITHUB_OUTPUT:-/dev/stdout}" DOCKER_BUILD_ARGS< str: + return f"sha256:{hashlib.sha256(payload).hexdigest()}" + + +def parse_build_args(values: list[str]) -> dict[str, str]: + """Convert repeated ``NAME=VALUE`` CLI options to a stable mapping.""" + build_args: dict[str, str] = {} + for value in values: + name, separator, argument = value.partition("=") + if not separator or not name: + raise ValueError(f"build argument must use NAME=VALUE syntax: {value!r}") + build_args[name] = argument + return dict(sorted(build_args.items())) + + +def conda_packages(metadata_dir: Path | None) -> list[dict[str, Any]]: + """Read exact installed-package facts from conda-meta records.""" + if metadata_dir is None: + return [] + packages: list[dict[str, Any]] = [] + for path in sorted(metadata_dir.glob("*.json")): + try: + record = json.loads(path.read_text(encoding="utf-8")) + except json.JSONDecodeError as exc: + raise ValueError(f"invalid conda metadata JSON: {path}") from exc + if not isinstance(record, dict): + raise ValueError(f"conda metadata must be an object: {path}") + name = str(record.get("name") or "").strip() + version = str(record.get("version") or "").strip() + build = str(record.get("build") or "").strip() + if not name or not version: + raise ValueError(f"conda metadata lacks name or version: {path}") + packages.append( + { + "name": name, + "version": version, + "build": build, + "build_number": record.get("build_number"), + "channel": str(record.get("channel") or record.get("url") or ""), + "url": str(record.get("url") or ""), + # Conda is a distribution format, not an upstream pURL type. + # A pURL is populated only once a vetted package mapping exists. + "purls": [], + "purl_source": "unmapped", + } + ) + return sorted( + packages, + key=lambda package: ( + package["name"], + package["version"], + package["build"], + package["channel"], + ), + ) + + +def parse_platform_manifest(value: str) -> dict[str, str]: + """Parse ``os/architecture|reference|digest`` into a manifest entry.""" + platform, separator, remainder = value.partition("|") + reference, separator2, digest = remainder.partition("|") + _os_name, separator3, _architecture = platform.partition("/") + if not separator or not separator2 or not separator3: + raise ValueError( + "platform manifests must use os/architecture|reference|sha256:digest" + ) + if not digest.startswith("sha256:"): + raise ValueError(f"platform manifest digest must be sha256: {digest!r}") + return { + "platform": platform, + "reference": reference, + "digest": digest, + } + + +def build_manifest( + context: ManifestContext, + *, + metadata_dir: Path | None = None, + platform_manifests: list[str] | None = None, +) -> dict[str, Any]: + """Construct the schema payload for a platform image or multiarch index.""" + if not context.image_digest.startswith("sha256:"): + raise ValueError(f"image digest must be sha256: {context.image_digest!r}") + packages = conda_packages(metadata_dir) + package_payload = json.dumps(packages, sort_keys=True, separators=(",", ":")).encode() + subject: dict[str, Any] = { + "reference": context.image_reference, + "digest": context.image_digest, + } + if context.platform != "multiarch": + os_name, separator, architecture = context.platform.partition("/") + if not separator: + raise ValueError("platform must be os/architecture or multiarch") + subject["platform"] = {"os": os_name, "architecture": architecture} + manifest: dict[str, Any] = { + "schema_version": SCHEMA_VERSION, + "generated_at": datetime.now(timezone.utc).isoformat(), + "subject": subject, + "image": { + "kind": context.image_kind, + "rapids_version": context.rapids_version, + "cuda_version": context.cuda_version, + "python_version": context.python_version, + }, + "build": { + "source_repository": context.source_repository, + "source_commit": context.source_commit, + "workflow_ref": context.workflow_ref, + "workflow_run_url": context.workflow_run_url, + "build_args": parse_build_args(context.build_args), + }, + "conda_packages": packages, + "conda_packages_sha256": _sha256(package_payload), + "platform_manifests": sorted( + (parse_platform_manifest(value) for value in platform_manifests or []), + key=lambda item: item["platform"], + ), + } + return manifest + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--output", type=Path, required=True) + parser.add_argument("--image-reference", required=True) + parser.add_argument("--image-digest", required=True) + parser.add_argument("--image-kind", required=True) + parser.add_argument("--rapids-version", required=True) + parser.add_argument("--cuda-version", required=True) + parser.add_argument("--python-version", required=True) + parser.add_argument("--platform", required=True) + parser.add_argument("--source-repository", required=True) + parser.add_argument("--source-commit", required=True) + parser.add_argument("--workflow-ref", required=True) + parser.add_argument("--workflow-run-url", required=True) + parser.add_argument("--conda-meta-dir", type=Path) + parser.add_argument("--build-arg", action="append", default=[]) + parser.add_argument("--platform-manifest", action="append", default=[]) + return parser.parse_args() + + +def main() -> None: + args = parse_args() + manifest = build_manifest( + ManifestContext( + image_reference=args.image_reference, + image_digest=args.image_digest, + image_kind=args.image_kind, + rapids_version=args.rapids_version, + cuda_version=args.cuda_version, + python_version=args.python_version, + platform=args.platform, + source_repository=args.source_repository, + source_commit=args.source_commit, + workflow_ref=args.workflow_ref, + workflow_run_url=args.workflow_run_url, + build_args=args.build_arg, + ), + metadata_dir=args.conda_meta_dir, + platform_manifests=args.platform_manifest, + ) + args.output.parent.mkdir(parents=True, exist_ok=True) + args.output.write_text(json.dumps(manifest, indent=2, sort_keys=True) + "\n") + + +if __name__ == "__main__": + main() diff --git a/ci/publish-image-provenance-index.sh b/ci/publish-image-provenance-index.sh new file mode 100755 index 00000000..64032802 --- /dev/null +++ b/ci/publish-image-provenance-index.sh @@ -0,0 +1,67 @@ +#!/bin/bash +# Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +set -eEuo pipefail + +: "${IMAGE_REFERENCE:?Set IMAGE_REFERENCE to the multiarch image tag}" +: "${IMAGE_KIND:?Set IMAGE_KIND to base or notebooks}" +: "${RAPIDS_VER:?Set RAPIDS_VER}" +: "${CUDA_VER:?Set CUDA_VER}" +: "${PYTHON_VER:?Set PYTHON_VER}" +: "${PLATFORM_REFERENCES:?Set PLATFORM_REFERENCES as platform=tag lines}" + +normalize_registry_reference() { + local reference="$1" + local first_component="${reference%%/*}" + if [[ $first_component != *.* && $first_component != *:* && $first_component != "localhost" ]]; then + reference="docker.io/${reference}" + fi + printf '%s\n' "$reference" +} + +output_dir="${RUNNER_TEMP:-/tmp}/image-provenance-index-${IMAGE_KIND}" +manifest_path="$output_dir/image-provenance-index.json" +workflow_run_url="${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}" +registry_reference="$(normalize_registry_reference "$IMAGE_REFERENCE")" +mkdir -p "$output_dir" + +image_digest="$(oras manifest fetch --descriptor "$registry_reference" | jq -r '.digest')" +platform_args=() +while IFS='=' read -r platform reference; do + [[ -z "$platform" ]] && continue + platform_reference="$(normalize_registry_reference "$reference")" + digest="$(oras manifest fetch --descriptor "$platform_reference" | jq -r '.digest')" + platform_args+=(--platform-manifest "${platform}|${platform_reference}|${digest}") +done <<<"$PLATFORM_REFERENCES" + +python3 ci/image_provenance_manifest.py \ + --output "$manifest_path" \ + --image-reference "$IMAGE_REFERENCE" \ + --image-digest "$image_digest" \ + --image-kind "$IMAGE_KIND" \ + --rapids-version "$RAPIDS_VER" \ + --cuda-version "$CUDA_VER" \ + --python-version "$PYTHON_VER" \ + --platform multiarch \ + --source-repository "${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}" \ + --source-commit "$GITHUB_SHA" \ + --workflow-ref "${GITHUB_WORKFLOW_REF:-$GITHUB_WORKFLOW}" \ + --workflow-run-url "$workflow_run_url" \ + --build-arg "RAPIDS_VER=$RAPIDS_VER" \ + --build-arg "CUDA_VER=$CUDA_VER" \ + --build-arg "PYTHON_VER=$PYTHON_VER" \ + "${platform_args[@]}" + +attached_manifest="$output_dir/attached-image-provenance-index.json" +( + cd "$output_dir" + oras attach \ + --artifact-type application/vnd.rapids.image.provenance.index.v1+json \ + --export-manifest "$attached_manifest" \ + "${registry_reference%@*}@${image_digest}" \ + "image-provenance-index.json:application/vnd.rapids.image.provenance.index.v1+json" +) + +artifact_digest="sha256:$(sha256sum "$attached_manifest" | awk '{print $1}')" +cosign sign --yes "${registry_reference%@*}@${artifact_digest}" diff --git a/ci/publish-image-provenance.sh b/ci/publish-image-provenance.sh new file mode 100755 index 00000000..617a837e --- /dev/null +++ b/ci/publish-image-provenance.sh @@ -0,0 +1,62 @@ +#!/bin/bash +# Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +set -eEuo pipefail + +: "${IMAGE_REFERENCE:?Set IMAGE_REFERENCE to the pushed image tag}" +: "${IMAGE_DIGEST:?Set IMAGE_DIGEST from docker/build-push-action output}" +: "${IMAGE_KIND:?Set IMAGE_KIND to base or notebooks}" +: "${IMAGE_PLATFORM:?Set IMAGE_PLATFORM to linux/architecture}" +: "${PROVENANCE_OUTPUT_DIR:?Set PROVENANCE_OUTPUT_DIR from export-image-provenance.sh}" +: "${RAPIDS_VER:?Set RAPIDS_VER}" +: "${CUDA_VER:?Set CUDA_VER}" +: "${PYTHON_VER:?Set PYTHON_VER}" + +normalize_registry_reference() { + local reference="$1" + local first_component="${reference%%/*}" + if [[ $first_component != *.* && $first_component != *:* && $first_component != "localhost" ]]; then + reference="docker.io/${reference}" + fi + printf '%s\n' "$reference" +} + +manifest_path="$PROVENANCE_OUTPUT_DIR/image-provenance.json" +workflow_run_url="${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}" +registry_reference="$(normalize_registry_reference "$IMAGE_REFERENCE")" + +generator_args=() +while IFS= read -r argument; do + [[ -z "$argument" ]] && continue + generator_args+=(--build-arg "$argument") +done <<<"${DOCKER_BUILD_ARGS:-}" + +python3 ci/image_provenance_manifest.py \ + --output "$manifest_path" \ + --image-reference "$IMAGE_REFERENCE" \ + --image-digest "$IMAGE_DIGEST" \ + --image-kind "$IMAGE_KIND" \ + --rapids-version "$RAPIDS_VER" \ + --cuda-version "$CUDA_VER" \ + --python-version "$PYTHON_VER" \ + --platform "$IMAGE_PLATFORM" \ + --source-repository "${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}" \ + --source-commit "$GITHUB_SHA" \ + --workflow-ref "${GITHUB_WORKFLOW_REF:-$GITHUB_WORKFLOW}" \ + --workflow-run-url "$workflow_run_url" \ + --conda-meta-dir "$PROVENANCE_OUTPUT_DIR/conda-meta" \ + "${generator_args[@]}" + +attached_manifest="$PROVENANCE_OUTPUT_DIR/attached-image-provenance.json" +( + cd "$PROVENANCE_OUTPUT_DIR" + oras attach \ + --artifact-type application/vnd.rapids.image.provenance.v1+json \ + --export-manifest "$attached_manifest" \ + "${registry_reference%@*}@${IMAGE_DIGEST}" \ + "image-provenance.json:application/vnd.rapids.image.provenance.v1+json" +) + +artifact_digest="sha256:$(sha256sum "$attached_manifest" | awk '{print $1}')" +cosign sign --yes "${registry_reference%@*}@${artifact_digest}" diff --git a/ci/release/update-version.sh b/ci/release/update-version.sh index d480f942..c5392dc2 100755 --- a/ci/release/update-version.sh +++ b/ci/release/update-version.sh @@ -1,5 +1,5 @@ #!/bin/bash -# Copyright (c) 2023-2026, NVIDIA CORPORATION. +# Copyright (c) 2023-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. ## Usage # Primary interface: bash update-version.sh [--run-context=main|release] @@ -76,11 +76,11 @@ NEXT_SHORT_TAG=${NEXT_MAJOR}.${NEXT_MINOR} # Set branch references based on RUN_CONTEXT if [[ "${RUN_CONTEXT}" == "main" ]]; then - RAPIDS_BRANCH_NAME="main" + RAPIDS_NOTEBOOKS_REF="main" WORKFLOW_BRANCH_REF="main" echo "Preparing development branch update $CURRENT_TAG => $NEXT_FULL_TAG (targeting main branch)" elif [[ "${RUN_CONTEXT}" == "release" ]]; then - RAPIDS_BRANCH_NAME="release/${NEXT_SHORT_TAG}" + RAPIDS_NOTEBOOKS_REF="release/${NEXT_SHORT_TAG}" WORKFLOW_BRANCH_REF="release/${NEXT_SHORT_TAG}" echo "Preparing release branch update $CURRENT_TAG => $NEXT_FULL_TAG (targeting release/${NEXT_SHORT_TAG} branch)" fi @@ -96,9 +96,9 @@ done sed_runner "s/com\.nvidia\.workbench\.image-version=.*/com.nvidia.workbench.image-version=\"${NEXT_FULL_TAG}\"/g" Dockerfile -# Dockerfile RAPIDS_BRANCH -sed_runner "s|ARG RAPIDS_BRANCH=\"release/[0-9]\+\.[0-9]\+\"|ARG RAPIDS_BRANCH=\"${RAPIDS_BRANCH_NAME}\"|g" Dockerfile -sed_runner "s|ARG RAPIDS_BRANCH=\"main\"|ARG RAPIDS_BRANCH=\"${RAPIDS_BRANCH_NAME}\"|g" Dockerfile +# Dockerfile RAPIDS_NOTEBOOKS_REF +sed_runner "s|ARG RAPIDS_NOTEBOOKS_REF=\"release/[0-9]\+\.[0-9]\+\"|ARG RAPIDS_NOTEBOOKS_REF=\"${RAPIDS_NOTEBOOKS_REF}\"|g" Dockerfile +sed_runner "s|ARG RAPIDS_NOTEBOOKS_REF=\"main\"|ARG RAPIDS_NOTEBOOKS_REF=\"${RAPIDS_NOTEBOOKS_REF}\"|g" Dockerfile # docs sed_runner "s|RAPIDS_VER=[[:digit:]]\+\.[[:digit:]]|RAPIDS_VER=${NEXT_SHORT_TAG}|g" CONTRIBUTING.md diff --git a/context/notebooks.sh b/context/notebooks.sh index 5677cc29..f442a331 100755 --- a/context/notebooks.sh +++ b/context/notebooks.sh @@ -1,9 +1,9 @@ #!/usr/bin/env bash -# Copyright (c) 2023-2026, NVIDIA CORPORATION. +# Copyright (c) 2023-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # Clones repos with notebooks & compiles notebook test dependencies # Requires environment variables: -# RAPIDS_BRANCH +# RAPIDS_NOTEBOOKS_REF # CUDA_VER # PYTHON_VER @@ -14,7 +14,7 @@ NOTEBOOK_REPOS=(cudf cuml cugraph) mkdir -p /notebooks /dependencies for REPO in "${NOTEBOOK_REPOS[@]}"; do echo "Cloning $REPO..." - git clone -b "${RAPIDS_BRANCH}" --depth 1 --single-branch "https://github.com/rapidsai/$REPO" "$REPO" + git clone -b "${RAPIDS_NOTEBOOKS_REF}" --depth 1 --single-branch "https://github.com/rapidsai/$REPO" "$REPO" SOURCE="$REPO/notebooks" DESTINATION="/notebooks/$REPO" diff --git a/requirements-test.txt b/requirements-test.txt new file mode 100644 index 00000000..2c78728c --- /dev/null +++ b/requirements-test.txt @@ -0,0 +1 @@ +pytest==8.3.5 diff --git a/tests/test_image_provenance_manifest.py b/tests/test_image_provenance_manifest.py new file mode 100644 index 00000000..4999ca6e --- /dev/null +++ b/tests/test_image_provenance_manifest.py @@ -0,0 +1,127 @@ +# Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# ruff: noqa: S101 + +from __future__ import annotations + +import importlib.util +import json +import sys +from pathlib import Path + +import pytest + + +MODULE_PATH = Path(__file__).parents[1] / "ci" / "image_provenance_manifest.py" +REPOSITORY_ROOT = MODULE_PATH.parents[1] +SPEC = importlib.util.spec_from_file_location("image_provenance_manifest", MODULE_PATH) +assert SPEC is not None and SPEC.loader is not None +MODULE = importlib.util.module_from_spec(SPEC) +sys.modules[SPEC.name] = MODULE +SPEC.loader.exec_module(MODULE) + + +def manifest_context(**overrides: object) -> object: + """Create valid provenance inputs, with targeted test overrides.""" + defaults = { + "image_reference": "rapidsai/base:26.08-cuda12-py312-amd64", + "image_digest": "sha256:image", + "image_kind": "base", + "rapids_version": "26.08", + "cuda_version": "12", + "python_version": "3.12", + "platform": "linux/amd64", + "source_repository": "https://github.com/rapidsai/docker", + "source_commit": "abc123", + "workflow_ref": ".github/workflows/build-rapids-image.yml@abc123", + "workflow_run_url": "https://github.com/rapidsai/docker/actions/runs/1", + "build_args": ["PYTHON_VER=3.12", "CUDA_VER=12"], + } + defaults.update(overrides) + return MODULE.ManifestContext(**defaults) + + +def test_build_manifest_records_exact_conda_package_facts(tmp_path: Path) -> None: + metadata_dir = tmp_path / "conda-meta" + metadata_dir.mkdir() + (metadata_dir / "c-ares-1.34.8-h1.json").write_text( + json.dumps( + { + "name": "c-ares", + "version": "1.34.8", + "build": "h1", + "build_number": 2, + "channel": "https://conda.anaconda.org/conda-forge", + "url": "https://conda.anaconda.org/conda-forge/linux-64/c-ares.conda", + } + ) + ) + + manifest = MODULE.build_manifest( + manifest_context(), + metadata_dir=metadata_dir, + ) + + assert manifest["subject"]["platform"] == { + "os": "linux", + "architecture": "amd64", + } + assert manifest["conda_packages"] == [ + { + "build": "h1", + "build_number": 2, + "channel": "https://conda.anaconda.org/conda-forge", + "name": "c-ares", + "purl_source": "unmapped", + "purls": [], + "url": "https://conda.anaconda.org/conda-forge/linux-64/c-ares.conda", + "version": "1.34.8", + } + ] + assert manifest["build"]["build_args"] == {"CUDA_VER": "12", "PYTHON_VER": "3.12"} + assert manifest["conda_packages_sha256"].startswith("sha256:") + + +def test_build_manifest_links_platform_manifests_for_multiarch_images() -> None: + manifest = MODULE.build_manifest( + manifest_context( + image_reference="rapidsai/base:26.08-cuda12-py312", + image_digest="sha256:index", + platform="multiarch", + workflow_ref="workflow", + workflow_run_url="run", + build_args=[], + ), + platform_manifests=[ + "linux/arm64|rapidsai/base:tag-arm64|sha256:arm", + "linux/amd64|rapidsai/base:tag-amd64|sha256:amd", + ], + ) + + assert "platform" not in manifest["subject"] + assert [item["platform"] for item in manifest["platform_manifests"]] == [ + "linux/amd64", + "linux/arm64", + ] + + +def test_invalid_build_argument_is_rejected() -> None: + with pytest.raises(ValueError, match="NAME=VALUE"): + MODULE.parse_build_args(["CUDA_VER"]) + + +@pytest.mark.parametrize( + ("script_name", "safe_title"), + [ + ("publish-image-provenance.sh", "image-provenance.json"), + ("publish-image-provenance-index.sh", "image-provenance-index.json"), + ], +) +def test_published_provenance_uses_safe_oci_layer_titles( + script_name: str, + safe_title: str, +) -> None: + script = (REPOSITORY_ROOT / "ci" / script_name).read_text() + + assert "--disable-path-validation" not in script + assert f'"{safe_title}:application/vnd.rapids.image.provenance' in script