Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
60 changes: 59 additions & 1 deletion .github/workflows/build-rapids-image.yml
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
# Copyright (c) 2025-2026, NVIDIA CORPORATION.
# Copyright (c) 2025-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
name: build-rapids

on:
Expand All @@ -25,13 +25,20 @@ on:
RAPIDS_VER:
required: true
type: string
RAPIDS_NOTEBOOKS_REF:
required: true
type: string
BASE_TAG:
required: true
type: string
NOTEBOOKS_TAG:
required: true
type: string

permissions:
contents: read
id-token: write

env:
# prevent buildx from creating arch-specific manifest lists...
# we want single images, that are later referenced together in a multiarch manifest
Expand Down Expand Up @@ -75,6 +82,18 @@ jobs:
with:
username: ${{ secrets.GPUCIBOT_DOCKERHUB_USER }}
password: ${{ secrets.GPUCIBOT_DOCKERHUB_TOKEN }}
- name: Install Go
uses: actions/setup-go@4a3601121dd01d1626a1e23e37211e3254c1c06c # v6
with:
go-version: '1.25.x'
cache: false
- name: Install ORAS
run: |
GOBIN="${RUNNER_TEMP}/oras-bin" go install oras.land/oras/cmd/oras@v1.3.0
echo "${RUNNER_TEMP}/oras-bin" >> "${GITHUB_PATH}"
"${RUNNER_TEMP}/oras-bin/oras" version
- name: Set up Cosign
uses: sigstore/cosign-installer@f713795cb21599bc4e5c4b58cbad1da852d7eeb9 # v3
- name: Set up Docker Context for Buildx
id: buildx-context
run: |
Expand All @@ -99,8 +118,10 @@ jobs:
LINUX_DISTRO_VER: ${{ inputs.LINUX_DISTRO_VER }}
LINUX_VER: ${{ inputs.LINUX_VER }}
PYTHON_VER: ${{ inputs.PYTHON_VER }}
RAPIDS_NOTEBOOKS_REF: ${{ inputs.RAPIDS_NOTEBOOKS_REF }}
RAPIDS_VER: ${{ inputs.RAPIDS_VER }}
- name: Build base image
id: build-base
uses: docker/build-push-action@10e90e3645eae34f1e60eeb005ba3a3d33f178e8 # v6
with:
context: context
Expand All @@ -113,7 +134,26 @@ jobs:
tags: ${{ inputs.BASE_TAG }}-${{ matrix.ARCH }}
# ensure only OCI mediatypes are used: https://docs.docker.com/build/exporters/#oci-media-types
outputs: type=registry,oci-mediatypes=true
- name: Export base image package provenance
env:
DOCKER_BUILD_ARGS: ${{ steps.generate-build-args.outputs.DOCKER_BUILD_ARGS }}
PROVENANCE_OUTPUT_DIR: ${{ runner.temp }}/provenance-base-${{ matrix.ARCH }}
PROVENANCE_TARGET: provenance-base
run: ci/export-image-provenance.sh
- name: Publish base image provenance
env:
CUDA_VER: ${{ inputs.CUDA_VER }}
DOCKER_BUILD_ARGS: ${{ steps.generate-build-args.outputs.DOCKER_BUILD_ARGS }}
IMAGE_DIGEST: ${{ steps.build-base.outputs.digest }}
IMAGE_KIND: base
IMAGE_PLATFORM: linux/${{ matrix.ARCH }}
IMAGE_REFERENCE: ${{ inputs.BASE_TAG }}-${{ matrix.ARCH }}
PROVENANCE_OUTPUT_DIR: ${{ runner.temp }}/provenance-base-${{ matrix.ARCH }}
PYTHON_VER: ${{ inputs.PYTHON_VER }}
RAPIDS_VER: ${{ inputs.RAPIDS_VER }}
run: ci/publish-image-provenance.sh
- name: Build notebooks image
id: build-notebooks
uses: docker/build-push-action@10e90e3645eae34f1e60eeb005ba3a3d33f178e8 # v6
with:
context: context
Expand All @@ -126,3 +166,21 @@ jobs:
tags: ${{ inputs.NOTEBOOKS_TAG }}-${{ matrix.ARCH }}
# ensure only OCI mediatypes are used: https://docs.docker.com/build/exporters/#oci-media-types
outputs: type=registry,oci-mediatypes=true
- name: Export notebooks image package provenance
env:
DOCKER_BUILD_ARGS: ${{ steps.generate-build-args.outputs.DOCKER_BUILD_ARGS }}
PROVENANCE_OUTPUT_DIR: ${{ runner.temp }}/provenance-notebooks-${{ matrix.ARCH }}
PROVENANCE_TARGET: provenance-notebooks
run: ci/export-image-provenance.sh
- name: Publish notebooks image provenance
env:
CUDA_VER: ${{ inputs.CUDA_VER }}
DOCKER_BUILD_ARGS: ${{ steps.generate-build-args.outputs.DOCKER_BUILD_ARGS }}
IMAGE_DIGEST: ${{ steps.build-notebooks.outputs.digest }}
IMAGE_KIND: notebooks
IMAGE_PLATFORM: linux/${{ matrix.ARCH }}
IMAGE_REFERENCE: ${{ inputs.NOTEBOOKS_TAG }}-${{ matrix.ARCH }}
PROVENANCE_OUTPUT_DIR: ${{ runner.temp }}/provenance-notebooks-${{ matrix.ARCH }}
PYTHON_VER: ${{ inputs.PYTHON_VER }}
RAPIDS_VER: ${{ inputs.RAPIDS_VER }}
run: ci/publish-image-provenance.sh
65 changes: 65 additions & 0 deletions .github/workflows/build-test-publish-images.yml
Original file line number Diff line number Diff line change
Expand Up @@ -55,6 +55,10 @@ jobs:
- name: Run hadolint
run: |
ci/lint-dockerfiles.sh
- name: Test image provenance manifest generator
run: |
python3 -m pip install --disable-pip-version-check -r requirements-test.txt
python3 -m pytest -q tests
compute-matrix:
runs-on: ubuntu-latest
outputs:
Expand All @@ -68,6 +72,7 @@ jobs:
NOTEBOOKS_TAG_PREFIX: ${{ steps.compute-tag-prefix.outputs.NOTEBOOKS_TAG_PREFIX }}
CUVS_BENCH_TAG_PREFIX: ${{ steps.compute-tag-prefix.outputs.CUVS_BENCH_TAG_PREFIX }}
CUVS_BENCH_CPU_TAG_PREFIX: ${{ steps.compute-tag-prefix.outputs.CUVS_BENCH_CPU_TAG_PREFIX }}
RAPIDS_NOTEBOOKS_REF: ${{ steps.compute-rapids-ver.outputs.RAPIDS_NOTEBOOKS_REF }}
RAPIDS_VER: ${{ steps.compute-rapids-ver.outputs.RAPIDS_VER }}
ALPHA_TAG: ${{ steps.compute-rapids-ver.outputs.ALPHA_TAG }}
steps:
Expand Down Expand Up @@ -119,6 +124,8 @@ jobs:
echo "CUVS_BENCH_CPU_IMAGE_REPO=${cuvs_bench_cpu_repo}" | tee -a ${GITHUB_OUTPUT}
- name: Compute RAPIDS_VER
id: compute-rapids-ver
env:
BUILD_TYPE: ${{ inputs.build_type }}
run: |
GIT_DESCRIBE_TAG="$(git describe --tags --first-parent --abbrev=0)"
GIT_DESCRIBE_TAG="${GIT_DESCRIBE_TAG:1}" # remove leading 'v'
Expand All @@ -128,7 +135,20 @@ jobs:
ALPHA_TAG="a"
fi
RAPIDS_VER="$(echo $GIT_DESCRIBE_TAG | awk 'BEGIN{FS=OFS="."} NF--')" # Convert full tag to YY.MM
RAPIDS_NOTEBOOKS_REF="main"
if [[ ! $GIT_DESCRIBE_TAG =~ [a-z] ]]; then
RAPIDS_NOTEBOOKS_REF="release/${RAPIDS_VER}"
fi
# The source branch advances before its packages are published. Keep
# pull-request images on a matching published package and source line.
if [[ $BUILD_TYPE == "pull-request" ]]; then
RAPIDS_VER="26.08"
RAPIDS_NOTEBOOKS_REF="release/26.08"
ALPHA_TAG="a"
fi
echo "RAPIDS_NOTEBOOKS_REF=${RAPIDS_NOTEBOOKS_REF}" | tee -a ${GITHUB_OUTPUT}
echo "RAPIDS_VER=${RAPIDS_VER}" | tee -a ${GITHUB_OUTPUT}
echo "ALPHA_TAG=${ALPHA_TAG}" | tee -a ${GITHUB_OUTPUT}
- name: Compute test matrix
Expand Down Expand Up @@ -159,6 +179,7 @@ jobs:
LINUX_DISTRO_VER: ${{ matrix.LINUX_DISTRO_VER }}
LINUX_VER: ${{ matrix.LINUX_VER }}
PYTHON_VER: ${{ matrix.PYTHON_VER }}
RAPIDS_NOTEBOOKS_REF: ${{ needs.compute-matrix.outputs.RAPIDS_NOTEBOOKS_REF }}
RAPIDS_VER: ${{ needs.compute-matrix.outputs.RAPIDS_VER }}
BASE_TAG:
"rapidsai/${{ needs.compute-matrix.outputs.BASE_IMAGE_REPO }}:\
Expand Down Expand Up @@ -224,6 +245,18 @@ jobs:
with:
username: ${{ secrets.GPUCIBOT_DOCKERHUB_USER }}
password: ${{ secrets.GPUCIBOT_DOCKERHUB_TOKEN }}
- name: Install Go
uses: actions/setup-go@4a3601121dd01d1626a1e23e37211e3254c1c06c # v6
with:
go-version: '1.25.x'
cache: false
- name: Install ORAS
run: |
GOBIN="${RUNNER_TEMP}/oras-bin" go install oras.land/oras/cmd/oras@v1.3.0
echo "${RUNNER_TEMP}/oras-bin" >> "${GITHUB_PATH}"
"${RUNNER_TEMP}/oras-bin/oras" version
- name: Set up Cosign
uses: sigstore/cosign-installer@f713795cb21599bc4e5c4b58cbad1da852d7eeb9 # v3
- name: Create multiarch manifest
shell: bash
env:
Expand All @@ -239,6 +272,38 @@ jobs:
GPUCIBOT_DOCKERHUB_TOKEN: ${{ secrets.GPUCIBOT_DOCKERHUB_TOKEN }}
ARCHES: ${{ toJSON(matrix.ARCHES) }}
run: ci/create-rapids-multiarch-manifest.sh
- name: Publish multiarch image provenance indexes
shell: bash
env:
BASE_IMAGE_REPO: ${{ needs.compute-matrix.outputs.BASE_IMAGE_REPO }}
BASE_TAG_PREFIX: ${{ needs.compute-matrix.outputs.BASE_TAG_PREFIX }}
RAPIDS_VER: ${{ needs.compute-matrix.outputs.RAPIDS_VER }}
ALPHA_TAG: ${{ needs.compute-matrix.outputs.ALPHA_TAG }}
CUDA_TAG: ${{ matrix.CUDA_TAG }}
PYTHON_VER: ${{ matrix.PYTHON_VER }}
NOTEBOOKS_IMAGE_REPO: ${{ needs.compute-matrix.outputs.NOTEBOOKS_IMAGE_REPO }}
NOTEBOOKS_TAG_PREFIX: ${{ needs.compute-matrix.outputs.NOTEBOOKS_TAG_PREFIX }}
ARCHES: ${{ toJSON(matrix.ARCHES) }}
run: |
base_reference="rapidsai/${BASE_IMAGE_REPO}:${BASE_TAG_PREFIX}${RAPIDS_VER}${ALPHA_TAG}-cuda${CUDA_TAG}-py${PYTHON_VER}"
notebooks_reference="rapidsai/${NOTEBOOKS_IMAGE_REPO}:${NOTEBOOKS_TAG_PREFIX}${RAPIDS_VER}${ALPHA_TAG}-cuda${CUDA_TAG}-py${PYTHON_VER}"
for image_kind in base notebooks; do
if [[ "$image_kind" == base ]]; then
image_reference="$base_reference"
else
image_reference="$notebooks_reference"
fi
platform_references="$({
while IFS= read -r arch; do
printf 'linux/%s=%s-%s\n' "$arch" "$image_reference" "$arch"
done < <(jq -r '.[]' <<< "$ARCHES")
})"
IMAGE_KIND="$image_kind" \
IMAGE_REFERENCE="$image_reference" \
PLATFORM_REFERENCES="$platform_references" \
CUDA_VER="$CUDA_TAG" \
ci/publish-image-provenance-index.sh
done
build-cuvs-multiarch-manifest:
needs: [build-cuvs, compute-matrix]
strategy:
Expand Down
2 changes: 2 additions & 0 deletions .gitignore
Original file line number Diff line number Diff line change
@@ -1,4 +1,6 @@
.vscode
__pycache__/
*.py[cod]

*.a
*.bz2
Expand Down
15 changes: 14 additions & 1 deletion Dockerfile
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@ FROM python:${PYTHON_VER} AS dependencies
ARG CPU_ARCH=notset
ARG CUDA_VER=notset
ARG PYTHON_VER=notset
ARG RAPIDS_BRANCH="main"
ARG RAPIDS_NOTEBOOKS_REF="main"
ARG RAPIDS_VER=26.10
ARG YQ_VER=notset

Expand Down Expand Up @@ -258,6 +258,7 @@ LABEL com.nvidia.workbench.os="linux"
LABEL com.nvidia.workbench.package-manager-environment.target="/opt/conda"
LABEL com.nvidia.workbench.package-manager-environment.type="conda"
LABEL com.nvidia.workbench.package-manager.apt.binary="/usr/bin/apt"

LABEL com.nvidia.workbench.package-manager.apt.installed-packages=""
LABEL com.nvidia.workbench.package-manager.conda3.binary="/opt/conda/bin/conda"
LABEL com.nvidia.workbench.package-manager.conda3.installed-packages="rapids cudf cuml cugraph rmm pylibraft cucim xgboost jupyterlab"
Expand All @@ -268,3 +269,15 @@ LABEL com.nvidia.workbench.schema-version="v2"
LABEL com.nvidia.workbench.user.gid="1000"
LABEL com.nvidia.workbench.user.uid="1001"
LABEL com.nvidia.workbench.user.username="rapids"

# Minimal BuildKit export targets used to publish image provenance without
# exporting or pulling the image filesystem. Each contains only conda package
# metadata copied from the corresponding final image target.
FROM scratch AS provenance-base
COPY --from=base /opt/conda/conda-meta /conda-meta

FROM scratch AS provenance-notebooks
COPY --from=notebooks /opt/conda/conda-meta /conda-meta

# Keep the default Docker build target runnable for local users.
FROM notebooks
23 changes: 23 additions & 0 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -39,6 +39,29 @@ The following environment variables can be passed to the `docker run` commands f
- `PIP_TIMEOUT` - how long (in seconds) the `pip` install should wait before exiting
- `UNQUOTE` - Whether the command line args to `docker run` should be [executed with or without being quoted](./context/entrypoint.sh). Default to false and it is unlikely that you need to change this.

## Image Provenance Manifests

Published `base` and `notebooks` images have an OCI referrer containing a
RAPIDS image provenance manifest. It is attached to the immutable image digest,
not to a mutable tag, and can be retrieved from the registry without downloading
image layers. The platform-specific manifest records the source commit and
workflow run plus the exact conda package name, version, build, channel, and
source URL taken from `conda-meta`. The multiarch image-index manifest links the
corresponding platform manifests.

Conda package records intentionally leave `purls` empty until an upstream pURL
mapping has been verified. Conda is a distribution format, not an upstream pURL
type, so consumers must not derive `pkg:conda/...` identifiers from these
records. The manifest uses the OCI artifact type
`application/vnd.rapids.image.provenance.v1+json` for platform images and
`application/vnd.rapids.image.provenance.index.v1+json` for multiarch indexes.

The build publishes these records with [ORAS](https://oras.land/), using an OCI
referrer to the image digest, and keylessly signs the attached artifact with
GitHub Actions OIDC. Scanner and triage tooling can discover the referrer,
retrieve its JSON, and compare the scanner-reported component version with the
build facts without pulling the image.

## Bind Mounts

Mounting files/folders to the locations specified below provide additional functionality for the images.
Expand Down
3 changes: 2 additions & 1 deletion ci/compute-build-args.sh
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@ LINUX_DISTRO: ${LINUX_DISTRO:-notset}
LINUX_DISTRO_VER: ${LINUX_DISTRO_VER:-notset}
LINUX_VER: ${LINUX_VER}
PYTHON_VER: ${PYTHON_VER}
RAPIDS_NOTEBOOKS_REF: ${RAPIDS_NOTEBOOKS_REF:-main}
RAPIDS_VER: ${RAPIDS_VER}
"
export ARGS
Expand All @@ -30,7 +31,7 @@ if [ -n "${GITHUB_ACTIONS:-}" ]; then
#
cat <<EOF > "${GITHUB_OUTPUT:-/dev/stdout}"
DOCKER_BUILD_ARGS<<EOT
$(yq -r '. + env(ARGS) | to_entries | map(.key + "=" + .value) | join(" \n")' versions.yaml)
$(yq -r '. + env(ARGS) | to_entries | map(.key + "=" + .value) | join("\n")' versions.yaml)
EOT
EOF
else
Expand Down
2 changes: 1 addition & 1 deletion ci/compute-matrix.sh
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
#!/bin/bash
# Copyright (c) 2023-2025, NVIDIA CORPORATION.
# Copyright (c) 2023-2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved.

set -euo pipefail

Expand Down
27 changes: 27 additions & 0 deletions ci/export-image-provenance.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,27 @@
#!/bin/bash
# Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
# SPDX-License-Identifier: Apache-2.0

set -eEuo pipefail

: "${PROVENANCE_TARGET:?Set PROVENANCE_TARGET to a Dockerfile provenance target}"
: "${PROVENANCE_OUTPUT_DIR:?Set PROVENANCE_OUTPUT_DIR for the local exporter}"
: "${DOCKER_BUILD_ARGS:?Set DOCKER_BUILD_ARGS from compute-build-args.sh}"

build_args=()
while IFS= read -r argument; do
[[ -z "$argument" ]] && continue
build_args+=(--build-arg "$argument")
done <<<"$DOCKER_BUILD_ARGS"

rm -rf "$PROVENANCE_OUTPUT_DIR"
mkdir -p "$PROVENANCE_OUTPUT_DIR"

docker buildx build \
--file Dockerfile \
--target "$PROVENANCE_TARGET" \
--output "type=local,dest=$PROVENANCE_OUTPUT_DIR" \
"${build_args[@]}" \
context

test -d "$PROVENANCE_OUTPUT_DIR/conda-meta"
Loading
Loading