Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
69 changes: 31 additions & 38 deletions .github/workflows/build.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -505,46 +505,39 @@ jobs:
push: true
cuda: '["12.9", "13.3"]'
# Build the cuDF Java JAR for every Maven classifier.
java-build:
java-build-matrix:
needs: [telemetry-setup]
strategy:
fail-fast: false
matrix:
include:
- { cuda: "12.9", cuda_major: "12", arch: "x86_64", runner: "linux-amd64-cpu16" }
- { cuda: "13.3", cuda_major: "13", arch: "x86_64", runner: "linux-amd64-cpu16" }
- { cuda: "12.9", cuda_major: "12", arch: "aarch64", runner: "linux-arm64-cpu16" }
- { cuda: "13.3", cuda_major: "13", arch: "aarch64", runner: "linux-arm64-cpu16" }
runs-on: ${{ matrix.runner }}
permissions:
contents: read
steps:
- name: Checkout code repo
uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1
with:
ref: ${{ inputs.sha }}
fetch-depth: 0
persist-credentials: false
- name: Build static libcudf
run: |
./java/ci/build_static_libcudf.sh \
--output-dir "${RUNNER_TEMP}/libcudf" \
--cuda-version "${{ matrix.cuda }}"
- name: Build cuDF Java JAR
run: |
./java/ci/build_cudf_java_jar.sh \
--libcudf-dir "${RUNNER_TEMP}/libcudf" \
--output-dir "${RUNNER_TEMP}/jars" \
--cuda-version "${{ matrix.cuda }}"
- name: Upload per-entry JAR artifact
uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
with:
name: cudf_java_cuda${{ matrix.cuda_major }}_${{ matrix.arch }}
# Ship only the JARs and POMs. Exclude the per-classifier Maven build scratch dir.
path: |
${{ runner.temp }}/jars
!${{ runner.temp }}/jars/.mvn-temp-target
if-no-files-found: error
uses: rapidsai/shared-workflows/.github/workflows/compute-matrix.yaml@main
with:
build_type: ${{ inputs.build_type || 'branch' }}
matrix_name: conda-cpp-build
java-build:
needs: [build-details, telemetry-setup, java-build-matrix]
permissions:
actions: read
contents: read
id-token: write
packages: read
pull-requests: read
secrets: inherit # zizmor: ignore[secrets-inherit]
uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@main
strategy:
fail-fast: false
matrix: ${{ fromJSON(needs.java-build-matrix.outputs.matrix) }}
with:
build_type: ${{ inputs.build_type || 'branch' }}
branch: ${{ inputs.branch }}
date: ${{ inputs.date }}
sha: ${{ inputs.sha }}
build-datetime: ${{ needs.build-details.outputs.build-datetime }}
arch: ${{ matrix.ARCH }}
node_type: cpu16
container_image: "rapidsai/ci-wheel:26.10-cuda${{ matrix.CUDA_VER }}-${{ matrix.LINUX_VER }}-py${{ matrix.PY_VER }}"
script: "env RAPIDS_CUDA_VERSION=${{ matrix.CUDA_VER }} ci/build_java.sh"
file_to_upload: output_jars
artifact-name: java_pkg_${{ matrix.CUDA_VER }}_${{ matrix.ARCH }}
# Assemble the per-classifier JARs into one Maven-repository layout.
java-gather:
needs: [java-build]
Expand All @@ -560,7 +553,7 @@ jobs:
- name: Download per-entry JAR artifacts
uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0
with:
pattern: cudf_java_cuda*
pattern: java_pkg_*
path: ${{ runner.temp }}/jars
merge-multiple: true
- name: Assemble Maven repository layout
Expand Down
72 changes: 72 additions & 0 deletions .github/workflows/pr.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,9 @@ jobs:
- conda-python-cudf-tests
- conda-python-other-tests
- conda-java-tests
- java-build-matrix
- java-build
- java-tests
- conda-notebook-tests
- docs-build
- publish-api-docs
Expand Down Expand Up @@ -526,6 +529,75 @@ jobs:
arch: "amd64"
container_image: "rapidsai/ci-conda:26.10-latest"
script: "ci/test_java.sh"
# Build and test the cuDF Java JAR for every Maven classifier.
java-build-matrix:
needs: [checks]
permissions:
contents: read
uses: rapidsai/shared-workflows/.github/workflows/compute-matrix.yaml@main
with:
build_type: pull-request
matrix_name: conda-cpp-build
java-build:
needs: [build-details, java-build-matrix, changed-files]
permissions:
actions: read
contents: read
id-token: write
packages: read
pull-requests: read
secrets: inherit # zizmor: ignore[secrets-inherit]
uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@main
if: fromJSON(needs.changed-files.outputs.changed_file_groups).test_java
strategy:
fail-fast: false
matrix: ${{ fromJSON(needs.java-build-matrix.outputs.matrix) }}
with:
build_type: pull-request
build-datetime: ${{ needs.build-details.outputs.build-datetime }}
arch: ${{ matrix.ARCH }}
node_type: cpu16
container_image: "rapidsai/ci-wheel:26.10-cuda${{ matrix.CUDA_VER }}-${{ matrix.LINUX_VER }}-py${{ matrix.PY_VER }}"
script: "env RAPIDS_CUDA_VERSION=${{ matrix.CUDA_VER }} ci/build_java.sh"
file_to_upload: output_jars
artifact-name: java_pkg_${{ matrix.CUDA_VER }}_${{ matrix.ARCH }}
java-tests:
needs: [java-build, java-build-matrix]
if: ${{ !cancelled() && needs.java-build.result == 'success' }}
permissions:
actions: read
contents: read
packages: read
strategy:
fail-fast: false
matrix: ${{ fromJSON(needs.java-build-matrix.outputs.matrix) }}
runs-on: linux-${{ matrix.ARCH }}-gpu-l4-latest-1
container:
image: rapidsai/ci-wheel:26.10-cuda${{ matrix.CUDA_VER }}-${{ matrix.LINUX_VER }}-py${{ matrix.PY_VER }} # zizmor: ignore[unpinned-images]
env:
NVIDIA_VISIBLE_DEVICES: ${{ env.NVIDIA_VISIBLE_DEVICES }}
defaults:
run:
shell: bash
steps:
- name: Checkout
uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1
with:
persist-credentials: false
- name: Download java-build artifact
uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0
with:
name: java_pkg_${{ matrix.CUDA_VER }}_${{ matrix.ARCH }}
path: java_pkg
- name: Run Java tests
env:
RAPIDS_CUDA_VERSION: ${{ matrix.CUDA_VER }}
LIBCUDF_LARGE_STRINGS_ENABLED: "0"
run: |
. java/ci/java_classifier.sh
JAVA_JAR="$(cudf_java_resolve_artifact_jar java_pkg)"
export JAVA_JAR
ci/test_packaged_java.sh
conda-notebook-tests:
needs: [conda-python-build, conda-python-build-noarch, changed-files]
permissions:
Expand Down
60 changes: 60 additions & 0 deletions ci/build_java.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,60 @@
#!/bin/bash
# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
# SPDX-License-Identifier: Apache-2.0

# CI entrypoint: static libcudf + classifier JAR for one matrix cell.
#
# Builds the static libcudf install tree and packages the matching classifier
# JAR, writing ./output_jars/<classifier>/ for the custom-job upload step.
#
# Inputs (environment variables):
# RAPIDS_CUDA_VERSION CUDA version, e.g. 12.9.2 (required).
# PARALLEL_LEVEL Build parallelism (default: nproc).
# CMAKE_CUDA_ARCHITECTURES Optional override for -DCMAKE_CUDA_ARCHITECTURES.
# JAVA_WORK_DIR Optional scratch dir (default: <repo>/.java-work).

set -euo pipefail

REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
cd "${REPO_ROOT}"

# shellcheck disable=SC1091
. "${REPO_ROOT}/java/ci/ci_wheel_image.sh"
# shellcheck disable=SC1091
. "${REPO_ROOT}/java/ci/java_classifier.sh"

if [[ -z ${RAPIDS_CUDA_VERSION:-} ]]; then
echo "Error: RAPIDS_CUDA_VERSION must be set" >&2
exit 1
fi

RAPIDS_CUDA_VERSION="$(cudf_java_normalize_cuda_version "${RAPIDS_CUDA_VERSION}")"
export RAPIDS_CUDA_VERSION
CLASSIFIER="$(cudf_java_maven_classifier "${RAPIDS_CUDA_VERSION}")"

WORK_DIR="${JAVA_WORK_DIR:-${REPO_ROOT}/.java-work}"
LIBCUDF_DIR="${WORK_DIR}/libcudf"
CLASSIFIER_OUT="${REPO_ROOT}/output_jars/${CLASSIFIER}"

cleanup_scratch() {
rm -rf "${WORK_DIR}"
}
trap cleanup_scratch EXIT

rm -rf "${LIBCUDF_DIR}" "${CLASSIFIER_OUT}"
mkdir -p "${LIBCUDF_DIR}" "${CLASSIFIER_OUT}"

export REPO_ROOT
export INSTALL_PREFIX="${LIBCUDF_DIR}"
export BUILD_DIR="${WORK_DIR}/libcudf-build"
export CUDF_INSTALL_DIR="${LIBCUDF_DIR}"
export OUTPUT_DIR="${CLASSIFIER_OUT}"

rapids-logger "Building static libcudf (${CLASSIFIER})"
bash "${REPO_ROOT}/java/ci/build_static_libcudf_in_container.sh"

rapids-logger "Packaging cuDF Java JAR (${CLASSIFIER})"
bash "${REPO_ROOT}/java/ci/build_cudf_java_jar_in_container.sh"

cudf_java_assert_classifier_artifacts "${CLASSIFIER_OUT}" "${CLASSIFIER}"
ls -la "${CLASSIFIER_OUT}"
47 changes: 47 additions & 0 deletions ci/test_packaged_java.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,47 @@
#!/bin/bash
# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
# SPDX-License-Identifier: Apache-2.0

# Run the Java tests against an already-packaged classifier JAR.
#
# Activates -Ppackaged-jar-tests so the surefire classpath uses the JAR at
# JAVA_JAR instead of a locally compiled target/classes tree. Caller places
# the JAR (e.g. via actions/download-artifact) before invoking this script.
#
# Inputs (environment variables):
# JAVA_JAR Absolute path to the classifier JAR (required).
# LIBCUDF_LARGE_STRINGS_ENABLED Optional; defaults to 0 (same as ci/test_java.sh).

set -euo pipefail

REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
cd "${REPO_ROOT}"

if [[ -z ${JAVA_JAR:-} || ! -f ${JAVA_JAR} ]]; then
echo "Error: JAVA_JAR must point to an existing classifier JAR" >&2
exit 1
fi

if ! command -v mvn >/dev/null 2>&1 || ! command -v java >/dev/null 2>&1; then
# shellcheck disable=SC1091
. "${REPO_ROOT}/java/ci/setup_java_env.sh"
fi

# Match the existing conda Java test entrypoint in ci/test_java.sh.
export LIBCUDF_LARGE_STRINGS_ENABLED="${LIBCUDF_LARGE_STRINGS_ENABLED:-0}"

PRODUCT_JAR="$(cd "$(dirname "${JAVA_JAR}")" && pwd)/$(basename "${JAVA_JAR}")"
rapids-logger "Product JAR: ${PRODUCT_JAR}"

rm -rf "${REPO_ROOT}/java/target"

pushd "${REPO_ROOT}/java" >/dev/null
set +e
timeout 30m mvn -B test -Ppackaged-jar-tests \
"-Dcudf.jar.path=${PRODUCT_JAR}" \
"-DCUDF_JNI_ENABLE_PROFILING=OFF"
EXITCODE=$?
set -e
popd >/dev/null
rapids-logger "Java tests exit=${EXITCODE}"
exit "${EXITCODE}"
15 changes: 13 additions & 2 deletions java/ci/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -4,14 +4,14 @@

The scripts under `java/ci/` build the cuDF Java JAR for every Maven classifier the
same way locally and in CI (GitHub Actions is only a thin wrapper that adds
artifact upload/download). Each script pulls the RAPIDS `ci-conda` build image,
artifact upload/download). Each script pulls the RAPIDS `ci-wheel` build image,
runs the build in a throwaway container, and writes its output to a host
directory. No local `docker build` is required, and no GPU is required to build.

### Prerequisites

1. Docker is installed and the current user can run `docker`.
2. Network access to pull `rapidsai/ci-conda:<rapids_version>-latest`.
2. Network access to pull `rapidsai/ci-wheel:<rapids>-cuda<ver>-rockylinux8-py3.11`.

### Local one-command shortcut

Expand Down Expand Up @@ -90,6 +90,17 @@ per-entry artifact. The separate `java-gather` job downloads them (with
`merge-multiple: true`, so all subdirs land in a single parent dir), runs
Step 3, and uploads the combined `cudf_java_maven_repo` artifact.

### Packaging-aware tests (local)

Plain `cd java && mvn test` does not exercise the classifier JAR. Use
`java/ci/test_packaged_java_local.sh` (or CI entrypoint
`ci/test_packaged_java.sh`) to run the existing Java tests against a packaged
JAR. Needs Docker + GPU.

```bash
./java/ci/test_packaged_java_local.sh --work-dir /tmp/java-build-test
```

## Legacy: manual Dockerfile.rocky build (obsolete)

> The `java/ci/Dockerfile.rocky` + `java/ci/build-in-docker.sh` flow below is the
Expand Down
2 changes: 1 addition & 1 deletion java/ci/build-in-docker.sh
Original file line number Diff line number Diff line change
Expand Up @@ -69,7 +69,7 @@ BUILD_ARG=(
"-DCUDF_USE_PER_THREAD_DEFAULT_STREAM=$ENABLE_PTDS"
"-DCUDF_JNI_LIBCUDF_STATIC=ON"
"-DUSE_GDS=$ENABLE_GDS"
"-Dtest=*,!CuFileTest,!CudaFatalTest,!ColumnViewNonEmptyNullsTest,!NativeDepsLoaderTest"
"-Dtest=*,!CuFileTest,!CudaFatalTest,!ColumnViewNonEmptyNullsTest,!NativeDepsLoaderTest,!PackagedJarOriginCheck"
)

if [ "$SIGN_FILE" == true ]; then
Expand Down
Loading
Loading