From 8619d034bb2c35a81163930263d9ddddbb5a9fea Mon Sep 17 00:00:00 2001 From: Paul Aiyedun <53453937+paul-aiyedun@users.noreply.github.com> Date: Thu, 13 Aug 2026 05:05:59 +0000 Subject: [PATCH 01/10] Add Rocky ci-wheel Java packaging build and packaged-JAR tests * Move `java-build` onto RAPIDS `ci-wheel` (Rocky 8) via `custom-job` and the shared `conda-cpp-build` matrix, replacing the previous host-side ci-conda docker wrappers. * Add `java-build` and `java-tests` jobs that run the existing suite against the uploaded classifier JAR. * Introduce `-Ppackaged-jar-tests` so Surefire uses the packaged JAR instead of `target/classes`, with `PackagedJarOriginCheck` to fail fast if classes load from the wrong place. * Factor classifier/image/env helpers (`java_classifier.sh`, `ci_wheel_image.sh`, `setup_java_env.sh`) and in-container entrypoints so local wrappers and CI share one packaging path. --- .github/workflows/build.yaml | 69 ++++----- .github/workflows/pr.yaml | 71 +++++++++ ci/build_java.sh | 60 ++++++++ ci/test_packaged_java.sh | 47 ++++++ java/ci/README.md | 15 +- java/ci/build-in-docker.sh | 2 +- java/ci/build_cudf_java_jar.sh | 85 +++------- java/ci/build_cudf_java_jar_in_container.sh | 145 ++++++++---------- java/ci/build_static_libcudf.sh | 22 +-- java/ci/build_static_libcudf_in_container.sh | 99 ++++++------ java/ci/ci_wheel_image.sh | 36 +++++ java/ci/java_classifier.sh | 116 ++++++++++++++ java/ci/setup_java_env.sh | 128 ++++++++++++++++ java/ci/test_packaged_java_local.sh | 63 ++++++++ java/pom.xml | 93 ++++++++++- .../rapids/cudf/PackagedJarOriginCheck.java | 30 ++++ 16 files changed, 824 insertions(+), 257 deletions(-) create mode 100755 ci/build_java.sh create mode 100755 ci/test_packaged_java.sh create mode 100755 java/ci/ci_wheel_image.sh create mode 100644 java/ci/java_classifier.sh create mode 100644 java/ci/setup_java_env.sh create mode 100755 java/ci/test_packaged_java_local.sh create mode 100644 java/src/test/java/ai/rapids/cudf/PackagedJarOriginCheck.java diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index 920cceb0cb66..356592e8bf70 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -505,46 +505,39 @@ jobs: push: true cuda: '["12.9", "13.3"]' # Build the cuDF Java JAR for every Maven classifier. - java-build: + java-build-matrix: needs: [telemetry-setup] - strategy: - fail-fast: false - matrix: - include: - - { cuda: "12.9", cuda_major: "12", arch: "x86_64", runner: "linux-amd64-cpu16" } - - { cuda: "13.3", cuda_major: "13", arch: "x86_64", runner: "linux-amd64-cpu16" } - - { cuda: "12.9", cuda_major: "12", arch: "aarch64", runner: "linux-arm64-cpu16" } - - { cuda: "13.3", cuda_major: "13", arch: "aarch64", runner: "linux-arm64-cpu16" } - runs-on: ${{ matrix.runner }} permissions: contents: read - steps: - - name: Checkout code repo - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1 - with: - ref: ${{ inputs.sha }} - fetch-depth: 0 - persist-credentials: false - - name: Build static libcudf - run: | - ./java/ci/build_static_libcudf.sh \ - --output-dir "${RUNNER_TEMP}/libcudf" \ - --cuda-version "${{ matrix.cuda }}" - - name: Build cuDF Java JAR - run: | - ./java/ci/build_cudf_java_jar.sh \ - --libcudf-dir "${RUNNER_TEMP}/libcudf" \ - --output-dir "${RUNNER_TEMP}/jars" \ - --cuda-version "${{ matrix.cuda }}" - - name: Upload per-entry JAR artifact - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 - with: - name: cudf_java_cuda${{ matrix.cuda_major }}_${{ matrix.arch }} - # Ship only the JARs and POMs. Exclude the per-classifier Maven build scratch dir. - path: | - ${{ runner.temp }}/jars - !${{ runner.temp }}/jars/.mvn-temp-target - if-no-files-found: error + uses: rapidsai/shared-workflows/.github/workflows/compute-matrix.yaml@main + with: + build_type: ${{ inputs.build_type || 'branch' }} + matrix_name: conda-cpp-build + java-build: + needs: [build-details, telemetry-setup, java-build-matrix] + permissions: + actions: read + contents: read + id-token: write + packages: read + pull-requests: read + secrets: inherit # zizmor: ignore[secrets-inherit] + uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@main + strategy: + fail-fast: false + matrix: ${{ fromJSON(needs.java-build-matrix.outputs.matrix) }} + with: + build_type: ${{ inputs.build_type || 'branch' }} + branch: ${{ inputs.branch }} + date: ${{ inputs.date }} + sha: ${{ inputs.sha }} + build-datetime: ${{ needs.build-details.outputs.build-datetime }} + arch: ${{ matrix.ARCH }} + node_type: cpu16 + container_image: "rapidsai/ci-wheel:26.10-cuda${{ matrix.CUDA_VER }}-${{ matrix.LINUX_VER }}-py${{ matrix.PY_VER }}" + script: "RAPIDS_CUDA_VERSION=${{ matrix.CUDA_VER }} ci/build_java.sh" + file_to_upload: output_jars + artifact-name: java_pkg_${{ matrix.CUDA_VER }}_${{ matrix.ARCH }} # Assemble the per-classifier JARs into one Maven-repository layout. java-gather: needs: [java-build] @@ -560,7 +553,7 @@ jobs: - name: Download per-entry JAR artifacts uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0 with: - pattern: cudf_java_cuda* + pattern: java_pkg_* path: ${{ runner.temp }}/jars merge-multiple: true - name: Assemble Maven repository layout diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index 500f5a71636b..9bd7f621affc 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -25,6 +25,8 @@ jobs: - conda-python-cudf-tests - conda-python-other-tests - conda-java-tests + - java-build + - java-tests - conda-notebook-tests - docs-build - publish-api-docs @@ -526,6 +528,75 @@ jobs: arch: "amd64" container_image: "rapidsai/ci-conda:26.10-latest" script: "ci/test_java.sh" + # Build and test the cuDF Java JAR for every Maven classifier. + java-build-matrix: + needs: [checks] + permissions: + contents: read + uses: rapidsai/shared-workflows/.github/workflows/compute-matrix.yaml@main + with: + build_type: pull-request + matrix_name: conda-cpp-build + java-build: + needs: [build-details, java-build-matrix, changed-files] + permissions: + actions: read + contents: read + id-token: write + packages: read + pull-requests: read + secrets: inherit # zizmor: ignore[secrets-inherit] + uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@main + if: fromJSON(needs.changed-files.outputs.changed_file_groups).test_java + strategy: + fail-fast: false + matrix: ${{ fromJSON(needs.java-build-matrix.outputs.matrix) }} + with: + build_type: pull-request + build-datetime: ${{ needs.build-details.outputs.build-datetime }} + arch: ${{ matrix.ARCH }} + node_type: cpu16 + container_image: "rapidsai/ci-wheel:26.10-cuda${{ matrix.CUDA_VER }}-${{ matrix.LINUX_VER }}-py${{ matrix.PY_VER }}" + script: "RAPIDS_CUDA_VERSION=${{ matrix.CUDA_VER }} ci/build_java.sh" + file_to_upload: output_jars + artifact-name: java_pkg_${{ matrix.CUDA_VER }}_${{ matrix.ARCH }} + java-tests: + needs: [java-build, java-build-matrix] + if: ${{ !cancelled() && needs.java-build.result == 'success' }} + permissions: + actions: read + contents: read + packages: read + strategy: + fail-fast: false + matrix: ${{ fromJSON(needs.java-build-matrix.outputs.matrix) }} + runs-on: linux-${{ matrix.ARCH }}-gpu-l4-latest-1 + container: + image: rapidsai/ci-wheel:26.10-cuda${{ matrix.CUDA_VER }}-${{ matrix.LINUX_VER }}-py${{ matrix.PY_VER }} # zizmor: ignore[unpinned-images] + env: + NVIDIA_VISIBLE_DEVICES: ${{ env.NVIDIA_VISIBLE_DEVICES }} + defaults: + run: + shell: bash + steps: + - name: Checkout + uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1 + with: + persist-credentials: false + - name: Download java-build artifact + uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0 + with: + name: java_pkg_${{ matrix.CUDA_VER }}_${{ matrix.ARCH }} + path: java_pkg + - name: Run Java tests + env: + RAPIDS_CUDA_VERSION: ${{ matrix.CUDA_VER }} + LIBCUDF_LARGE_STRINGS_ENABLED: "0" + run: | + . java/ci/java_classifier.sh + JAVA_JAR="$(cudf_java_resolve_artifact_jar java_pkg)" + export JAVA_JAR + ci/test_packaged_java.sh conda-notebook-tests: needs: [conda-python-build, conda-python-build-noarch, changed-files] permissions: diff --git a/ci/build_java.sh b/ci/build_java.sh new file mode 100755 index 000000000000..2cfaabdd4365 --- /dev/null +++ b/ci/build_java.sh @@ -0,0 +1,60 @@ +#!/bin/bash +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +# CI entrypoint: static libcudf + classifier JAR for one matrix cell. +# +# Builds the static libcudf install tree and packages the matching classifier +# JAR, writing ./output_jars// for the custom-job upload step. +# +# Inputs (environment variables): +# RAPIDS_CUDA_VERSION CUDA version, e.g. 12.9.2 (required). +# PARALLEL_LEVEL Build parallelism (default: nproc). +# CMAKE_CUDA_ARCHITECTURES Optional override for -DCMAKE_CUDA_ARCHITECTURES. +# JAVA_WORK_DIR Optional scratch dir (default: /.java-work). + +set -euo pipefail + +REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +cd "${REPO_ROOT}" + +# shellcheck disable=SC1091 +. "${REPO_ROOT}/java/ci/ci_wheel_image.sh" +# shellcheck disable=SC1091 +. "${REPO_ROOT}/java/ci/java_classifier.sh" + +if [[ -z ${RAPIDS_CUDA_VERSION:-} ]]; then + echo "Error: RAPIDS_CUDA_VERSION must be set" >&2 + exit 1 +fi + +RAPIDS_CUDA_VERSION="$(cudf_java_normalize_cuda_version "${RAPIDS_CUDA_VERSION}")" +export RAPIDS_CUDA_VERSION +CLASSIFIER="$(cudf_java_maven_classifier "${RAPIDS_CUDA_VERSION}")" + +WORK_DIR="${JAVA_WORK_DIR:-${REPO_ROOT}/.java-work}" +LIBCUDF_DIR="${WORK_DIR}/libcudf" +CLASSIFIER_OUT="${REPO_ROOT}/output_jars/${CLASSIFIER}" + +cleanup_scratch() { + rm -rf "${WORK_DIR}" +} +trap cleanup_scratch EXIT + +rm -rf "${LIBCUDF_DIR}" "${CLASSIFIER_OUT}" +mkdir -p "${LIBCUDF_DIR}" "${CLASSIFIER_OUT}" + +export REPO_ROOT +export INSTALL_PREFIX="${LIBCUDF_DIR}" +export BUILD_DIR="${WORK_DIR}/libcudf-build" +export CUDF_INSTALL_DIR="${LIBCUDF_DIR}" +export OUTPUT_DIR="${CLASSIFIER_OUT}" + +rapids-logger "Building static libcudf (${CLASSIFIER})" +bash "${REPO_ROOT}/java/ci/build_static_libcudf_in_container.sh" + +rapids-logger "Packaging cuDF Java JAR (${CLASSIFIER})" +bash "${REPO_ROOT}/java/ci/build_cudf_java_jar_in_container.sh" + +cudf_java_assert_classifier_artifacts "${CLASSIFIER_OUT}" "${CLASSIFIER}" +ls -la "${CLASSIFIER_OUT}" diff --git a/ci/test_packaged_java.sh b/ci/test_packaged_java.sh new file mode 100755 index 000000000000..6ed0c94e5f41 --- /dev/null +++ b/ci/test_packaged_java.sh @@ -0,0 +1,47 @@ +#!/bin/bash +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +# Run the Java tests against an already-packaged classifier JAR. +# +# Activates -Ppackaged-jar-tests so the surefire classpath uses the JAR at +# JAVA_JAR instead of a locally compiled target/classes tree. Caller places +# the JAR (e.g. via actions/download-artifact) before invoking this script. +# +# Inputs (environment variables): +# JAVA_JAR Absolute path to the classifier JAR (required). +# LIBCUDF_LARGE_STRINGS_ENABLED Optional; defaults to 0 (same as ci/test_java.sh). + +set -euo pipefail + +REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +cd "${REPO_ROOT}" + +if [[ -z ${JAVA_JAR:-} || ! -f ${JAVA_JAR} ]]; then + echo "Error: JAVA_JAR must point to an existing classifier JAR" >&2 + exit 1 +fi + +if ! command -v mvn >/dev/null 2>&1 || ! command -v java >/dev/null 2>&1; then + # shellcheck disable=SC1091 + . "${REPO_ROOT}/java/ci/setup_java_env.sh" +fi + +# Match the existing conda Java test entrypoint in ci/test_java.sh. +export LIBCUDF_LARGE_STRINGS_ENABLED="${LIBCUDF_LARGE_STRINGS_ENABLED:-0}" + +PRODUCT_JAR="$(cd "$(dirname "${JAVA_JAR}")" && pwd)/$(basename "${JAVA_JAR}")" +rapids-logger "Product JAR: ${PRODUCT_JAR}" + +rm -rf "${REPO_ROOT}/java/target" + +pushd "${REPO_ROOT}/java" >/dev/null +set +e +timeout 30m mvn -B test -Ppackaged-jar-tests \ + "-Dcudf.jar.path=${PRODUCT_JAR}" \ + "-DCUDF_JNI_ENABLE_PROFILING=OFF" +EXITCODE=$? +set -e +popd >/dev/null +rapids-logger "Java tests exit=${EXITCODE}" +exit "${EXITCODE}" diff --git a/java/ci/README.md b/java/ci/README.md index 49ab3b6074ec..7a668695f6e3 100644 --- a/java/ci/README.md +++ b/java/ci/README.md @@ -4,14 +4,14 @@ The scripts under `java/ci/` build the cuDF Java JAR for every Maven classifier the same way locally and in CI (GitHub Actions is only a thin wrapper that adds -artifact upload/download). Each script pulls the RAPIDS `ci-conda` build image, +artifact upload/download). Each script pulls the RAPIDS `ci-wheel` build image, runs the build in a throwaway container, and writes its output to a host directory. No local `docker build` is required, and no GPU is required to build. ### Prerequisites 1. Docker is installed and the current user can run `docker`. -2. Network access to pull `rapidsai/ci-conda:-latest`. +2. Network access to pull `rapidsai/ci-wheel:-cuda-rockylinux8-py3.11`. ### Local one-command shortcut @@ -90,6 +90,17 @@ per-entry artifact. The separate `java-gather` job downloads them (with `merge-multiple: true`, so all subdirs land in a single parent dir), runs Step 3, and uploads the combined `cudf_java_maven_repo` artifact. +### Packaging-aware tests (local) + +Plain `cd java && mvn test` does not exercise the classifier JAR. Use +`java/ci/test_packaged_java_local.sh` (or CI entrypoint +`ci/test_packaged_java.sh`) to run the existing Java tests against a packaged +JAR. Needs Docker + GPU. + +```bash +./java/ci/test_packaged_java_local.sh --work-dir /tmp/java-build-test +``` + ## Legacy: manual Dockerfile.rocky build (obsolete) > The `java/ci/Dockerfile.rocky` + `java/ci/build-in-docker.sh` flow below is the diff --git a/java/ci/build-in-docker.sh b/java/ci/build-in-docker.sh index b212d4bec5ad..fedac011e838 100755 --- a/java/ci/build-in-docker.sh +++ b/java/ci/build-in-docker.sh @@ -69,7 +69,7 @@ BUILD_ARG=( "-DCUDF_USE_PER_THREAD_DEFAULT_STREAM=$ENABLE_PTDS" "-DCUDF_JNI_LIBCUDF_STATIC=ON" "-DUSE_GDS=$ENABLE_GDS" - "-Dtest=*,!CuFileTest,!CudaFatalTest,!ColumnViewNonEmptyNullsTest,!NativeDepsLoaderTest" + "-Dtest=*,!CuFileTest,!CudaFatalTest,!ColumnViewNonEmptyNullsTest,!NativeDepsLoaderTest,!PackagedJarOriginCheck" ) if [ "$SIGN_FILE" == true ]; then diff --git a/java/ci/build_cudf_java_jar.sh b/java/ci/build_cudf_java_jar.sh index 084b00c68478..f9cf9c98dd2f 100755 --- a/java/ci/build_cudf_java_jar.sh +++ b/java/ci/build_cudf_java_jar.sh @@ -5,7 +5,7 @@ # Self-contained packaging of the cuDF Java JAR for a single classifier. # # Consumes a prebuilt static libcudf install tree (from build_static_libcudf.sh), -# compiles the JNI layer against it inside a throwaway RAPIDS ci-conda container, +# compiles the JNI layer against it inside a throwaway RAPIDS ci-wheel container, # and emits the single classifier JAR (plus its POM) to a per-classifier # subdirectory under --output-dir. This script is layout-agnostic: it produces # one classifier's artifacts and knows nothing about the combined @@ -18,6 +18,10 @@ REPO_ROOT="$(git -C "${SCRIPT_DIR}" rev-parse --show-toplevel)" # shellcheck disable=SC1091 . "${SCRIPT_DIR}/argparse.sh" +# shellcheck disable=SC1091 +. "${SCRIPT_DIR}/ci_wheel_image.sh" +# shellcheck disable=SC1091 +. "${SCRIPT_DIR}/java_classifier.sh" LIBCUDF_DIR="" OUTPUT_DIR="" @@ -31,11 +35,10 @@ print_help() { Usage: build_cudf_java_jar.sh --libcudf-dir --output-dir \\ --cuda-version [OPTIONS] -Packages the cuDF Java JAR for a single classifier inside a RAPIDS ci-conda +Packages the cuDF Java JAR for a single classifier inside a RAPIDS ci-wheel container, linking against a prebuilt static libcudf. Always builds for the -host architecture (uname -m). The build image is fixed to -rapidsai/ci-conda:-latest (version derived from the VERSION -file). +host architecture (uname -m). The build image is derived from --cuda-version +and the VERSION file (see java/ci/ci_wheel_image.sh). The classifier is derived from --cuda-version (major) + host arch (uname -m), mirroring the pom.xml Groovy logic: "cuda" for x86_64, @@ -49,7 +52,7 @@ REQUIRED: -o, --output-dir Host parent directory. The script creates and writes to //, which must not already exist. - -c, --cuda-version CUDA version to build for (e.g. "12.9" or "12.9.1"). + -c, --cuda-version CUDA version to build for (e.g. "12.9" or "12.9.2"). Must match --cuda-version of the static libcudf tree; determines the cuda12/cuda13 classifier. @@ -126,26 +129,9 @@ if [[ ! -d ${LIBCUDF_DIR} ]]; then exit 1 fi -# Derive the Maven classifier from --cuda-version major + host arch, mirroring -# the pom.xml Groovy logic: "cuda" for x86_64, "cuda-arm64" for -# aarch64. -CUDA_MAJOR="$(echo "${CUDA_VERSION}" | cut -d. -f1)" -HOST_ARCH="$(uname -m)" -case "${HOST_ARCH}" in - x86_64) - CLASSIFIER="cuda${CUDA_MAJOR}" - ;; - aarch64|arm64) - CLASSIFIER="cuda${CUDA_MAJOR}-arm64" - ;; - *) - echo "Error: Unsupported host arch '${HOST_ARCH}' (expected x86_64 or aarch64)" >&2 - exit 1 - ;; -esac - -RAPIDS_VERSION="$(head -1 "${REPO_ROOT}/VERSION" | cut -d. -f1,2)" -IMAGE="rapidsai/ci-conda:${RAPIDS_VERSION}-latest" +CLASSIFIER="$(cudf_java_maven_classifier "${CUDA_VERSION}")" +IMAGE="$(cudf_java_ci_wheel_image "${CUDA_VERSION}")" +CUDA_VERSION_FULL="$(cudf_java_normalize_cuda_version "${CUDA_VERSION}")" mkdir -p "${OUTPUT_DIR}" OUTPUT_DIR="$(cd "${OUTPUT_DIR}" && pwd)" @@ -173,7 +159,7 @@ mkdir -p "${TARGET_SCRATCH}" echo "Packaging cuDF Java JAR" echo " image: ${IMAGE}" -echo " cuda version: ${CUDA_VERSION}" +echo " cuda version: ${CUDA_VERSION_FULL}" echo " classifier: ${CLASSIFIER}" echo " parallel: ${PARALLEL_LEVEL}" echo " libcudf dir: ${LIBCUDF_DIR}" @@ -190,10 +176,13 @@ DOCKER_ARGS=( --volume "${CLASSIFIER_OUT}:/output" --volume "${TARGET_SCRATCH}:/repo/java/target" --workdir /repo - --env RAPIDS_CUDA_VERSION="${CUDA_VERSION}" + --env RAPIDS_CUDA_VERSION="${CUDA_VERSION_FULL}" --env PARALLEL_LEVEL="${PARALLEL_LEVEL}" --env HOST_UID="$(id -u)" --env HOST_GID="$(id -g)" + --env CUDF_INSTALL_DIR=/libcudf + --env OUTPUT_DIR=/output + --env REPO_ROOT=/repo ) if [[ -n ${CMAKE_CUDA_ARCHITECTURES} ]]; then @@ -205,42 +194,4 @@ docker run "${DOCKER_ARGS[@]}" "${IMAGE}" \ # Post-run: assert exactly one main classifier JAR + one POM, and that the # JAR's classifier suffix matches the subdir name we chose (catches pom drift). -PRODUCED_JAR="" -for candidate in "${CLASSIFIER_OUT}"/cudf-*-"${CLASSIFIER}".jar; do - if [[ -f "${candidate}" ]]; then - if [[ -n "${PRODUCED_JAR}" ]]; then - echo "Error: multiple JARs matching cudf-*-${CLASSIFIER}.jar found in ${CLASSIFIER_OUT}" - ls -1 "${CLASSIFIER_OUT}" - exit 1 - fi - PRODUCED_JAR=${candidate} - fi -done - -if [[ -z "${PRODUCED_JAR}" ]]; then - echo "Error: no cudf-*-${CLASSIFIER}.jar found in ${CLASSIFIER_OUT}" - ls -1 "${CLASSIFIER_OUT}" || true - exit 1 -fi - -PRODUCED_POM="" -for candidate in "${CLASSIFIER_OUT}"/cudf-*.pom; do - if [[ -f "${candidate}" ]]; then - if [[ -n "${PRODUCED_POM}" ]]; then - echo "Error: multiple POMs found in ${CLASSIFIER_OUT}" - ls -1 "${CLASSIFIER_OUT}" - exit 1 - fi - PRODUCED_POM=${candidate} - fi -done - -if [[ -z "${PRODUCED_POM}" ]]; then - echo "Error: no cudf-*.pom found in ${CLASSIFIER_OUT}" - ls -1 "${CLASSIFIER_OUT}" || true - exit 1 -fi - -echo "cuDF Java JAR build succeeded:" -echo " $(basename "${PRODUCED_JAR}")" -echo " $(basename "${PRODUCED_POM}")" +cudf_java_assert_classifier_artifacts "${CLASSIFIER_OUT}" "${CLASSIFIER}" diff --git a/java/ci/build_cudf_java_jar_in_container.sh b/java/ci/build_cudf_java_jar_in_container.sh index e02ea3171508..53fc028ce875 100755 --- a/java/ci/build_cudf_java_jar_in_container.sh +++ b/java/ci/build_cudf_java_jar_in_container.sh @@ -4,116 +4,92 @@ # In-container packaging of the cuDF Java JAR for a single classifier. # -# This script runs inside the rapidsai/ci-conda container launched by -# java/ci/build_cudf_java_jar.sh. It generates the build_java conda toolchain -# environment, installs a JDK 17 side-prefix for the javadoc-jdk17 profile, -# compiles the JNI layer against a prebuilt static libcudf (mounted at -# /libcudf), and packages the cuDF Java JAR. The resulting classifier JAR, -# sources JAR, javadoc JAR, and POM are copied to /output. /output and -# /repo/java/target are chowned to HOST_UID:HOST_GID on exit so the host user -# owns the outputs. +# Expects a RAPIDS ci-wheel environment. Sources the toolchain from +# setup_java_env.sh, compiles the JNI layer against the prebuilt static +# libcudf at CUDF_INSTALL_DIR, and packages the cuDF Java JAR. The classifier +# JAR, sources JAR, javadoc JAR, and POM are copied to OUTPUT_DIR. When +# HOST_UID / HOST_GID are set, OUTPUT_DIR and java/target are chowned on exit +# so the host user owns the outputs. # # Inputs (environment variables): -# RAPIDS_CUDA_VERSION CUDA version, e.g. 12.9 or 12.9.1 (required). +# RAPIDS_CUDA_VERSION CUDA version, e.g. 12.9.2 (required). # PARALLEL_LEVEL Build parallelism (default: nproc). # CMAKE_CUDA_ARCHITECTURES Optional override for -DCMAKE_CUDA_ARCHITECTURES. -# HOST_UID / HOST_GID Chown target for /output and /repo/java/target -# (both required). +# CUDF_INSTALL_DIR Static libcudf install tree (default: /libcudf). +# OUTPUT_DIR Artifact output dir (default: /output). +# REPO_ROOT cuDF checkout (default: /repo). +# HOST_UID / HOST_GID Optional chown target for OUTPUT_DIR + java/target. set -e -OUTPUT_DIR=/output -REPO_ROOT=/repo -CUDF_INSTALL_DIR=/libcudf +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +OUTPUT_DIR="${OUTPUT_DIR:-/output}" +REPO_ROOT="${REPO_ROOT:-/repo}" +CUDF_INSTALL_DIR="${CUDF_INSTALL_DIR:-/libcudf}" -. /opt/conda/etc/profile.d/conda.sh +# shellcheck disable=SC1091 +. "${SCRIPT_DIR}/setup_java_env.sh" -if [[ -z ${RAPIDS_CUDA_VERSION} ]]; then +if [[ -z ${RAPIDS_CUDA_VERSION:-} ]]; then echo "Error: RAPIDS_CUDA_VERSION must be set" >&2 exit 1 fi -if [[ -z ${HOST_UID} || -z ${HOST_GID} ]]; then - echo "Error: HOST_UID and HOST_GID must both be set" >&2 - exit 1 -fi - _chown_outputs_on_exit() { - chown -R "${HOST_UID}:${HOST_GID}" "${OUTPUT_DIR}" "${REPO_ROOT}/java/target" 2>/dev/null || true + if [[ -n ${HOST_UID:-} && -n ${HOST_GID:-} ]]; then + chown -R "${HOST_UID}:${HOST_GID}" "${OUTPUT_DIR}" "${REPO_ROOT}/java/target" 2>/dev/null || true + fi } trap _chown_outputs_on_exit EXIT -if [[ -z ${PARALLEL_LEVEL} ]]; then - PARALLEL_LEVEL=$(nproc) -fi - -CUDA_MAJOR_MINOR=$(echo "${RAPIDS_CUDA_VERSION}" | cut -d. -f1,2) - -rapids-logger "Configuring conda strict channel priority" -conda config --set channel_priority strict - -rapids-logger "Generating build_java conda environment (cuda=${CUDA_MAJOR_MINOR}, arch=$(arch))" -ENV_YAML_DIR="$(mktemp -d)" -rapids-dependency-file-generator \ - --output conda \ - --file-key build_java \ - --matrix "cuda=${CUDA_MAJOR_MINOR};arch=$(arch)" | tee "${ENV_YAML_DIR}/env.yaml" - -rapids-mamba-retry env create --yes -f "${ENV_YAML_DIR}/env.yaml" -n build_java -conda activate build_java - -rapids-print-env - -# The `javadoc-jdk17` profile in java/pom.xml points -# at ${env.JDK17_HOME}/bin/javadoc. The build_java env -# above provides only JDK 8 (mvn's own JVM), so install JDK 17 into a -# dedicated prefix that JDK17_HOME can point to. The primary mvn JVM stays -# on JDK 8; only the javadoc binary is invoked from this prefix. -rapids-logger "Installing JDK 17 into /opt/jdk17 for javadoc-jdk17 profile" -rapids-mamba-retry create --yes --prefix /opt/jdk17 openjdk=17.* -export JDK17_HOME=/opt/jdk17 - -if [[ -z ${CUDACXX} ]]; then - export CUDACXX="${CONDA_PREFIX}/bin/nvcc" -fi -if [[ -z ${LIBCUDF_KERNEL_CACHE_PATH} ]]; then - export LIBCUDF_KERNEL_CACHE_PATH=/tmp/rapids-kernel-cache -fi - BUILD_ARG=( -B - # Prefix every log line with HH:mm:ss.SSS so the elapsed time of individual - # plugin executions is recorded. + # Prefix every log line with HH:mm:ss.SSS to record per-plugin elapsed time. "-Dorg.slf4j.simpleLogger.showDateTime=true" "-Dorg.slf4j.simpleLogger.dateTimeFormat=HH:mm:ss.SSS" - "-Dmaven.repo.local=/tmp/.m2" + "-Dmaven.repo.local=${MAVEN_REPO_LOCAL:-/tmp/.m2}" "-Dparallel.level=${PARALLEL_LEVEL}" "-DskipTests=true" "-DCUDF_USE_PER_THREAD_DEFAULT_STREAM=ON" "-DCUDF_JNI_LIBCUDF_STATIC=ON" "-DUSE_GDS=OFF" - # -Prelease produces the sources.jar file via maven-source-plugin; - # -Pjavadoc-jdk17 produces the javadoc.jar file via maven-javadoc-plugin - # running against ${env.JDK17_HOME}/bin/javadoc. Both are required by - # Maven Central for every published release. + # -Prelease adds the sources.jar; -Pjavadoc-jdk17 adds the javadoc.jar (built + # against ${JDK17_HOME}/bin/javadoc). Both are required by Maven Central. "-Prelease" "-Pjavadoc-jdk17" ) - -if [[ -n ${CMAKE_CUDA_ARCHITECTURES} ]]; then +if [[ -n ${CMAKE_CUDA_ARCHITECTURES:-} ]]; then BUILD_ARG+=("-DCMAKE_CUDA_ARCHITECTURES=${CMAKE_CUDA_ARCHITECTURES}") fi +# Pass the toolchain + sccache launchers from setup_java_env.sh to the JNI cmake +# invocation via the pom's cmake.ccache.opts property. +CMAKE_CCACHE_OPTS=() +if [[ -n ${CMAKE_C_COMPILER_LAUNCHER:-} ]]; then + CMAKE_CCACHE_OPTS+=("-DCMAKE_C_COMPILER_LAUNCHER=${CMAKE_C_COMPILER_LAUNCHER}") +fi +if [[ -n ${CMAKE_CXX_COMPILER_LAUNCHER:-} ]]; then + CMAKE_CCACHE_OPTS+=("-DCMAKE_CXX_COMPILER_LAUNCHER=${CMAKE_CXX_COMPILER_LAUNCHER}") +fi +if [[ -n ${CMAKE_CUDA_COMPILER_LAUNCHER:-} ]]; then + CMAKE_CCACHE_OPTS+=("-DCMAKE_CUDA_COMPILER_LAUNCHER=${CMAKE_CUDA_COMPILER_LAUNCHER}") +fi +CMAKE_CCACHE_OPTS+=("-DCMAKE_C_COMPILER=${CC}" "-DCMAKE_CXX_COMPILER=${CXX}") +CMAKE_CCACHE_OPTS+=("-DCMAKE_CUDA_HOST_COMPILER=${CMAKE_CUDA_HOST_COMPILER}") +if [[ -d ${BOOST_PREFIX} ]]; then + CMAKE_CCACHE_OPTS+=("-DCMAKE_PREFIX_PATH=${BOOST_PREFIX}") +fi +BUILD_ARG+=("-Dcmake.ccache.opts=${CMAKE_CCACHE_OPTS[*]}") + cd "${REPO_ROOT}/java" -CUDF_VERSION="$(mvn help:evaluate -Dexpression=project.version -q -DforceStdout "${BUILD_ARG[@]}")" -rapids-logger "Packaging cuDF Java JAR version ${CUDF_VERSION} (libcudf: ${CUDF_INSTALL_DIR})" +CUDF_VERSION="$(cudf_java_scl mvn help:evaluate -Dexpression=project.version -q -DforceStdout "${BUILD_ARG[@]}")" +rapids-logger "Packaging cuDF Java JAR ${CUDF_VERSION}" -# The `clean` goal is intentionally omitted: /repo/java/target is a -# bind-mount point, so when `mvn clean` attempts to remove the directory, -# it fails with EBUSY. The host wrapper (build_cudf_java_jar.sh) recreates -# the scratch dir before each container launch to guarantee target/ starts empty. -CUDF_INSTALL_DIR="${CUDF_INSTALL_DIR}" mvn package "${BUILD_ARG[@]}" +# Omit the `clean` goal: java/target may be a bind-mount point, so `mvn clean` +# fails with EBUSY. The host wrapper recreates the scratch dir before each +# launch to guarantee target/ starts empty. +CUDF_INSTALL_DIR="${CUDF_INSTALL_DIR}" cudf_java_scl mvn package "${BUILD_ARG[@]}" mkdir -p "${OUTPUT_DIR}" @@ -132,8 +108,7 @@ for candidate in target/cudf-"${CUDF_VERSION}"-*.jar; do *) if [[ -f ${candidate} ]]; then if [[ -n ${MAIN_JAR} ]]; then - echo "Error: multiple main classifier JARs matched under target/" >&2 - ls -l target/ >&2 || true + echo "Error: multiple main classifier JARs under target/" >&2 exit 1 fi MAIN_JAR=${candidate} @@ -143,23 +118,25 @@ for candidate in target/cudf-"${CUDF_VERSION}"-*.jar; do done if [[ -z ${MAIN_JAR} ]]; then - echo "Error: no cuDF classifier JAR produced under target/" >&2 - ls -l target/ >&2 || true + echo "Error: no classifier JAR under target/" >&2 + ls -l target/ >&2 exit 1 fi # Assert the release-profile artifacts landed. A missing file here means -# -Prelease or -Pjavadoc-jdk17 did not activate, or JDK17_HOME did not -# resolve to a usable javadoc binary. +# -Prelease or -Pjavadoc-jdk17 did not activate, or JDK17_HOME did not resolve +# to a usable javadoc binary. for required in "${OUTPUT_DIR}/cudf-${CUDF_VERSION}-sources.jar" \ "${OUTPUT_DIR}/cudf-${CUDF_VERSION}-javadoc.jar"; do if [[ ! -f ${required} ]]; then - echo "Error: expected ${required} not found (mvn -Prelease -Pjavadoc-jdk17 did not produce it)" >&2 + echo "Error: missing ${required}" >&2 exit 1 fi done cp -f "${MAIN_JAR}" "${OUTPUT_DIR}/" cp -f pom.xml "${OUTPUT_DIR}/cudf-${CUDF_VERSION}.pom" - -rapids-logger "Emitted $(basename "${MAIN_JAR}"), cudf-${CUDF_VERSION}-sources.jar, cudf-${CUDF_VERSION}-javadoc.jar, and cudf-${CUDF_VERSION}.pom to ${OUTPUT_DIR}" +rapids-logger "Emitted artifacts to ${OUTPUT_DIR}" +if command -v sccache >/dev/null 2>&1; then + sccache --show-adv-stats || true +fi diff --git a/java/ci/build_static_libcudf.sh b/java/ci/build_static_libcudf.sh index 2fa2e435471c..2ffac68b3c8d 100755 --- a/java/ci/build_static_libcudf.sh +++ b/java/ci/build_static_libcudf.sh @@ -4,7 +4,7 @@ # Self-contained build of a static libcudf install tree. # -# Pulls the RAPIDS ci-conda image, builds libcudf with BUILD_SHARED_LIBS=OFF +# Pulls the RAPIDS ci-wheel image, builds libcudf with BUILD_SHARED_LIBS=OFF # inside a throwaway container, and installs the static libcudf tree (libcudf.a # plus its static dependencies) into a directory on the host. No GPU is required # to build. @@ -16,6 +16,8 @@ REPO_ROOT="$(git -C "${SCRIPT_DIR}" rev-parse --show-toplevel)" # shellcheck disable=SC1091 . "${SCRIPT_DIR}/argparse.sh" +# shellcheck disable=SC1091 +. "${SCRIPT_DIR}/ci_wheel_image.sh" OUTPUT_DIR="" CUDA_VERSION="" @@ -27,15 +29,15 @@ print_help() { Usage: build_static_libcudf.sh --output-dir --cuda-version [OPTIONS] -Builds a static libcudf install tree inside a RAPIDS ci-conda container and +Builds a static libcudf install tree inside a RAPIDS ci-wheel container and writes it to a directory on the host. Always builds for the host architecture -(uname -m). The build image is fixed to rapidsai/ci-conda:-latest -(version derived from the VERSION file). +(uname -m). The build image is derived from --cuda-version and the VERSION file +(see java/ci/ci_wheel_image.sh). REQUIRED: -o, --output-dir Host directory to receive the static install tree (libcudf.a and its static dependencies). - -c, --cuda-version CUDA version to build for (e.g. "12.9" or "12.9.1"). + -c, --cuda-version CUDA version to build for (e.g. "12.9" or "12.9.2"). OPTIONS: -A, --cmake-cuda-architectures @@ -97,15 +99,15 @@ parse_args "$@" require_arg --output-dir "${OUTPUT_DIR}" require_arg --cuda-version "${CUDA_VERSION}" -RAPIDS_VERSION="$(head -1 "${REPO_ROOT}/VERSION" | cut -d. -f1,2)" -IMAGE="rapidsai/ci-conda:${RAPIDS_VERSION}-latest" +IMAGE="$(cudf_java_ci_wheel_image "${CUDA_VERSION}")" +CUDA_VERSION_FULL="$(cudf_java_normalize_cuda_version "${CUDA_VERSION}")" mkdir -p "${OUTPUT_DIR}" OUTPUT_DIR="$(cd "${OUTPUT_DIR}" && pwd)" echo "Building static libcudf" echo " image: ${IMAGE}" -echo " cuda version: ${CUDA_VERSION}" +echo " cuda version: ${CUDA_VERSION_FULL}" echo " parallel: ${PARALLEL_LEVEL}" echo " output dir: ${OUTPUT_DIR}" if [[ -n ${CMAKE_CUDA_ARCHITECTURES} ]]; then @@ -117,10 +119,12 @@ DOCKER_ARGS=( --volume "${REPO_ROOT}:/repo" --volume "${OUTPUT_DIR}:/output" --workdir /repo - --env RAPIDS_CUDA_VERSION="${CUDA_VERSION}" + --env RAPIDS_CUDA_VERSION="${CUDA_VERSION_FULL}" --env PARALLEL_LEVEL="${PARALLEL_LEVEL}" --env HOST_UID="$(id -u)" --env HOST_GID="$(id -g)" + --env INSTALL_PREFIX=/output + --env REPO_ROOT=/repo ) if [[ -n ${CMAKE_CUDA_ARCHITECTURES} ]]; then diff --git a/java/ci/build_static_libcudf_in_container.sh b/java/ci/build_static_libcudf_in_container.sh index e77637e747b2..b911b131aad3 100755 --- a/java/ci/build_static_libcudf_in_container.sh +++ b/java/ci/build_static_libcudf_in_container.sh @@ -4,69 +4,44 @@ # In-container build of a static libcudf install tree. # -# This script runs inside the rapidsai/ci-conda container launched by -# java/ci/build_static_libcudf.sh. It generates the build_java conda toolchain -# environment, builds libcudf with BUILD_SHARED_LIBS=OFF, and installs the -# resulting static libcudf (plus its static dependencies) into /output. Then -# chowns /output to HOST_UID:HOST_GID so the host user owns the outputs. +# Expects a RAPIDS ci-wheel environment. Sources the toolchain from +# setup_java_env.sh, builds libcudf with BUILD_SHARED_LIBS=OFF, and installs +# the resulting static libcudf (plus its static dependencies) into +# INSTALL_PREFIX. When HOST_UID / HOST_GID are set, chowns the install tree so +# the host user owns the outputs. # # Inputs (environment variables): -# RAPIDS_CUDA_VERSION CUDA version, e.g. 12.9 or 12.9.1 (required). +# RAPIDS_CUDA_VERSION CUDA version, e.g. 12.9.2 (required). # PARALLEL_LEVEL Build parallelism (default: nproc). # CMAKE_CUDA_ARCHITECTURES Optional override for -DCMAKE_CUDA_ARCHITECTURES. -# HOST_UID / HOST_GID Chown target for /output (both required). +# INSTALL_PREFIX Static libcudf install dir (default: /output). +# REPO_ROOT cuDF checkout (default: /repo). +# BUILD_DIR CMake build dir (default: /tmp/libcudf-build). +# HOST_UID / HOST_GID Optional chown target for INSTALL_PREFIX. set -e -INSTALL_PREFIX=/output -REPO_ROOT=/repo -BUILD_DIR=/tmp/libcudf-build +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +INSTALL_PREFIX="${INSTALL_PREFIX:-/output}" +REPO_ROOT="${REPO_ROOT:-/repo}" +BUILD_DIR="${BUILD_DIR:-/tmp/libcudf-build}" -. /opt/conda/etc/profile.d/conda.sh +# shellcheck disable=SC1091 +. "${SCRIPT_DIR}/setup_java_env.sh" -if [[ -z ${RAPIDS_CUDA_VERSION} ]]; then +if [[ -z ${RAPIDS_CUDA_VERSION:-} ]]; then echo "Error: RAPIDS_CUDA_VERSION must be set" >&2 exit 1 fi -if [[ -z ${HOST_UID} || -z ${HOST_GID} ]]; then - echo "Error: HOST_UID and HOST_GID must both be set" >&2 - exit 1 -fi - -if [[ -z ${PARALLEL_LEVEL} ]]; then - PARALLEL_LEVEL=$(nproc) -fi - -CUDA_MAJOR_MINOR=$(echo "${RAPIDS_CUDA_VERSION}" | cut -d. -f1,2) - -rapids-logger "Configuring conda strict channel priority" -conda config --set channel_priority strict - -rapids-logger "Generating build_java conda environment (cuda=${CUDA_MAJOR_MINOR}, arch=$(arch))" -ENV_YAML_DIR="$(mktemp -d)" -rapids-dependency-file-generator \ - --output conda \ - --file-key build_java \ - --matrix "cuda=${CUDA_MAJOR_MINOR};arch=$(arch)" | tee "${ENV_YAML_DIR}/env.yaml" - -rapids-mamba-retry env create --yes -f "${ENV_YAML_DIR}/env.yaml" -n build_java -conda activate build_java - -rapids-print-env - -if [[ -z ${CUDACXX} ]]; then - export CUDACXX="${CONDA_PREFIX}/bin/nvcc" -fi -if [[ -z ${LIBCUDF_KERNEL_CACHE_PATH} ]]; then - export LIBCUDF_KERNEL_CACHE_PATH=/tmp/rapids-kernel-cache -fi - CMAKE_ARGS=( -S "${REPO_ROOT}/cpp" -B "${BUILD_DIR}" -GNinja -DCMAKE_INSTALL_PREFIX="${INSTALL_PREFIX}" + -DCMAKE_C_COMPILER="${CC}" + -DCMAKE_CXX_COMPILER="${CXX}" + -DCMAKE_CUDA_HOST_COMPILER="${CMAKE_CUDA_HOST_COMPILER}" -DBUILD_SHARED_LIBS=OFF -DBUILD_TESTS=OFF -DUSE_NVTX=ON @@ -77,19 +52,33 @@ CMAKE_ARGS=( -DRMM_LOGGING_LEVEL=OFF -DCUDF_KVIKIO_REMOTE_IO=OFF ) - -if [[ -n ${CMAKE_CUDA_ARCHITECTURES} ]]; then +if [[ -n ${CMAKE_CUDA_ARCHITECTURES:-} ]]; then CMAKE_ARGS+=("-DCMAKE_CUDA_ARCHITECTURES=${CMAKE_CUDA_ARCHITECTURES}") fi +# Forward the sccache launchers exported by setup_java_env.sh when present. +if [[ -n ${CMAKE_C_COMPILER_LAUNCHER:-} ]]; then + CMAKE_ARGS+=("-DCMAKE_C_COMPILER_LAUNCHER=${CMAKE_C_COMPILER_LAUNCHER}") +fi +if [[ -n ${CMAKE_CXX_COMPILER_LAUNCHER:-} ]]; then + CMAKE_ARGS+=("-DCMAKE_CXX_COMPILER_LAUNCHER=${CMAKE_CXX_COMPILER_LAUNCHER}") +fi +if [[ -n ${CMAKE_CUDA_COMPILER_LAUNCHER:-} ]]; then + CMAKE_ARGS+=("-DCMAKE_CUDA_COMPILER_LAUNCHER=${CMAKE_CUDA_COMPILER_LAUNCHER}") +fi +# setup_java_env.sh builds the static Boost archives into BOOST_PREFIX. +if [[ -d ${BOOST_PREFIX} ]]; then + CMAKE_ARGS+=("-DCMAKE_PREFIX_PATH=${BOOST_PREFIX}") +fi -rapids-logger "Configuring static libcudf" -cmake "${CMAKE_ARGS[@]}" - -rapids-logger "Building static libcudf with ${PARALLEL_LEVEL} jobs" +rapids-logger "Configuring/building static libcudf (cuda=${RAPIDS_CUDA_VERSION})" +cudf_java_scl cmake "${CMAKE_ARGS[@]}" cmake --build "${BUILD_DIR}" --parallel "${PARALLEL_LEVEL}" - -rapids-logger "Installing static libcudf to ${INSTALL_PREFIX}" cmake --install "${BUILD_DIR}" -rapids-logger "Chowning ${INSTALL_PREFIX} to ${HOST_UID}:${HOST_GID}" -chown -R "${HOST_UID}:${HOST_GID}" "${INSTALL_PREFIX}" +# Hand the install tree back to the host user (host wrapper passes HOST_UID/GID). +if [[ -n ${HOST_UID:-} && -n ${HOST_GID:-} ]]; then + chown -R "${HOST_UID}:${HOST_GID}" "${INSTALL_PREFIX}" +fi +if command -v sccache >/dev/null 2>&1; then + sccache --show-adv-stats || true +fi diff --git a/java/ci/ci_wheel_image.sh b/java/ci/ci_wheel_image.sh new file mode 100755 index 000000000000..8879406106ef --- /dev/null +++ b/java/ci/ci_wheel_image.sh @@ -0,0 +1,36 @@ +#!/bin/bash +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +# Image-tag helpers for Java packaging containers. Source, do not execute. +# +# cudf_java_normalize_cuda_version maps short forms (12.9, 13.3) to the full +# tags used by rapidsai/ci-wheel; cudf_java_ci_wheel_image builds the image +# name from VERSION + CUDA + RAPIDS_PY_VERSION. + +_java_ci_image_script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +_java_ci_image_repo_root="${REPO_ROOT:-$(git -C "${_java_ci_image_script_dir}" rev-parse --show-toplevel)}" + +cudf_java_normalize_cuda_version() { + local ver=$1 + case "${ver}" in + 12.9) + echo "12.9.2" + ;; + 13.3) + echo "13.3.0" + ;; + *) + echo "${ver}" + ;; + esac +} + +cudf_java_ci_wheel_image() { + local cuda_ver=${1:?cuda version required} + local rapids_ver cuda_full py_ver + rapids_ver="$(head -1 "${_java_ci_image_repo_root}/VERSION" | cut -d. -f1,2)" + cuda_full="$(cudf_java_normalize_cuda_version "${cuda_ver}")" + py_ver="${RAPIDS_PY_VERSION:-3.11}" + echo "rapidsai/ci-wheel:${rapids_ver}-cuda${cuda_full}-rockylinux8-py${py_ver}" +} diff --git a/java/ci/java_classifier.sh b/java/ci/java_classifier.sh new file mode 100644 index 000000000000..bd967f105fdf --- /dev/null +++ b/java/ci/java_classifier.sh @@ -0,0 +1,116 @@ +#!/bin/bash +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +# Maven classifier helpers for cuDF Java JARs. Source, do not execute. +# +# Derives the cuda[-arm64] classifier, locates the unique classifier JAR +# under a build output dir, and asserts the expected JAR + POM pair. + +# Print Maven classifier for a CUDA version + host arch (e.g. cuda12, cuda12-arm64). +cudf_java_maven_classifier() { + local cuda_ver=${1:?cuda version required} + local major host_arch + major="$(echo "${cuda_ver}" | cut -d. -f1)" + host_arch="$(uname -m)" + case "${host_arch}" in + x86_64) + echo "cuda${major}" + ;; + aarch64|arm64) + echo "cuda${major}-arm64" + ;; + *) + echo "Error: unsupported host arch '${host_arch}'" >&2 + return 1 + ;; + esac +} + +# Echo the unique cudf-*-${classifier}.jar under dir; error if 0 or >1. +cudf_java_find_classifier_jar() { + local dir=${1:?classifier out dir required} + local classifier=${2:?classifier required} + local jar="" candidate + for candidate in "${dir}"/cudf-*-"${classifier}".jar; do + if [[ ! -f ${candidate} ]]; then + continue + fi + if [[ -n ${jar} ]]; then + echo "Error: multiple JARs matching cudf-*-${classifier}.jar in ${dir}" >&2 + ls -1 "${dir}" >&2 + return 1 + fi + jar=${candidate} + done + if [[ -z ${jar} ]]; then + echo "Error: no cudf-*-${classifier}.jar in ${dir}" >&2 + ls -1 "${dir}" >&2 + return 1 + fi + echo "${jar}" +} + +# Assert unique classifier JAR + unique cudf-*.pom; print basenames. +cudf_java_assert_classifier_artifacts() { + local dir=${1:?classifier out dir required} + local classifier=${2:?classifier required} + local jar pom="" candidate + if ! jar="$(cudf_java_find_classifier_jar "${dir}" "${classifier}")"; then + return 1 + fi + for candidate in "${dir}"/cudf-*.pom; do + if [[ ! -f ${candidate} ]]; then + continue + fi + if [[ -n ${pom} ]]; then + echo "Error: multiple POMs in ${dir}" >&2 + ls -1 "${dir}" >&2 + return 1 + fi + pom=${candidate} + done + if [[ -z ${pom} ]]; then + echo "Error: no cudf-*.pom in ${dir}" >&2 + ls -1 "${dir}" >&2 + return 1 + fi + echo "cuDF Java JAR build succeeded:" + echo " $(basename "${jar}")" + echo " $(basename "${pom}")" +} + +# Echo the absolute path of the unique classifier JAR under a java-build +# artifact tree. Classifier from $2 or RAPIDS_CUDA_VERSION; errors if 0 or >1. +cudf_java_resolve_artifact_jar() { + local root=${1:?artifact root required} + local cuda_ver=${2:-${RAPIDS_CUDA_VERSION:-}} + local classifier + local -a matches + + if [[ -z ${cuda_ver} ]]; then + echo "Error: set RAPIDS_CUDA_VERSION or pass a cuda version" >&2 + return 1 + fi + if [[ ! -d ${root} ]]; then + echo "Error: artifact root '${root}' is not a directory" >&2 + return 1 + fi + if ! classifier="$(cudf_java_maven_classifier "${cuda_ver}")"; then + return 1 + fi + + # -${classifier}.jar excludes the sources/javadoc/tests JARs. + mapfile -t matches < <(find "${root}" -type f -name "cudf-*-${classifier}.jar" | sort) + if [[ ${#matches[@]} -eq 0 ]]; then + echo "Error: no cudf-*-${classifier}.jar under ${root}" >&2 + find "${root}" -type f >&2 + return 1 + fi + if [[ ${#matches[@]} -gt 1 ]]; then + echo "Error: multiple cudf-*-${classifier}.jar under ${root}" >&2 + printf '%s\n' "${matches[@]}" >&2 + return 1 + fi + realpath "${matches[0]}" +} diff --git a/java/ci/setup_java_env.sh b/java/ci/setup_java_env.sh new file mode 100644 index 000000000000..ab1bac253bc1 --- /dev/null +++ b/java/ci/setup_java_env.sh @@ -0,0 +1,128 @@ +#!/bin/bash +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +# Toolchain for Java builds inside a RAPIDS ci-wheel container. +# +# Installs/configures JDK 8 + 17, ninja, cmake, Boost.Filesystem/System static +# archives, the RAPIDS GCC toolset, and sccache. Safe to source repeatedly +# (JAVA_ENV_READY short-circuits after the first load). + +TOOLSET_VERSION="${TOOLSET_VERSION:-14}" +NINJA_VERSION="${NINJA_VERSION:-v1.13.1}" +BOOST_VERSION="${BOOST_VERSION:-1.79.0}" +BOOST_PREFIX="${BOOST_PREFIX:-/usr/local}" + +if [[ -n "${JAVA_ENV_READY:-}" ]]; then + return 0 +fi + +if ! type rapids-logger >/dev/null 2>&1; then + rapids-logger() { + echo ">>>> $*" >&2 + } +fi + +# JDK 8 for javac; JDK 17 for the -Pjavadoc-jdk17 Maven profile. +dnf install -y maven java-1.8.0-openjdk-devel java-17-openjdk-devel + +# ci-wheel images do not ship ninja; CMake uses it as the preferred generator. +if ! command -V ninja >/dev/null 2>&1; then + case "$(uname -m)" in + x86_64) + wget --no-hsts -q -O /tmp/ninja-linux.zip \ + "https://github.com/ninja-build/ninja/releases/download/${NINJA_VERSION}/ninja-linux.zip" + ;; + aarch64) + wget --no-hsts -q -O /tmp/ninja-linux.zip \ + "https://github.com/ninja-build/ninja/releases/download/${NINJA_VERSION}/ninja-linux-aarch64.zip" + ;; + *) + echo "Unrecognized platform '$(uname -m)'" >&2 + exit 1 + ;; + esac + unzip -d /usr/bin /tmp/ninja-linux.zip + chmod +x /usr/bin/ninja + rm -f /tmp/ninja-linux.zip +fi + +if command -v rapids-pip-retry >/dev/null 2>&1; then + rapids-pip-retry install cmake +else + pip install cmake +fi + +# Refresh pyenv shims so the cmake we just installed is on PATH. +if command -v pyenv >/dev/null 2>&1; then + pyenv rehash || true +fi + +# Static libcudf needs Boost.Filesystem / Boost.System. ci-wheel does not ship +# those static archives, so build them from source into BOOST_PREFIX when missing. +if [[ ! -f "${BOOST_PREFIX}/lib/libboost_filesystem.a" && \ + ! -f "${BOOST_PREFIX}/lib64/libboost_filesystem.a" ]]; then + BOOST_DIR="boost_${BOOST_VERSION//./_}" + wget -q -O /tmp/boost.tgz \ + "https://archives.boost.io/release/${BOOST_VERSION}/source/${BOOST_DIR}.tar.gz" + tar -xzf /tmp/boost.tgz -C /tmp + ( + if ! cd "/tmp/${BOOST_DIR}"; then + exit 1 + fi + ./bootstrap.sh --prefix="${BOOST_PREFIX}" + ./b2 install --prefix="${BOOST_PREFIX}" --with-filesystem --with-system -j"$(nproc)" + ) + rm -rf "/tmp/${BOOST_DIR}" /tmp/boost.tgz +fi + +if [[ -z "${CUDACXX:-}" ]]; then + if [[ -x /usr/local/cuda/bin/nvcc ]]; then + export CUDACXX=/usr/local/cuda/bin/nvcc + elif command -v nvcc >/dev/null 2>&1; then + CUDACXX="$(command -v nvcc)" + export CUDACXX + fi +fi + +export JAVA_HOME="${JAVA_HOME:-/usr/lib/jvm/java-1.8.0-openjdk}" +if [[ -z "${JDK17_HOME:-}" ]]; then + if [[ -d /usr/lib/jvm/java-17-openjdk ]]; then + export JDK17_HOME=/usr/lib/jvm/java-17-openjdk + else + JDK17_HOME="$(ls -d /usr/lib/jvm/java-17-openjdk* 2>/dev/null | head -1)" + export JDK17_HOME + fi +fi + +export LIBCUDF_KERNEL_CACHE_PATH="${LIBCUDF_KERNEL_CACHE_PATH:-/tmp/rapids-kernel-cache}" +mkdir -p "${LIBCUDF_KERNEL_CACHE_PATH}" +export CMAKE_GENERATOR="${CMAKE_GENERATOR:-Ninja}" +export PARALLEL_LEVEL="${PARALLEL_LEVEL:-$(nproc)}" + +# Prefer gcc-toolset over Rocky's default system compiler. +GCC_TOOLSET_ROOT="/opt/rh/gcc-toolset-${TOOLSET_VERSION}/root" +export CC="${CC:-${GCC_TOOLSET_ROOT}/usr/bin/gcc}" +export CXX="${CXX:-${GCC_TOOLSET_ROOT}/usr/bin/g++}" +export CMAKE_CUDA_HOST_COMPILER="${CMAKE_CUDA_HOST_COMPILER:-${CC}}" + +if command -v rapids-configure-sccache >/dev/null 2>&1; then + # shellcheck disable=SC1091 + source rapids-configure-sccache || true +fi + +if command -v sccache >/dev/null 2>&1; then + export CMAKE_C_COMPILER_LAUNCHER="${CMAKE_C_COMPILER_LAUNCHER:-sccache}" + export CMAKE_CXX_COMPILER_LAUNCHER="${CMAKE_CXX_COMPILER_LAUNCHER:-sccache}" + export CMAKE_CUDA_COMPILER_LAUNCHER="${CMAKE_CUDA_COMPILER_LAUNCHER:-sccache}" + # Restart so the next compile picks up the launcher env we just set. + sccache --stop-server 2>/dev/null || true +fi + +# Rocky SCL wrapper: child build steps run under gcc-toolset-${TOOLSET_VERSION}. +cudf_java_scl() { + scl enable "gcc-toolset-${TOOLSET_VERSION}" -- "$@" +} +export -f cudf_java_scl +export TOOLSET_VERSION BOOST_PREFIX +export JAVA_ENV_READY=1 diff --git a/java/ci/test_packaged_java_local.sh b/java/ci/test_packaged_java_local.sh new file mode 100755 index 000000000000..ee6a283d7d14 --- /dev/null +++ b/java/ci/test_packaged_java_local.sh @@ -0,0 +1,63 @@ +#!/bin/bash +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +# Local GPU wrapper around ci/test_packaged_java.sh. +# +# Resolves the classifier JAR from a test_java_build_local.sh --work-dir and +# runs the packaged-JAR tests in a GPU-enabled ci-wheel container. Set +# RAPIDS_CUDA_VERSION to select which classifier to test (default: 12.9). + +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +REPO_ROOT="$(git -C "${SCRIPT_DIR}" rev-parse --show-toplevel)" + +# shellcheck disable=SC1091 +. "${SCRIPT_DIR}/ci_wheel_image.sh" +# shellcheck disable=SC1091 +. "${SCRIPT_DIR}/java_classifier.sh" + +print_help() { + cat << EOF +Usage: test_packaged_java_local.sh --work-dir + +Uses the output from test_java_build_local.sh. Set RAPIDS_CUDA_VERSION to +select the classifier to test (default: 12.9). +EOF +} + +if [[ ${1:-} == "-h" || ${1:-} == "--help" ]]; then + print_help + exit 0 +fi +if [[ $# -ne 2 ]]; then + print_help + exit 1 +fi +if [[ $1 != "-w" && $1 != "--work-dir" ]]; then + print_help + exit 1 +fi + +WORK_DIR="$(cd "$2" && pwd)" +CUDA_VERSION="${RAPIDS_CUDA_VERSION:-12.9}" +CLASSIFIER="$(cudf_java_maven_classifier "${CUDA_VERSION}")" +JAR_PATH="$(realpath "$(cudf_java_find_classifier_jar "${WORK_DIR}/jars/${CLASSIFIER}" "${CLASSIFIER}")")" +IMAGE="$(cudf_java_ci_wheel_image "${CUDA_VERSION}")" +CUDA_VERSION_FULL="$(cudf_java_normalize_cuda_version "${CUDA_VERSION}")" + +echo "Running cuDF Java tests" +echo " image: ${IMAGE}" +echo " cuda version: ${CUDA_VERSION_FULL}" +echo " classifier: ${CLASSIFIER}" +echo " jar: ${JAR_PATH}" + +docker run --rm --gpus all \ + --volume "${REPO_ROOT}:/repo" \ + --volume "${JAR_PATH}:/product/cudf.jar:ro" \ + --workdir /repo \ + --env RAPIDS_CUDA_VERSION="${CUDA_VERSION_FULL}" \ + --env JAVA_JAR=/product/cudf.jar \ + --env LIBCUDF_LARGE_STRINGS_ENABLED=0 \ + "${IMAGE}" bash /repo/ci/test_packaged_java.sh diff --git a/java/pom.xml b/java/pom.xml index 8521de5567e4..6a9668c5067a 100644 --- a/java/pom.xml +++ b/java/pom.xml @@ -225,6 +225,92 @@ -Wno-deprecated-declarations + + + packaged-jar-tests + + true + + + + ai.rapids + cudf-packaged + ${project.version} + system + ${cudf.jar.path} + + + + + + maven-antrun-plugin + + + cmake + none + + + + + org.codehaus.gmavenplus + gmavenplus-plugin + + + setproperty + none + + + + + maven-compiler-plugin + + true + + + + + maven-resources-plugin + + + copy-native-libs + none + + + + + maven-surefire-plugin + + + packaged-jar-origin-check + + test + + + + + **/PackagedJarOriginCheck.java + + true + + ${cudf.jar.path} + + + + + + + + default-tests @@ -236,6 +322,7 @@ **/CudaFatalTest.java **/ColumnViewNonEmptyNullsTest.java **/NativeDepsLoaderTest.java + **/PackagedJarOriginCheck.java @@ -302,6 +389,7 @@ **/CuFileTest.java **/CudaFatalTest.java **/NativeDepsLoaderTest.java + **/PackagedJarOriginCheck.java @@ -705,9 +793,12 @@ + JVM via the dedicated execution defined in the test profiles. + PackagedJarOriginCheck needs a packaged classifier JAR and is + only run by the packaged-jar-tests profile. --> **/NativeDepsLoaderTest.java + **/PackagedJarOriginCheck.java diff --git a/java/src/test/java/ai/rapids/cudf/PackagedJarOriginCheck.java b/java/src/test/java/ai/rapids/cudf/PackagedJarOriginCheck.java new file mode 100644 index 000000000000..28a4793552ef --- /dev/null +++ b/java/src/test/java/ai/rapids/cudf/PackagedJarOriginCheck.java @@ -0,0 +1,30 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +package ai.rapids.cudf; + +import org.junit.jupiter.api.Test; + +import java.nio.file.Path; +import java.nio.file.Paths; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +/** + * Guard for {@code -Ppackaged-jar-tests}: fail fast if cuDF classes were loaded + * from {@code target/classes} (or any other path) instead of the packaged + * classifier JAR supplied via {@code -Dcudf.jar.path} / the + * {@code cudf.packaged.jar} system property. + */ +class PackagedJarOriginCheck { + @Test + void cudfClassesAreLoadedFromPackagedJar() throws Exception { + Path expected = Paths.get(System.getProperty("cudf.packaged.jar")).toRealPath(); + Path actual = Paths.get( + Cuda.class.getProtectionDomain().getCodeSource().getLocation().toURI()) + .toRealPath(); + assertEquals(expected, actual); + } +} From dd5776583c40cc036a6161c665cf9141638f53b1 Mon Sep 17 00:00:00 2001 From: Paul Aiyedun <53453937+paul-aiyedun@users.noreply.github.com> Date: Thu, 13 Aug 2026 05:35:58 +0000 Subject: [PATCH 02/10] Add missing dependent job --- .github/workflows/pr.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index 9bd7f621affc..d7993657d0cc 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -25,6 +25,7 @@ jobs: - conda-python-cudf-tests - conda-python-other-tests - conda-java-tests + - java-build-matrix - java-build - java-tests - conda-notebook-tests From ee5859d9ad016c0e55a3b255493b1af8c1bf08e5 Mon Sep 17 00:00:00 2001 From: Paul Aiyedun <53453937+paul-aiyedun@users.noreply.github.com> Date: Thu, 13 Aug 2026 15:23:45 +0000 Subject: [PATCH 03/10] Fix script value --- .github/workflows/build.yaml | 2 +- .github/workflows/pr.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index 356592e8bf70..7bd1a7732a4f 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -535,7 +535,7 @@ jobs: arch: ${{ matrix.ARCH }} node_type: cpu16 container_image: "rapidsai/ci-wheel:26.10-cuda${{ matrix.CUDA_VER }}-${{ matrix.LINUX_VER }}-py${{ matrix.PY_VER }}" - script: "RAPIDS_CUDA_VERSION=${{ matrix.CUDA_VER }} ci/build_java.sh" + script: "env RAPIDS_CUDA_VERSION=${{ matrix.CUDA_VER }} ci/build_java.sh" file_to_upload: output_jars artifact-name: java_pkg_${{ matrix.CUDA_VER }}_${{ matrix.ARCH }} # Assemble the per-classifier JARs into one Maven-repository layout. diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index d7993657d0cc..70c9257ee77f 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -558,7 +558,7 @@ jobs: arch: ${{ matrix.ARCH }} node_type: cpu16 container_image: "rapidsai/ci-wheel:26.10-cuda${{ matrix.CUDA_VER }}-${{ matrix.LINUX_VER }}-py${{ matrix.PY_VER }}" - script: "RAPIDS_CUDA_VERSION=${{ matrix.CUDA_VER }} ci/build_java.sh" + script: "env RAPIDS_CUDA_VERSION=${{ matrix.CUDA_VER }} ci/build_java.sh" file_to_upload: output_jars artifact-name: java_pkg_${{ matrix.CUDA_VER }}_${{ matrix.ARCH }} java-tests: From bdf3fc847bdffc61a2cd9525444d39cd745a87a3 Mon Sep 17 00:00:00 2001 From: Paul Aiyedun <53453937+paul-aiyedun@users.noreply.github.com> Date: Thu, 13 Aug 2026 15:32:55 +0000 Subject: [PATCH 04/10] TEMPORARY: trim down pr.yaml for faster CI test --- .github/workflows/pr.yaml | 113 +++++++++++++++++++------------------- 1 file changed, 58 insertions(+), 55 deletions(-) diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index 70c9257ee77f..51e231bb9f82 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -9,46 +9,16 @@ concurrency: permissions: {} jobs: # Please keep pr-builder as the top job here + # TEMPORARY: java packaging CI only — revert before merge. pr-builder: needs: - build-details - check-nightly-ci - changed-files - checks - - cmake-tests - - conda-cpp-build - - cpp-linters - - conda-cpp-checks - - conda-cpp-tests - - conda-python-build - - conda-python-build-noarch - - conda-python-cudf-tests - - conda-python-other-tests - - conda-java-tests - - java-build-matrix - java-build - java-tests - - conda-notebook-tests - - docs-build - - publish-api-docs - - wheel-build-libcudf - - wheel-build-libcudf-streaming - - wheel-build-cudf-streaming - - wheel-tests-cudf-streaming - - wheel-build-pylibcudf - - wheel-build-cudf - - wheel-tests-cudf - - wheel-build-cudf-polars - - wheel-tests-cudf-polars - - cudf-polars-polars-tests - - wheel-build-dask-cudf - - wheel-tests-dask-cudf - - devcontainer - - unit-tests-cudf-pandas - - pandas-tests - - narwhals-tests - telemetry-setup - - third-party-integration-tests-cudf-pandas uses: rapidsai/shared-workflows/.github/workflows/pr-builder.yaml@main permissions: contents: read @@ -373,8 +343,20 @@ jobs: uses: rapidsai/shared-workflows/.github/workflows/checks.yaml@main with: enable_check_generated_files: false - ignored_pr_jobs: "telemetry-summarize spark-rapids-jni cuml-compat-tests" + ignored_pr_jobs: >- + telemetry-summarize spark-rapids-jni cuml-compat-tests + cmake-tests conda-cpp-build cpp-linters conda-cpp-checks conda-cpp-tests + conda-python-build conda-python-build-noarch conda-python-cudf-tests + conda-python-other-tests conda-java-tests java-build-matrix + conda-notebook-tests docs-build publish-api-docs + wheel-build-libcudf wheel-build-libcudf-streaming wheel-build-cudf-streaming + wheel-tests-cudf-streaming wheel-build-pylibcudf wheel-build-cudf + wheel-tests-cudf wheel-build-cudf-polars wheel-tests-cudf-polars + cudf-polars-polars-tests wheel-build-dask-cudf wheel-tests-dask-cudf + devcontainer unit-tests-cudf-pandas pandas-tests narwhals-tests + third-party-integration-tests-cudf-pandas conda-cpp-build: + if: false # TEMPORARY: java-only CI needs: [build-details, checks] permissions: actions: read @@ -390,6 +372,7 @@ jobs: node_type: cpu16 script: ci/build_cpp.sh cmake-tests: + if: false # TEMPORARY: java-only CI needs: checks permissions: actions: read @@ -406,6 +389,7 @@ jobs: container_image: "rapidsai/ci-conda:26.10-latest" script: "ci/test_cmake.sh" cpp-linters: + if: false # TEMPORARY: java-only CI permissions: actions: read contents: read @@ -419,6 +403,7 @@ jobs: build_type: pull-request script: "ci/cpp_linters.sh" conda-cpp-checks: + if: false # TEMPORARY: java-only CI needs: conda-cpp-build permissions: actions: read @@ -440,13 +425,14 @@ jobs: pull-requests: read secrets: inherit # zizmor: ignore[secrets-inherit] uses: rapidsai/shared-workflows/.github/workflows/conda-cpp-tests.yaml@main - if: fromJSON(needs.changed-files.outputs.changed_file_groups).test_cpp + if: false # TEMPORARY: java-only CI with: build_type: pull-request script: ci/test_cpp.sh # https://github.com/rapidsai/cudf/issues/23498 matrix_filter: map(select(.GPU != "gb300" and .GPU != "gh200")) conda-python-build: + if: false # TEMPORARY: java-only CI needs: [build-details, conda-cpp-build] permissions: actions: read @@ -463,6 +449,7 @@ jobs: # Build a conda package for each CUDA x ARCH x minimum supported Python version matrix_filter: group_by({CUDA_VER, ARCH}) | map(min_by(.PY_VER | split(".") | map(tonumber))) conda-python-build-noarch: + if: false # TEMPORARY: java-only CI needs: [build-details, conda-cpp-build, conda-python-build] permissions: actions: read @@ -487,7 +474,7 @@ jobs: pull-requests: read secrets: inherit # zizmor: ignore[secrets-inherit] uses: rapidsai/shared-workflows/.github/workflows/conda-python-tests.yaml@main - if: fromJSON(needs.changed-files.outputs.changed_file_groups).test_python_conda && fromJSON(needs.changed-files.outputs.changed_file_groups).neither_cudf_polars_nor_dask_cudf + if: false # TEMPORARY: java-only CI with: build_type: pull-request script: "ci/test_python_cudf.sh" @@ -504,7 +491,7 @@ jobs: pull-requests: read secrets: inherit # zizmor: ignore[secrets-inherit] uses: rapidsai/shared-workflows/.github/workflows/conda-python-tests.yaml@main - if: fromJSON(needs.changed-files.outputs.changed_file_groups).test_python_conda + if: false # TEMPORARY: java-only CI with: build_type: pull-request # https://github.com/rapidsai/cudf/pull/22381/changes#r3196736965 @@ -522,7 +509,7 @@ jobs: pull-requests: read secrets: inherit # zizmor: ignore[secrets-inherit] uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@main - if: fromJSON(needs.changed-files.outputs.changed_file_groups).test_java + if: false # TEMPORARY: java-only CI with: build_type: pull-request node_type: "gpu-l4-latest-1" @@ -530,7 +517,9 @@ jobs: container_image: "rapidsai/ci-conda:26.10-latest" script: "ci/test_java.sh" # Build and test the cuDF Java JAR for every Maven classifier. + # TEMPORARY: single amd64/cuda12 cell for faster CI; restore full matrix before merge. java-build-matrix: + if: false # TEMPORARY: java-only CI needs: [checks] permissions: contents: read @@ -539,7 +528,7 @@ jobs: build_type: pull-request matrix_name: conda-cpp-build java-build: - needs: [build-details, java-build-matrix, changed-files] + needs: [build-details, checks, changed-files] permissions: actions: read contents: read @@ -548,10 +537,13 @@ jobs: pull-requests: read secrets: inherit # zizmor: ignore[secrets-inherit] uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@main - if: fromJSON(needs.changed-files.outputs.changed_file_groups).test_java + # TEMPORARY: always run (skip changed-files gate) + if: true strategy: fail-fast: false - matrix: ${{ fromJSON(needs.java-build-matrix.outputs.matrix) }} + matrix: + include: + - { ARCH: "amd64", CUDA_VER: "12.9", LINUX_VER: "rockylinux8", PY_VER: "3.11" } with: build_type: pull-request build-datetime: ${{ needs.build-details.outputs.build-datetime }} @@ -562,7 +554,7 @@ jobs: file_to_upload: output_jars artifact-name: java_pkg_${{ matrix.CUDA_VER }}_${{ matrix.ARCH }} java-tests: - needs: [java-build, java-build-matrix] + needs: [java-build] if: ${{ !cancelled() && needs.java-build.result == 'success' }} permissions: actions: read @@ -570,7 +562,9 @@ jobs: packages: read strategy: fail-fast: false - matrix: ${{ fromJSON(needs.java-build-matrix.outputs.matrix) }} + matrix: + include: + - { ARCH: "amd64", CUDA_VER: "12.9", LINUX_VER: "rockylinux8", PY_VER: "3.11" } runs-on: linux-${{ matrix.ARCH }}-gpu-l4-latest-1 container: image: rapidsai/ci-wheel:26.10-cuda${{ matrix.CUDA_VER }}-${{ matrix.LINUX_VER }}-py${{ matrix.PY_VER }} # zizmor: ignore[unpinned-images] @@ -608,7 +602,7 @@ jobs: pull-requests: read secrets: inherit # zizmor: ignore[secrets-inherit] uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@main - if: fromJSON(needs.changed-files.outputs.changed_file_groups).test_notebooks + if: false # TEMPORARY: java-only CI with: build_type: pull-request node_type: "gpu-l4-latest-1" @@ -625,7 +619,7 @@ jobs: pull-requests: read secrets: inherit # zizmor: ignore[secrets-inherit] uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@main - if: fromJSON(needs.changed-files.outputs.changed_file_groups).build_docs + if: false # TEMPORARY: java-only CI with: build_type: pull-request node_type: "gpu-l4-latest-1" @@ -633,6 +627,7 @@ jobs: container_image: "rapidsai/ci-conda:26.10-latest" script: "ci/build_docs.sh" publish-api-docs: + if: false # TEMPORARY: java-only CI needs: [docs-build] permissions: contents: read @@ -653,6 +648,7 @@ jobs: dry-run: true version-map: '{"26.04": "legacy", "26.06": "stable"}' wheel-build-libcudf: + if: false # TEMPORARY: java-only CI needs: [build-details, checks] permissions: actions: read @@ -672,6 +668,7 @@ jobs: package-name: libcudf package-type: cpp wheel-build-libcudf-streaming: + if: false # TEMPORARY: java-only CI needs: [build-details, checks, wheel-build-libcudf] permissions: actions: read @@ -691,6 +688,7 @@ jobs: package-name: libcudf_streaming package-type: cpp wheel-build-cudf-streaming: + if: false # TEMPORARY: java-only CI needs: [build-details, checks, wheel-build-libcudf-streaming, wheel-build-pylibcudf] permissions: actions: read @@ -719,13 +717,14 @@ jobs: pull-requests: read secrets: inherit # zizmor: ignore[secrets-inherit] uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@main - if: fromJSON(needs.changed-files.outputs.changed_file_groups).test_python_wheels + if: false # TEMPORARY: java-only CI with: build_type: pull-request script: ci/test_wheel_cudf_streaming.sh # https://github.com/rapidsai/cudf/issues/23498 matrix_filter: map(select(.GPU != "gb300" and .GPU != "gh200")) wheel-build-pylibcudf: + if: false # TEMPORARY: java-only CI needs: [build-details, checks, wheel-build-libcudf] permissions: actions: read @@ -745,6 +744,7 @@ jobs: # Build a wheel for each CUDA x ARCH x minimum supported Python version matrix_filter: group_by({CUDA_VER, ARCH}) | map(min_by(.PY_VER | split(".") | map(tonumber))) wheel-build-cudf: + if: false # TEMPORARY: java-only CI needs: [build-details, wheel-build-pylibcudf] permissions: actions: read @@ -773,13 +773,14 @@ jobs: pull-requests: read secrets: inherit # zizmor: ignore[secrets-inherit] uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@main - if: fromJSON(needs.changed-files.outputs.changed_file_groups).test_python_wheels && fromJSON(needs.changed-files.outputs.changed_file_groups).neither_cudf_polars_nor_dask_cudf + if: false # TEMPORARY: java-only CI with: build_type: pull-request script: ci/test_wheel_cudf.sh # https://github.com/rapidsai/cudf/issues/23498 matrix_filter: map(select(.GPU != "gb300" and .GPU != "gh200")) wheel-build-cudf-polars: + if: false # TEMPORARY: java-only CI needs: [build-details, wheel-build-pylibcudf] permissions: actions: read @@ -809,7 +810,7 @@ jobs: pull-requests: read secrets: inherit # zizmor: ignore[secrets-inherit] uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@main - if: fromJSON(needs.changed-files.outputs.changed_file_groups).test_python_wheels && fromJSON(needs.changed-files.outputs.changed_file_groups).neither_cudf_nor_dask_cudf + if: false # TEMPORARY: java-only CI with: # This selects "ARCH=amd64 + the latest supported Python + CUDA". matrix_filter: map(select(.ARCH == "amd64")) | group_by(.CUDA_VER|split(".")|map(tonumber)|.[0]) | map(max_by([(.PY_VER|split(".")|map(tonumber)), (.CUDA_VER|split(".")|map(tonumber))])) @@ -826,13 +827,14 @@ jobs: pull-requests: read secrets: inherit # zizmor: ignore[secrets-inherit] uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@main - if: fromJSON(needs.changed-files.outputs.changed_file_groups).test_python_wheels && fromJSON(needs.changed-files.outputs.changed_file_groups).neither_cudf_nor_dask_cudf + if: false # TEMPORARY: java-only CI with: # This selects "ARCH=amd64 + the latest supported Python + CUDA". matrix_filter: map(select(.ARCH == "amd64")) | group_by(.CUDA_VER|split(".")|map(tonumber)|.[0]) | map(max_by([(.PY_VER|split(".")|map(tonumber)), (.CUDA_VER|split(".")|map(tonumber))])) build_type: pull-request script: "ci/test_cudf_polars_polars_tests.sh" wheel-build-dask-cudf: + if: false # TEMPORARY: java-only CI needs: [build-details, wheel-build-cudf] permissions: actions: read @@ -862,7 +864,7 @@ jobs: pull-requests: read secrets: inherit # zizmor: ignore[secrets-inherit] uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@main - if: fromJSON(needs.changed-files.outputs.changed_file_groups).test_python_wheels && fromJSON(needs.changed-files.outputs.changed_file_groups).not_cudf_polars + if: false # TEMPORARY: java-only CI with: # This selects "ARCH=amd64 + the latest supported Python + CUDA". matrix_filter: map(select(.ARCH == "amd64")) | group_by(.CUDA_VER|split(".")|map(tonumber)|.[0]) | map(max_by([(.PY_VER|split(".")|map(tonumber)), (.CUDA_VER|split(".")|map(tonumber))])) @@ -870,6 +872,7 @@ jobs: script: ci/test_wheel_dask_cudf.sh # build cudf with benchmarks but disable for cudf_streaming since we can't use MPI/UCX in wheel-based builds devcontainer: + if: false # TEMPORARY: java-only CI permissions: actions: read contents: read @@ -902,7 +905,7 @@ jobs: pull-requests: read secrets: inherit # zizmor: ignore[secrets-inherit] uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@main - if: (fromJSON(needs.changed-files.outputs.changed_file_groups).test_python_wheels || fromJSON(needs.changed-files.outputs.changed_file_groups).test_cudf_pandas) && fromJSON(needs.changed-files.outputs.changed_file_groups).neither_cudf_polars_nor_dask_cudf + if: false # TEMPORARY: java-only CI with: # This selects the latest supported Python + CUDA minor versions for each ARCH/CUDA major version combo # Filter out GB300 due to https://github.com/rapidsai/cudf/issues/23498 @@ -919,7 +922,7 @@ jobs: pull-requests: read secrets: inherit # zizmor: ignore[secrets-inherit] uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@main - if: fromJSON(needs.changed-files.outputs.changed_file_groups).test_python_conda && fromJSON(needs.changed-files.outputs.changed_file_groups).neither_cpp_nor_cudf_polars_nor_dask_cudf + if: false # TEMPORARY: java-only CI with: build_type: pull-request branch: ${{ inputs.branch }} @@ -940,7 +943,7 @@ jobs: pull-requests: read secrets: inherit # zizmor: ignore[secrets-inherit] uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@main - if: fromJSON(needs.changed-files.outputs.changed_file_groups).test_python_conda && fromJSON(needs.changed-files.outputs.changed_file_groups).neither_cudf_polars_nor_dask_cudf + if: false # TEMPORARY: java-only CI with: build_type: pull-request branch: ${{ inputs.branch }} @@ -961,7 +964,7 @@ jobs: pull-requests: read secrets: inherit # zizmor: ignore[secrets-inherit] uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@main - if: fromJSON(needs.changed-files.outputs.changed_file_groups).test_python_wheels && fromJSON(needs.changed-files.outputs.changed_file_groups).neither_cpp_nor_cudf_polars_nor_dask_cudf + if: false # TEMPORARY: java-only CI with: build_type: pull-request branch: ${{ inputs.branch }} @@ -980,7 +983,7 @@ jobs: pull-requests: read secrets: inherit # zizmor: ignore[secrets-inherit] uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@main - if: fromJSON(needs.changed-files.outputs.changed_file_groups).test_python_conda && fromJSON(needs.changed-files.outputs.changed_file_groups).neither_cpp_nor_cudf_polars_nor_dask_cudf + if: false # TEMPORARY: java-only CI with: build_type: pull-request branch: ${{ inputs.branch }} @@ -990,6 +993,7 @@ jobs: container_image: "rapidsai/ci-conda:26.10-latest" script: ci/test_narwhals.sh spark-rapids-jni: + if: false # TEMPORARY: java-only CI needs: changed-files permissions: actions: read @@ -998,14 +1002,13 @@ jobs: packages: read pull-requests: read uses: ./.github/workflows/spark-rapids-jni.yaml - if: fromJSON(needs.changed-files.outputs.changed_file_groups).test_java telemetry-summarize: + if: false # TEMPORARY: java-only CI # This job must use a self-hosted runner to record telemetry traces. runs-on: linux-amd64-cpu4 permissions: actions: write needs: pr-builder - if: ${{ vars.TELEMETRY_ENABLED == 'true' && !cancelled() }} continue-on-error: true steps: - name: Telemetry summarize From e2a1eec88059506d7650aaa856a976acd43337e6 Mon Sep 17 00:00:00 2001 From: Paul Aiyedun <53453937+paul-aiyedun@users.noreply.github.com> Date: Thu, 13 Aug 2026 15:42:44 +0000 Subject: [PATCH 05/10] Fix cuda version --- .github/workflows/pr.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index 51e231bb9f82..8ea73cf8b357 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -543,7 +543,7 @@ jobs: fail-fast: false matrix: include: - - { ARCH: "amd64", CUDA_VER: "12.9", LINUX_VER: "rockylinux8", PY_VER: "3.11" } + - { ARCH: "amd64", CUDA_VER: "12.9.2", LINUX_VER: "rockylinux8", PY_VER: "3.11" } with: build_type: pull-request build-datetime: ${{ needs.build-details.outputs.build-datetime }} @@ -564,7 +564,7 @@ jobs: fail-fast: false matrix: include: - - { ARCH: "amd64", CUDA_VER: "12.9", LINUX_VER: "rockylinux8", PY_VER: "3.11" } + - { ARCH: "amd64", CUDA_VER: "12.9.2", LINUX_VER: "rockylinux8", PY_VER: "3.11" } runs-on: linux-${{ matrix.ARCH }}-gpu-l4-latest-1 container: image: rapidsai/ci-wheel:26.10-cuda${{ matrix.CUDA_VER }}-${{ matrix.LINUX_VER }}-py${{ matrix.PY_VER }} # zizmor: ignore[unpinned-images] From 74fedeef024b1d1c1d8d0119e7c9e8052609501f Mon Sep 17 00:00:00 2001 From: Paul Aiyedun <53453937+paul-aiyedun@users.noreply.github.com> Date: Thu, 13 Aug 2026 20:00:52 +0000 Subject: [PATCH 06/10] Revert "Fix cuda version" This reverts commit e2a1eec88059506d7650aaa856a976acd43337e6. --- .github/workflows/pr.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index 8ea73cf8b357..51e231bb9f82 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -543,7 +543,7 @@ jobs: fail-fast: false matrix: include: - - { ARCH: "amd64", CUDA_VER: "12.9.2", LINUX_VER: "rockylinux8", PY_VER: "3.11" } + - { ARCH: "amd64", CUDA_VER: "12.9", LINUX_VER: "rockylinux8", PY_VER: "3.11" } with: build_type: pull-request build-datetime: ${{ needs.build-details.outputs.build-datetime }} @@ -564,7 +564,7 @@ jobs: fail-fast: false matrix: include: - - { ARCH: "amd64", CUDA_VER: "12.9.2", LINUX_VER: "rockylinux8", PY_VER: "3.11" } + - { ARCH: "amd64", CUDA_VER: "12.9", LINUX_VER: "rockylinux8", PY_VER: "3.11" } runs-on: linux-${{ matrix.ARCH }}-gpu-l4-latest-1 container: image: rapidsai/ci-wheel:26.10-cuda${{ matrix.CUDA_VER }}-${{ matrix.LINUX_VER }}-py${{ matrix.PY_VER }} # zizmor: ignore[unpinned-images] From f59e0fff01d01f4ac6a957847a7180e03deacd00 Mon Sep 17 00:00:00 2001 From: Paul Aiyedun <53453937+paul-aiyedun@users.noreply.github.com> Date: Thu, 13 Aug 2026 20:01:08 +0000 Subject: [PATCH 07/10] Revert "TEMPORARY: trim down pr.yaml for faster CI test" This reverts commit bdf3fc847bdffc61a2cd9525444d39cd745a87a3. --- .github/workflows/pr.yaml | 113 +++++++++++++++++++------------------- 1 file changed, 55 insertions(+), 58 deletions(-) diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index 51e231bb9f82..70c9257ee77f 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -9,16 +9,46 @@ concurrency: permissions: {} jobs: # Please keep pr-builder as the top job here - # TEMPORARY: java packaging CI only — revert before merge. pr-builder: needs: - build-details - check-nightly-ci - changed-files - checks + - cmake-tests + - conda-cpp-build + - cpp-linters + - conda-cpp-checks + - conda-cpp-tests + - conda-python-build + - conda-python-build-noarch + - conda-python-cudf-tests + - conda-python-other-tests + - conda-java-tests + - java-build-matrix - java-build - java-tests + - conda-notebook-tests + - docs-build + - publish-api-docs + - wheel-build-libcudf + - wheel-build-libcudf-streaming + - wheel-build-cudf-streaming + - wheel-tests-cudf-streaming + - wheel-build-pylibcudf + - wheel-build-cudf + - wheel-tests-cudf + - wheel-build-cudf-polars + - wheel-tests-cudf-polars + - cudf-polars-polars-tests + - wheel-build-dask-cudf + - wheel-tests-dask-cudf + - devcontainer + - unit-tests-cudf-pandas + - pandas-tests + - narwhals-tests - telemetry-setup + - third-party-integration-tests-cudf-pandas uses: rapidsai/shared-workflows/.github/workflows/pr-builder.yaml@main permissions: contents: read @@ -343,20 +373,8 @@ jobs: uses: rapidsai/shared-workflows/.github/workflows/checks.yaml@main with: enable_check_generated_files: false - ignored_pr_jobs: >- - telemetry-summarize spark-rapids-jni cuml-compat-tests - cmake-tests conda-cpp-build cpp-linters conda-cpp-checks conda-cpp-tests - conda-python-build conda-python-build-noarch conda-python-cudf-tests - conda-python-other-tests conda-java-tests java-build-matrix - conda-notebook-tests docs-build publish-api-docs - wheel-build-libcudf wheel-build-libcudf-streaming wheel-build-cudf-streaming - wheel-tests-cudf-streaming wheel-build-pylibcudf wheel-build-cudf - wheel-tests-cudf wheel-build-cudf-polars wheel-tests-cudf-polars - cudf-polars-polars-tests wheel-build-dask-cudf wheel-tests-dask-cudf - devcontainer unit-tests-cudf-pandas pandas-tests narwhals-tests - third-party-integration-tests-cudf-pandas + ignored_pr_jobs: "telemetry-summarize spark-rapids-jni cuml-compat-tests" conda-cpp-build: - if: false # TEMPORARY: java-only CI needs: [build-details, checks] permissions: actions: read @@ -372,7 +390,6 @@ jobs: node_type: cpu16 script: ci/build_cpp.sh cmake-tests: - if: false # TEMPORARY: java-only CI needs: checks permissions: actions: read @@ -389,7 +406,6 @@ jobs: container_image: "rapidsai/ci-conda:26.10-latest" script: "ci/test_cmake.sh" cpp-linters: - if: false # TEMPORARY: java-only CI permissions: actions: read contents: read @@ -403,7 +419,6 @@ jobs: build_type: pull-request script: "ci/cpp_linters.sh" conda-cpp-checks: - if: false # TEMPORARY: java-only CI needs: conda-cpp-build permissions: actions: read @@ -425,14 +440,13 @@ jobs: pull-requests: read secrets: inherit # zizmor: ignore[secrets-inherit] uses: rapidsai/shared-workflows/.github/workflows/conda-cpp-tests.yaml@main - if: false # TEMPORARY: java-only CI + if: fromJSON(needs.changed-files.outputs.changed_file_groups).test_cpp with: build_type: pull-request script: ci/test_cpp.sh # https://github.com/rapidsai/cudf/issues/23498 matrix_filter: map(select(.GPU != "gb300" and .GPU != "gh200")) conda-python-build: - if: false # TEMPORARY: java-only CI needs: [build-details, conda-cpp-build] permissions: actions: read @@ -449,7 +463,6 @@ jobs: # Build a conda package for each CUDA x ARCH x minimum supported Python version matrix_filter: group_by({CUDA_VER, ARCH}) | map(min_by(.PY_VER | split(".") | map(tonumber))) conda-python-build-noarch: - if: false # TEMPORARY: java-only CI needs: [build-details, conda-cpp-build, conda-python-build] permissions: actions: read @@ -474,7 +487,7 @@ jobs: pull-requests: read secrets: inherit # zizmor: ignore[secrets-inherit] uses: rapidsai/shared-workflows/.github/workflows/conda-python-tests.yaml@main - if: false # TEMPORARY: java-only CI + if: fromJSON(needs.changed-files.outputs.changed_file_groups).test_python_conda && fromJSON(needs.changed-files.outputs.changed_file_groups).neither_cudf_polars_nor_dask_cudf with: build_type: pull-request script: "ci/test_python_cudf.sh" @@ -491,7 +504,7 @@ jobs: pull-requests: read secrets: inherit # zizmor: ignore[secrets-inherit] uses: rapidsai/shared-workflows/.github/workflows/conda-python-tests.yaml@main - if: false # TEMPORARY: java-only CI + if: fromJSON(needs.changed-files.outputs.changed_file_groups).test_python_conda with: build_type: pull-request # https://github.com/rapidsai/cudf/pull/22381/changes#r3196736965 @@ -509,7 +522,7 @@ jobs: pull-requests: read secrets: inherit # zizmor: ignore[secrets-inherit] uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@main - if: false # TEMPORARY: java-only CI + if: fromJSON(needs.changed-files.outputs.changed_file_groups).test_java with: build_type: pull-request node_type: "gpu-l4-latest-1" @@ -517,9 +530,7 @@ jobs: container_image: "rapidsai/ci-conda:26.10-latest" script: "ci/test_java.sh" # Build and test the cuDF Java JAR for every Maven classifier. - # TEMPORARY: single amd64/cuda12 cell for faster CI; restore full matrix before merge. java-build-matrix: - if: false # TEMPORARY: java-only CI needs: [checks] permissions: contents: read @@ -528,7 +539,7 @@ jobs: build_type: pull-request matrix_name: conda-cpp-build java-build: - needs: [build-details, checks, changed-files] + needs: [build-details, java-build-matrix, changed-files] permissions: actions: read contents: read @@ -537,13 +548,10 @@ jobs: pull-requests: read secrets: inherit # zizmor: ignore[secrets-inherit] uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@main - # TEMPORARY: always run (skip changed-files gate) - if: true + if: fromJSON(needs.changed-files.outputs.changed_file_groups).test_java strategy: fail-fast: false - matrix: - include: - - { ARCH: "amd64", CUDA_VER: "12.9", LINUX_VER: "rockylinux8", PY_VER: "3.11" } + matrix: ${{ fromJSON(needs.java-build-matrix.outputs.matrix) }} with: build_type: pull-request build-datetime: ${{ needs.build-details.outputs.build-datetime }} @@ -554,7 +562,7 @@ jobs: file_to_upload: output_jars artifact-name: java_pkg_${{ matrix.CUDA_VER }}_${{ matrix.ARCH }} java-tests: - needs: [java-build] + needs: [java-build, java-build-matrix] if: ${{ !cancelled() && needs.java-build.result == 'success' }} permissions: actions: read @@ -562,9 +570,7 @@ jobs: packages: read strategy: fail-fast: false - matrix: - include: - - { ARCH: "amd64", CUDA_VER: "12.9", LINUX_VER: "rockylinux8", PY_VER: "3.11" } + matrix: ${{ fromJSON(needs.java-build-matrix.outputs.matrix) }} runs-on: linux-${{ matrix.ARCH }}-gpu-l4-latest-1 container: image: rapidsai/ci-wheel:26.10-cuda${{ matrix.CUDA_VER }}-${{ matrix.LINUX_VER }}-py${{ matrix.PY_VER }} # zizmor: ignore[unpinned-images] @@ -602,7 +608,7 @@ jobs: pull-requests: read secrets: inherit # zizmor: ignore[secrets-inherit] uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@main - if: false # TEMPORARY: java-only CI + if: fromJSON(needs.changed-files.outputs.changed_file_groups).test_notebooks with: build_type: pull-request node_type: "gpu-l4-latest-1" @@ -619,7 +625,7 @@ jobs: pull-requests: read secrets: inherit # zizmor: ignore[secrets-inherit] uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@main - if: false # TEMPORARY: java-only CI + if: fromJSON(needs.changed-files.outputs.changed_file_groups).build_docs with: build_type: pull-request node_type: "gpu-l4-latest-1" @@ -627,7 +633,6 @@ jobs: container_image: "rapidsai/ci-conda:26.10-latest" script: "ci/build_docs.sh" publish-api-docs: - if: false # TEMPORARY: java-only CI needs: [docs-build] permissions: contents: read @@ -648,7 +653,6 @@ jobs: dry-run: true version-map: '{"26.04": "legacy", "26.06": "stable"}' wheel-build-libcudf: - if: false # TEMPORARY: java-only CI needs: [build-details, checks] permissions: actions: read @@ -668,7 +672,6 @@ jobs: package-name: libcudf package-type: cpp wheel-build-libcudf-streaming: - if: false # TEMPORARY: java-only CI needs: [build-details, checks, wheel-build-libcudf] permissions: actions: read @@ -688,7 +691,6 @@ jobs: package-name: libcudf_streaming package-type: cpp wheel-build-cudf-streaming: - if: false # TEMPORARY: java-only CI needs: [build-details, checks, wheel-build-libcudf-streaming, wheel-build-pylibcudf] permissions: actions: read @@ -717,14 +719,13 @@ jobs: pull-requests: read secrets: inherit # zizmor: ignore[secrets-inherit] uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@main - if: false # TEMPORARY: java-only CI + if: fromJSON(needs.changed-files.outputs.changed_file_groups).test_python_wheels with: build_type: pull-request script: ci/test_wheel_cudf_streaming.sh # https://github.com/rapidsai/cudf/issues/23498 matrix_filter: map(select(.GPU != "gb300" and .GPU != "gh200")) wheel-build-pylibcudf: - if: false # TEMPORARY: java-only CI needs: [build-details, checks, wheel-build-libcudf] permissions: actions: read @@ -744,7 +745,6 @@ jobs: # Build a wheel for each CUDA x ARCH x minimum supported Python version matrix_filter: group_by({CUDA_VER, ARCH}) | map(min_by(.PY_VER | split(".") | map(tonumber))) wheel-build-cudf: - if: false # TEMPORARY: java-only CI needs: [build-details, wheel-build-pylibcudf] permissions: actions: read @@ -773,14 +773,13 @@ jobs: pull-requests: read secrets: inherit # zizmor: ignore[secrets-inherit] uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@main - if: false # TEMPORARY: java-only CI + if: fromJSON(needs.changed-files.outputs.changed_file_groups).test_python_wheels && fromJSON(needs.changed-files.outputs.changed_file_groups).neither_cudf_polars_nor_dask_cudf with: build_type: pull-request script: ci/test_wheel_cudf.sh # https://github.com/rapidsai/cudf/issues/23498 matrix_filter: map(select(.GPU != "gb300" and .GPU != "gh200")) wheel-build-cudf-polars: - if: false # TEMPORARY: java-only CI needs: [build-details, wheel-build-pylibcudf] permissions: actions: read @@ -810,7 +809,7 @@ jobs: pull-requests: read secrets: inherit # zizmor: ignore[secrets-inherit] uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@main - if: false # TEMPORARY: java-only CI + if: fromJSON(needs.changed-files.outputs.changed_file_groups).test_python_wheels && fromJSON(needs.changed-files.outputs.changed_file_groups).neither_cudf_nor_dask_cudf with: # This selects "ARCH=amd64 + the latest supported Python + CUDA". matrix_filter: map(select(.ARCH == "amd64")) | group_by(.CUDA_VER|split(".")|map(tonumber)|.[0]) | map(max_by([(.PY_VER|split(".")|map(tonumber)), (.CUDA_VER|split(".")|map(tonumber))])) @@ -827,14 +826,13 @@ jobs: pull-requests: read secrets: inherit # zizmor: ignore[secrets-inherit] uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@main - if: false # TEMPORARY: java-only CI + if: fromJSON(needs.changed-files.outputs.changed_file_groups).test_python_wheels && fromJSON(needs.changed-files.outputs.changed_file_groups).neither_cudf_nor_dask_cudf with: # This selects "ARCH=amd64 + the latest supported Python + CUDA". matrix_filter: map(select(.ARCH == "amd64")) | group_by(.CUDA_VER|split(".")|map(tonumber)|.[0]) | map(max_by([(.PY_VER|split(".")|map(tonumber)), (.CUDA_VER|split(".")|map(tonumber))])) build_type: pull-request script: "ci/test_cudf_polars_polars_tests.sh" wheel-build-dask-cudf: - if: false # TEMPORARY: java-only CI needs: [build-details, wheel-build-cudf] permissions: actions: read @@ -864,7 +862,7 @@ jobs: pull-requests: read secrets: inherit # zizmor: ignore[secrets-inherit] uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@main - if: false # TEMPORARY: java-only CI + if: fromJSON(needs.changed-files.outputs.changed_file_groups).test_python_wheels && fromJSON(needs.changed-files.outputs.changed_file_groups).not_cudf_polars with: # This selects "ARCH=amd64 + the latest supported Python + CUDA". matrix_filter: map(select(.ARCH == "amd64")) | group_by(.CUDA_VER|split(".")|map(tonumber)|.[0]) | map(max_by([(.PY_VER|split(".")|map(tonumber)), (.CUDA_VER|split(".")|map(tonumber))])) @@ -872,7 +870,6 @@ jobs: script: ci/test_wheel_dask_cudf.sh # build cudf with benchmarks but disable for cudf_streaming since we can't use MPI/UCX in wheel-based builds devcontainer: - if: false # TEMPORARY: java-only CI permissions: actions: read contents: read @@ -905,7 +902,7 @@ jobs: pull-requests: read secrets: inherit # zizmor: ignore[secrets-inherit] uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@main - if: false # TEMPORARY: java-only CI + if: (fromJSON(needs.changed-files.outputs.changed_file_groups).test_python_wheels || fromJSON(needs.changed-files.outputs.changed_file_groups).test_cudf_pandas) && fromJSON(needs.changed-files.outputs.changed_file_groups).neither_cudf_polars_nor_dask_cudf with: # This selects the latest supported Python + CUDA minor versions for each ARCH/CUDA major version combo # Filter out GB300 due to https://github.com/rapidsai/cudf/issues/23498 @@ -922,7 +919,7 @@ jobs: pull-requests: read secrets: inherit # zizmor: ignore[secrets-inherit] uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@main - if: false # TEMPORARY: java-only CI + if: fromJSON(needs.changed-files.outputs.changed_file_groups).test_python_conda && fromJSON(needs.changed-files.outputs.changed_file_groups).neither_cpp_nor_cudf_polars_nor_dask_cudf with: build_type: pull-request branch: ${{ inputs.branch }} @@ -943,7 +940,7 @@ jobs: pull-requests: read secrets: inherit # zizmor: ignore[secrets-inherit] uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@main - if: false # TEMPORARY: java-only CI + if: fromJSON(needs.changed-files.outputs.changed_file_groups).test_python_conda && fromJSON(needs.changed-files.outputs.changed_file_groups).neither_cudf_polars_nor_dask_cudf with: build_type: pull-request branch: ${{ inputs.branch }} @@ -964,7 +961,7 @@ jobs: pull-requests: read secrets: inherit # zizmor: ignore[secrets-inherit] uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@main - if: false # TEMPORARY: java-only CI + if: fromJSON(needs.changed-files.outputs.changed_file_groups).test_python_wheels && fromJSON(needs.changed-files.outputs.changed_file_groups).neither_cpp_nor_cudf_polars_nor_dask_cudf with: build_type: pull-request branch: ${{ inputs.branch }} @@ -983,7 +980,7 @@ jobs: pull-requests: read secrets: inherit # zizmor: ignore[secrets-inherit] uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@main - if: false # TEMPORARY: java-only CI + if: fromJSON(needs.changed-files.outputs.changed_file_groups).test_python_conda && fromJSON(needs.changed-files.outputs.changed_file_groups).neither_cpp_nor_cudf_polars_nor_dask_cudf with: build_type: pull-request branch: ${{ inputs.branch }} @@ -993,7 +990,6 @@ jobs: container_image: "rapidsai/ci-conda:26.10-latest" script: ci/test_narwhals.sh spark-rapids-jni: - if: false # TEMPORARY: java-only CI needs: changed-files permissions: actions: read @@ -1002,13 +998,14 @@ jobs: packages: read pull-requests: read uses: ./.github/workflows/spark-rapids-jni.yaml + if: fromJSON(needs.changed-files.outputs.changed_file_groups).test_java telemetry-summarize: - if: false # TEMPORARY: java-only CI # This job must use a self-hosted runner to record telemetry traces. runs-on: linux-amd64-cpu4 permissions: actions: write needs: pr-builder + if: ${{ vars.TELEMETRY_ENABLED == 'true' && !cancelled() }} continue-on-error: true steps: - name: Telemetry summarize From 68db0c1768ecec400712648d43a9658e2187bd7b Mon Sep 17 00:00:00 2001 From: Paul Aiyedun <53453937+paul-aiyedun@users.noreply.github.com> Date: Fri, 14 Aug 2026 03:30:12 +0000 Subject: [PATCH 08/10] Fix nvcomp loading error --- .../java/ai/rapids/cudf/NativeDepsLoader.java | 26 +++++++++++++++++-- 1 file changed, 24 insertions(+), 2 deletions(-) diff --git a/java/src/main/java/ai/rapids/cudf/NativeDepsLoader.java b/java/src/main/java/ai/rapids/cudf/NativeDepsLoader.java index c1b91844e1b0..54fd1513d113 100755 --- a/java/src/main/java/ai/rapids/cudf/NativeDepsLoader.java +++ b/java/src/main/java/ai/rapids/cudf/NativeDepsLoader.java @@ -80,7 +80,8 @@ public class NativeDepsLoader { * stages where all the dependencies in a stage are not interdependent and * therefore can be loaded in parallel. All dependencies within an earlier * stage are guaranteed to have finished loading before any dependencies in - * subsequent stages are loaded. + * subsequent stages are loaded. nvcomp is skipped if its resource is absent + * because static libcudf builds include it. */ private static final String[][] loadOrder = new String[][]{ new String[]{ @@ -103,7 +104,14 @@ public class NativeDepsLoader { public static synchronized void loadNativeDeps() { if (!loaded) { try { - loadNativeDeps(loadOrder, preserveDepsAfterLoad); + String[][] deps = loadOrder; + if (!hasNativeResource("nvcomp")) { + log.info("Skipping optional native dependency {}", System.mapLibraryName("nvcomp")); + deps = Arrays.stream(loadOrder) + .filter(stage -> Arrays.stream(stage).noneMatch("nvcomp"::equals)) + .toArray(String[][]::new); + } + loadNativeDeps(deps, preserveDepsAfterLoad); loaded = true; } catch (Throwable t) { log.error("Could not load cudf jni library...", t); @@ -111,6 +119,20 @@ public static synchronized void loadNativeDeps() { } } + private static boolean hasNativeResource(String baseName) { + String mapped = System.mapLibraryName(baseName); + if (libNativeDir != null) { + return new File(libNativeDir, mapped).isFile(); + } + String os = System.getProperty("os.name"); + String arch = System.getProperty("os.arch"); + String path = arch + "/" + os + "/" + mapped; + if (loader.getResource(path + CHUNK_MANIFEST_SUFFIX) != null) { + return true; + } + return loader.getResource(path) != null; + } + /** * Allows other libraries to reuse the same native deps loading logic. Libraries will be searched * for under ${os.arch}/${os.name}/ in the class path using the class loader for this class. From 979508b69415338a50f353ea31283018b689d277 Mon Sep 17 00:00:00 2001 From: Paul Aiyedun <53453937+paul-aiyedun@users.noreply.github.com> Date: Fri, 14 Aug 2026 17:51:59 +0000 Subject: [PATCH 09/10] Address CodeRabbit comments --- ci/build_java.sh | 7 ++++++- ci/test_packaged_java.sh | 3 +++ java/ci/setup_java_env.sh | 8 ++++++-- 3 files changed, 15 insertions(+), 3 deletions(-) diff --git a/ci/build_java.sh b/ci/build_java.sh index 2cfaabdd4365..93f84cc955c9 100755 --- a/ci/build_java.sh +++ b/ci/build_java.sh @@ -37,7 +37,12 @@ LIBCUDF_DIR="${WORK_DIR}/libcudf" CLASSIFIER_OUT="${REPO_ROOT}/output_jars/${CLASSIFIER}" cleanup_scratch() { - rm -rf "${WORK_DIR}" + # Do not delete a caller-provided JAVA_WORK_DIR. + if [[ -z ${JAVA_WORK_DIR:-} ]]; then + rm -rf "${WORK_DIR}" + else + rm -rf "${LIBCUDF_DIR}" "${BUILD_DIR}" + fi } trap cleanup_scratch EXIT diff --git a/ci/test_packaged_java.sh b/ci/test_packaged_java.sh index 6ed0c94e5f41..dd5c1394413f 100755 --- a/ci/test_packaged_java.sh +++ b/ci/test_packaged_java.sh @@ -33,6 +33,9 @@ export LIBCUDF_LARGE_STRINGS_ENABLED="${LIBCUDF_LARGE_STRINGS_ENABLED:-0}" PRODUCT_JAR="$(cd "$(dirname "${JAVA_JAR}")" && pwd)/$(basename "${JAVA_JAR}")" rapids-logger "Product JAR: ${PRODUCT_JAR}" +rapids-logger "Check GPU usage" +nvidia-smi + rm -rf "${REPO_ROOT}/java/target" pushd "${REPO_ROOT}/java" >/dev/null diff --git a/java/ci/setup_java_env.sh b/java/ci/setup_java_env.sh index ab1bac253bc1..0fe5a8cb648d 100644 --- a/java/ci/setup_java_env.sh +++ b/java/ci/setup_java_env.sh @@ -60,8 +60,12 @@ fi # Static libcudf needs Boost.Filesystem / Boost.System. ci-wheel does not ship # those static archives, so build them from source into BOOST_PREFIX when missing. -if [[ ! -f "${BOOST_PREFIX}/lib/libboost_filesystem.a" && \ - ! -f "${BOOST_PREFIX}/lib64/libboost_filesystem.a" ]]; then +has_boost_static_lib() { + local name=$1 + [[ -f "${BOOST_PREFIX}/lib/${name}" || -f "${BOOST_PREFIX}/lib64/${name}" ]] +} + +if ! has_boost_static_lib libboost_filesystem.a || ! has_boost_static_lib libboost_system.a; then BOOST_DIR="boost_${BOOST_VERSION//./_}" wget -q -O /tmp/boost.tgz \ "https://archives.boost.io/release/${BOOST_VERSION}/source/${BOOST_DIR}.tar.gz" From b9f9bc488fb546f1c5ee45a4938f6eeb996a4e54 Mon Sep 17 00:00:00 2001 From: Paul Aiyedun <53453937+paul-aiyedun@users.noreply.github.com> Date: Fri, 14 Aug 2026 18:56:41 +0000 Subject: [PATCH 10/10] Address PR feedback * Drop redundant RAPIDS_CUDA_VERSION overrides. * Update artifact name to match convention used for other cudf artifacts. --- .github/workflows/build.yaml | 7 ++++--- .github/workflows/pr.yaml | 8 ++++---- 2 files changed, 8 insertions(+), 7 deletions(-) diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index 7bd1a7732a4f..cafb34011191 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -513,6 +513,7 @@ jobs: with: build_type: ${{ inputs.build_type || 'branch' }} matrix_name: conda-cpp-build + matrix_filter: 'map(. + {CUDA_MAJOR: (.CUDA_VER | split(".") | .[0])})' java-build: needs: [build-details, telemetry-setup, java-build-matrix] permissions: @@ -535,9 +536,9 @@ jobs: arch: ${{ matrix.ARCH }} node_type: cpu16 container_image: "rapidsai/ci-wheel:26.10-cuda${{ matrix.CUDA_VER }}-${{ matrix.LINUX_VER }}-py${{ matrix.PY_VER }}" - script: "env RAPIDS_CUDA_VERSION=${{ matrix.CUDA_VER }} ci/build_java.sh" + script: "ci/build_java.sh" file_to_upload: output_jars - artifact-name: java_pkg_${{ matrix.CUDA_VER }}_${{ matrix.ARCH }} + artifact-name: cudf_java_${{ matrix.ARCH }}_cu${{ matrix.CUDA_MAJOR }} # Assemble the per-classifier JARs into one Maven-repository layout. java-gather: needs: [java-build] @@ -553,7 +554,7 @@ jobs: - name: Download per-entry JAR artifacts uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0 with: - pattern: java_pkg_* + pattern: cudf_java_* path: ${{ runner.temp }}/jars merge-multiple: true - name: Assemble Maven repository layout diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index 96949884962b..bdffafce0d7b 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -538,6 +538,7 @@ jobs: with: build_type: pull-request matrix_name: conda-cpp-build + matrix_filter: 'map(. + {CUDA_MAJOR: (.CUDA_VER | split(".") | .[0])})' java-build: needs: [build-details, java-build-matrix, changed-files] permissions: @@ -558,9 +559,9 @@ jobs: arch: ${{ matrix.ARCH }} node_type: cpu16 container_image: "rapidsai/ci-wheel:26.10-cuda${{ matrix.CUDA_VER }}-${{ matrix.LINUX_VER }}-py${{ matrix.PY_VER }}" - script: "env RAPIDS_CUDA_VERSION=${{ matrix.CUDA_VER }} ci/build_java.sh" + script: "ci/build_java.sh" file_to_upload: output_jars - artifact-name: java_pkg_${{ matrix.CUDA_VER }}_${{ matrix.ARCH }} + artifact-name: cudf_java_${{ matrix.ARCH }}_cu${{ matrix.CUDA_MAJOR }} java-tests: needs: [java-build, java-build-matrix] if: ${{ !cancelled() && needs.java-build.result == 'success' }} @@ -587,11 +588,10 @@ jobs: - name: Download java-build artifact uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0 with: - name: java_pkg_${{ matrix.CUDA_VER }}_${{ matrix.ARCH }} + name: cudf_java_${{ matrix.ARCH }}_cu${{ matrix.CUDA_MAJOR }} path: java_pkg - name: Run Java tests env: - RAPIDS_CUDA_VERSION: ${{ matrix.CUDA_VER }} LIBCUDF_LARGE_STRINGS_ENABLED: "0" run: | . java/ci/java_classifier.sh