diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index 920cceb0cb66..7bd1a7732a4f 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -505,46 +505,39 @@ jobs: push: true cuda: '["12.9", "13.3"]' # Build the cuDF Java JAR for every Maven classifier. - java-build: + java-build-matrix: needs: [telemetry-setup] - strategy: - fail-fast: false - matrix: - include: - - { cuda: "12.9", cuda_major: "12", arch: "x86_64", runner: "linux-amd64-cpu16" } - - { cuda: "13.3", cuda_major: "13", arch: "x86_64", runner: "linux-amd64-cpu16" } - - { cuda: "12.9", cuda_major: "12", arch: "aarch64", runner: "linux-arm64-cpu16" } - - { cuda: "13.3", cuda_major: "13", arch: "aarch64", runner: "linux-arm64-cpu16" } - runs-on: ${{ matrix.runner }} permissions: contents: read - steps: - - name: Checkout code repo - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1 - with: - ref: ${{ inputs.sha }} - fetch-depth: 0 - persist-credentials: false - - name: Build static libcudf - run: | - ./java/ci/build_static_libcudf.sh \ - --output-dir "${RUNNER_TEMP}/libcudf" \ - --cuda-version "${{ matrix.cuda }}" - - name: Build cuDF Java JAR - run: | - ./java/ci/build_cudf_java_jar.sh \ - --libcudf-dir "${RUNNER_TEMP}/libcudf" \ - --output-dir "${RUNNER_TEMP}/jars" \ - --cuda-version "${{ matrix.cuda }}" - - name: Upload per-entry JAR artifact - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 - with: - name: cudf_java_cuda${{ matrix.cuda_major }}_${{ matrix.arch }} - # Ship only the JARs and POMs. Exclude the per-classifier Maven build scratch dir. - path: | - ${{ runner.temp }}/jars - !${{ runner.temp }}/jars/.mvn-temp-target - if-no-files-found: error + uses: rapidsai/shared-workflows/.github/workflows/compute-matrix.yaml@main + with: + build_type: ${{ inputs.build_type || 'branch' }} + matrix_name: conda-cpp-build + java-build: + needs: [build-details, telemetry-setup, java-build-matrix] + permissions: + actions: read + contents: read + id-token: write + packages: read + pull-requests: read + secrets: inherit # zizmor: ignore[secrets-inherit] + uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@main + strategy: + fail-fast: false + matrix: ${{ fromJSON(needs.java-build-matrix.outputs.matrix) }} + with: + build_type: ${{ inputs.build_type || 'branch' }} + branch: ${{ inputs.branch }} + date: ${{ inputs.date }} + sha: ${{ inputs.sha }} + build-datetime: ${{ needs.build-details.outputs.build-datetime }} + arch: ${{ matrix.ARCH }} + node_type: cpu16 + container_image: "rapidsai/ci-wheel:26.10-cuda${{ matrix.CUDA_VER }}-${{ matrix.LINUX_VER }}-py${{ matrix.PY_VER }}" + script: "env RAPIDS_CUDA_VERSION=${{ matrix.CUDA_VER }} ci/build_java.sh" + file_to_upload: output_jars + artifact-name: java_pkg_${{ matrix.CUDA_VER }}_${{ matrix.ARCH }} # Assemble the per-classifier JARs into one Maven-repository layout. java-gather: needs: [java-build] @@ -560,7 +553,7 @@ jobs: - name: Download per-entry JAR artifacts uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0 with: - pattern: cudf_java_cuda* + pattern: java_pkg_* path: ${{ runner.temp }}/jars merge-multiple: true - name: Assemble Maven repository layout diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index 500f5a71636b..70c9257ee77f 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -25,6 +25,9 @@ jobs: - conda-python-cudf-tests - conda-python-other-tests - conda-java-tests + - java-build-matrix + - java-build + - java-tests - conda-notebook-tests - docs-build - publish-api-docs @@ -526,6 +529,75 @@ jobs: arch: "amd64" container_image: "rapidsai/ci-conda:26.10-latest" script: "ci/test_java.sh" + # Build and test the cuDF Java JAR for every Maven classifier. + java-build-matrix: + needs: [checks] + permissions: + contents: read + uses: rapidsai/shared-workflows/.github/workflows/compute-matrix.yaml@main + with: + build_type: pull-request + matrix_name: conda-cpp-build + java-build: + needs: [build-details, java-build-matrix, changed-files] + permissions: + actions: read + contents: read + id-token: write + packages: read + pull-requests: read + secrets: inherit # zizmor: ignore[secrets-inherit] + uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@main + if: fromJSON(needs.changed-files.outputs.changed_file_groups).test_java + strategy: + fail-fast: false + matrix: ${{ fromJSON(needs.java-build-matrix.outputs.matrix) }} + with: + build_type: pull-request + build-datetime: ${{ needs.build-details.outputs.build-datetime }} + arch: ${{ matrix.ARCH }} + node_type: cpu16 + container_image: "rapidsai/ci-wheel:26.10-cuda${{ matrix.CUDA_VER }}-${{ matrix.LINUX_VER }}-py${{ matrix.PY_VER }}" + script: "env RAPIDS_CUDA_VERSION=${{ matrix.CUDA_VER }} ci/build_java.sh" + file_to_upload: output_jars + artifact-name: java_pkg_${{ matrix.CUDA_VER }}_${{ matrix.ARCH }} + java-tests: + needs: [java-build, java-build-matrix] + if: ${{ !cancelled() && needs.java-build.result == 'success' }} + permissions: + actions: read + contents: read + packages: read + strategy: + fail-fast: false + matrix: ${{ fromJSON(needs.java-build-matrix.outputs.matrix) }} + runs-on: linux-${{ matrix.ARCH }}-gpu-l4-latest-1 + container: + image: rapidsai/ci-wheel:26.10-cuda${{ matrix.CUDA_VER }}-${{ matrix.LINUX_VER }}-py${{ matrix.PY_VER }} # zizmor: ignore[unpinned-images] + env: + NVIDIA_VISIBLE_DEVICES: ${{ env.NVIDIA_VISIBLE_DEVICES }} + defaults: + run: + shell: bash + steps: + - name: Checkout + uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1 + with: + persist-credentials: false + - name: Download java-build artifact + uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0 + with: + name: java_pkg_${{ matrix.CUDA_VER }}_${{ matrix.ARCH }} + path: java_pkg + - name: Run Java tests + env: + RAPIDS_CUDA_VERSION: ${{ matrix.CUDA_VER }} + LIBCUDF_LARGE_STRINGS_ENABLED: "0" + run: | + . java/ci/java_classifier.sh + JAVA_JAR="$(cudf_java_resolve_artifact_jar java_pkg)" + export JAVA_JAR + ci/test_packaged_java.sh conda-notebook-tests: needs: [conda-python-build, conda-python-build-noarch, changed-files] permissions: diff --git a/ci/build_java.sh b/ci/build_java.sh new file mode 100755 index 000000000000..2cfaabdd4365 --- /dev/null +++ b/ci/build_java.sh @@ -0,0 +1,60 @@ +#!/bin/bash +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +# CI entrypoint: static libcudf + classifier JAR for one matrix cell. +# +# Builds the static libcudf install tree and packages the matching classifier +# JAR, writing ./output_jars// for the custom-job upload step. +# +# Inputs (environment variables): +# RAPIDS_CUDA_VERSION CUDA version, e.g. 12.9.2 (required). +# PARALLEL_LEVEL Build parallelism (default: nproc). +# CMAKE_CUDA_ARCHITECTURES Optional override for -DCMAKE_CUDA_ARCHITECTURES. +# JAVA_WORK_DIR Optional scratch dir (default: /.java-work). + +set -euo pipefail + +REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +cd "${REPO_ROOT}" + +# shellcheck disable=SC1091 +. "${REPO_ROOT}/java/ci/ci_wheel_image.sh" +# shellcheck disable=SC1091 +. "${REPO_ROOT}/java/ci/java_classifier.sh" + +if [[ -z ${RAPIDS_CUDA_VERSION:-} ]]; then + echo "Error: RAPIDS_CUDA_VERSION must be set" >&2 + exit 1 +fi + +RAPIDS_CUDA_VERSION="$(cudf_java_normalize_cuda_version "${RAPIDS_CUDA_VERSION}")" +export RAPIDS_CUDA_VERSION +CLASSIFIER="$(cudf_java_maven_classifier "${RAPIDS_CUDA_VERSION}")" + +WORK_DIR="${JAVA_WORK_DIR:-${REPO_ROOT}/.java-work}" +LIBCUDF_DIR="${WORK_DIR}/libcudf" +CLASSIFIER_OUT="${REPO_ROOT}/output_jars/${CLASSIFIER}" + +cleanup_scratch() { + rm -rf "${WORK_DIR}" +} +trap cleanup_scratch EXIT + +rm -rf "${LIBCUDF_DIR}" "${CLASSIFIER_OUT}" +mkdir -p "${LIBCUDF_DIR}" "${CLASSIFIER_OUT}" + +export REPO_ROOT +export INSTALL_PREFIX="${LIBCUDF_DIR}" +export BUILD_DIR="${WORK_DIR}/libcudf-build" +export CUDF_INSTALL_DIR="${LIBCUDF_DIR}" +export OUTPUT_DIR="${CLASSIFIER_OUT}" + +rapids-logger "Building static libcudf (${CLASSIFIER})" +bash "${REPO_ROOT}/java/ci/build_static_libcudf_in_container.sh" + +rapids-logger "Packaging cuDF Java JAR (${CLASSIFIER})" +bash "${REPO_ROOT}/java/ci/build_cudf_java_jar_in_container.sh" + +cudf_java_assert_classifier_artifacts "${CLASSIFIER_OUT}" "${CLASSIFIER}" +ls -la "${CLASSIFIER_OUT}" diff --git a/ci/test_packaged_java.sh b/ci/test_packaged_java.sh new file mode 100755 index 000000000000..6ed0c94e5f41 --- /dev/null +++ b/ci/test_packaged_java.sh @@ -0,0 +1,47 @@ +#!/bin/bash +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +# Run the Java tests against an already-packaged classifier JAR. +# +# Activates -Ppackaged-jar-tests so the surefire classpath uses the JAR at +# JAVA_JAR instead of a locally compiled target/classes tree. Caller places +# the JAR (e.g. via actions/download-artifact) before invoking this script. +# +# Inputs (environment variables): +# JAVA_JAR Absolute path to the classifier JAR (required). +# LIBCUDF_LARGE_STRINGS_ENABLED Optional; defaults to 0 (same as ci/test_java.sh). + +set -euo pipefail + +REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +cd "${REPO_ROOT}" + +if [[ -z ${JAVA_JAR:-} || ! -f ${JAVA_JAR} ]]; then + echo "Error: JAVA_JAR must point to an existing classifier JAR" >&2 + exit 1 +fi + +if ! command -v mvn >/dev/null 2>&1 || ! command -v java >/dev/null 2>&1; then + # shellcheck disable=SC1091 + . "${REPO_ROOT}/java/ci/setup_java_env.sh" +fi + +# Match the existing conda Java test entrypoint in ci/test_java.sh. +export LIBCUDF_LARGE_STRINGS_ENABLED="${LIBCUDF_LARGE_STRINGS_ENABLED:-0}" + +PRODUCT_JAR="$(cd "$(dirname "${JAVA_JAR}")" && pwd)/$(basename "${JAVA_JAR}")" +rapids-logger "Product JAR: ${PRODUCT_JAR}" + +rm -rf "${REPO_ROOT}/java/target" + +pushd "${REPO_ROOT}/java" >/dev/null +set +e +timeout 30m mvn -B test -Ppackaged-jar-tests \ + "-Dcudf.jar.path=${PRODUCT_JAR}" \ + "-DCUDF_JNI_ENABLE_PROFILING=OFF" +EXITCODE=$? +set -e +popd >/dev/null +rapids-logger "Java tests exit=${EXITCODE}" +exit "${EXITCODE}" diff --git a/java/ci/README.md b/java/ci/README.md index 49ab3b6074ec..7a668695f6e3 100644 --- a/java/ci/README.md +++ b/java/ci/README.md @@ -4,14 +4,14 @@ The scripts under `java/ci/` build the cuDF Java JAR for every Maven classifier the same way locally and in CI (GitHub Actions is only a thin wrapper that adds -artifact upload/download). Each script pulls the RAPIDS `ci-conda` build image, +artifact upload/download). Each script pulls the RAPIDS `ci-wheel` build image, runs the build in a throwaway container, and writes its output to a host directory. No local `docker build` is required, and no GPU is required to build. ### Prerequisites 1. Docker is installed and the current user can run `docker`. -2. Network access to pull `rapidsai/ci-conda:-latest`. +2. Network access to pull `rapidsai/ci-wheel:-cuda-rockylinux8-py3.11`. ### Local one-command shortcut @@ -90,6 +90,17 @@ per-entry artifact. The separate `java-gather` job downloads them (with `merge-multiple: true`, so all subdirs land in a single parent dir), runs Step 3, and uploads the combined `cudf_java_maven_repo` artifact. +### Packaging-aware tests (local) + +Plain `cd java && mvn test` does not exercise the classifier JAR. Use +`java/ci/test_packaged_java_local.sh` (or CI entrypoint +`ci/test_packaged_java.sh`) to run the existing Java tests against a packaged +JAR. Needs Docker + GPU. + +```bash +./java/ci/test_packaged_java_local.sh --work-dir /tmp/java-build-test +``` + ## Legacy: manual Dockerfile.rocky build (obsolete) > The `java/ci/Dockerfile.rocky` + `java/ci/build-in-docker.sh` flow below is the diff --git a/java/ci/build-in-docker.sh b/java/ci/build-in-docker.sh index b212d4bec5ad..fedac011e838 100755 --- a/java/ci/build-in-docker.sh +++ b/java/ci/build-in-docker.sh @@ -69,7 +69,7 @@ BUILD_ARG=( "-DCUDF_USE_PER_THREAD_DEFAULT_STREAM=$ENABLE_PTDS" "-DCUDF_JNI_LIBCUDF_STATIC=ON" "-DUSE_GDS=$ENABLE_GDS" - "-Dtest=*,!CuFileTest,!CudaFatalTest,!ColumnViewNonEmptyNullsTest,!NativeDepsLoaderTest" + "-Dtest=*,!CuFileTest,!CudaFatalTest,!ColumnViewNonEmptyNullsTest,!NativeDepsLoaderTest,!PackagedJarOriginCheck" ) if [ "$SIGN_FILE" == true ]; then diff --git a/java/ci/build_cudf_java_jar.sh b/java/ci/build_cudf_java_jar.sh index 084b00c68478..f9cf9c98dd2f 100755 --- a/java/ci/build_cudf_java_jar.sh +++ b/java/ci/build_cudf_java_jar.sh @@ -5,7 +5,7 @@ # Self-contained packaging of the cuDF Java JAR for a single classifier. # # Consumes a prebuilt static libcudf install tree (from build_static_libcudf.sh), -# compiles the JNI layer against it inside a throwaway RAPIDS ci-conda container, +# compiles the JNI layer against it inside a throwaway RAPIDS ci-wheel container, # and emits the single classifier JAR (plus its POM) to a per-classifier # subdirectory under --output-dir. This script is layout-agnostic: it produces # one classifier's artifacts and knows nothing about the combined @@ -18,6 +18,10 @@ REPO_ROOT="$(git -C "${SCRIPT_DIR}" rev-parse --show-toplevel)" # shellcheck disable=SC1091 . "${SCRIPT_DIR}/argparse.sh" +# shellcheck disable=SC1091 +. "${SCRIPT_DIR}/ci_wheel_image.sh" +# shellcheck disable=SC1091 +. "${SCRIPT_DIR}/java_classifier.sh" LIBCUDF_DIR="" OUTPUT_DIR="" @@ -31,11 +35,10 @@ print_help() { Usage: build_cudf_java_jar.sh --libcudf-dir --output-dir \\ --cuda-version [OPTIONS] -Packages the cuDF Java JAR for a single classifier inside a RAPIDS ci-conda +Packages the cuDF Java JAR for a single classifier inside a RAPIDS ci-wheel container, linking against a prebuilt static libcudf. Always builds for the -host architecture (uname -m). The build image is fixed to -rapidsai/ci-conda:-latest (version derived from the VERSION -file). +host architecture (uname -m). The build image is derived from --cuda-version +and the VERSION file (see java/ci/ci_wheel_image.sh). The classifier is derived from --cuda-version (major) + host arch (uname -m), mirroring the pom.xml Groovy logic: "cuda" for x86_64, @@ -49,7 +52,7 @@ REQUIRED: -o, --output-dir Host parent directory. The script creates and writes to //, which must not already exist. - -c, --cuda-version CUDA version to build for (e.g. "12.9" or "12.9.1"). + -c, --cuda-version CUDA version to build for (e.g. "12.9" or "12.9.2"). Must match --cuda-version of the static libcudf tree; determines the cuda12/cuda13 classifier. @@ -126,26 +129,9 @@ if [[ ! -d ${LIBCUDF_DIR} ]]; then exit 1 fi -# Derive the Maven classifier from --cuda-version major + host arch, mirroring -# the pom.xml Groovy logic: "cuda" for x86_64, "cuda-arm64" for -# aarch64. -CUDA_MAJOR="$(echo "${CUDA_VERSION}" | cut -d. -f1)" -HOST_ARCH="$(uname -m)" -case "${HOST_ARCH}" in - x86_64) - CLASSIFIER="cuda${CUDA_MAJOR}" - ;; - aarch64|arm64) - CLASSIFIER="cuda${CUDA_MAJOR}-arm64" - ;; - *) - echo "Error: Unsupported host arch '${HOST_ARCH}' (expected x86_64 or aarch64)" >&2 - exit 1 - ;; -esac - -RAPIDS_VERSION="$(head -1 "${REPO_ROOT}/VERSION" | cut -d. -f1,2)" -IMAGE="rapidsai/ci-conda:${RAPIDS_VERSION}-latest" +CLASSIFIER="$(cudf_java_maven_classifier "${CUDA_VERSION}")" +IMAGE="$(cudf_java_ci_wheel_image "${CUDA_VERSION}")" +CUDA_VERSION_FULL="$(cudf_java_normalize_cuda_version "${CUDA_VERSION}")" mkdir -p "${OUTPUT_DIR}" OUTPUT_DIR="$(cd "${OUTPUT_DIR}" && pwd)" @@ -173,7 +159,7 @@ mkdir -p "${TARGET_SCRATCH}" echo "Packaging cuDF Java JAR" echo " image: ${IMAGE}" -echo " cuda version: ${CUDA_VERSION}" +echo " cuda version: ${CUDA_VERSION_FULL}" echo " classifier: ${CLASSIFIER}" echo " parallel: ${PARALLEL_LEVEL}" echo " libcudf dir: ${LIBCUDF_DIR}" @@ -190,10 +176,13 @@ DOCKER_ARGS=( --volume "${CLASSIFIER_OUT}:/output" --volume "${TARGET_SCRATCH}:/repo/java/target" --workdir /repo - --env RAPIDS_CUDA_VERSION="${CUDA_VERSION}" + --env RAPIDS_CUDA_VERSION="${CUDA_VERSION_FULL}" --env PARALLEL_LEVEL="${PARALLEL_LEVEL}" --env HOST_UID="$(id -u)" --env HOST_GID="$(id -g)" + --env CUDF_INSTALL_DIR=/libcudf + --env OUTPUT_DIR=/output + --env REPO_ROOT=/repo ) if [[ -n ${CMAKE_CUDA_ARCHITECTURES} ]]; then @@ -205,42 +194,4 @@ docker run "${DOCKER_ARGS[@]}" "${IMAGE}" \ # Post-run: assert exactly one main classifier JAR + one POM, and that the # JAR's classifier suffix matches the subdir name we chose (catches pom drift). -PRODUCED_JAR="" -for candidate in "${CLASSIFIER_OUT}"/cudf-*-"${CLASSIFIER}".jar; do - if [[ -f "${candidate}" ]]; then - if [[ -n "${PRODUCED_JAR}" ]]; then - echo "Error: multiple JARs matching cudf-*-${CLASSIFIER}.jar found in ${CLASSIFIER_OUT}" - ls -1 "${CLASSIFIER_OUT}" - exit 1 - fi - PRODUCED_JAR=${candidate} - fi -done - -if [[ -z "${PRODUCED_JAR}" ]]; then - echo "Error: no cudf-*-${CLASSIFIER}.jar found in ${CLASSIFIER_OUT}" - ls -1 "${CLASSIFIER_OUT}" || true - exit 1 -fi - -PRODUCED_POM="" -for candidate in "${CLASSIFIER_OUT}"/cudf-*.pom; do - if [[ -f "${candidate}" ]]; then - if [[ -n "${PRODUCED_POM}" ]]; then - echo "Error: multiple POMs found in ${CLASSIFIER_OUT}" - ls -1 "${CLASSIFIER_OUT}" - exit 1 - fi - PRODUCED_POM=${candidate} - fi -done - -if [[ -z "${PRODUCED_POM}" ]]; then - echo "Error: no cudf-*.pom found in ${CLASSIFIER_OUT}" - ls -1 "${CLASSIFIER_OUT}" || true - exit 1 -fi - -echo "cuDF Java JAR build succeeded:" -echo " $(basename "${PRODUCED_JAR}")" -echo " $(basename "${PRODUCED_POM}")" +cudf_java_assert_classifier_artifacts "${CLASSIFIER_OUT}" "${CLASSIFIER}" diff --git a/java/ci/build_cudf_java_jar_in_container.sh b/java/ci/build_cudf_java_jar_in_container.sh index e02ea3171508..53fc028ce875 100755 --- a/java/ci/build_cudf_java_jar_in_container.sh +++ b/java/ci/build_cudf_java_jar_in_container.sh @@ -4,116 +4,92 @@ # In-container packaging of the cuDF Java JAR for a single classifier. # -# This script runs inside the rapidsai/ci-conda container launched by -# java/ci/build_cudf_java_jar.sh. It generates the build_java conda toolchain -# environment, installs a JDK 17 side-prefix for the javadoc-jdk17 profile, -# compiles the JNI layer against a prebuilt static libcudf (mounted at -# /libcudf), and packages the cuDF Java JAR. The resulting classifier JAR, -# sources JAR, javadoc JAR, and POM are copied to /output. /output and -# /repo/java/target are chowned to HOST_UID:HOST_GID on exit so the host user -# owns the outputs. +# Expects a RAPIDS ci-wheel environment. Sources the toolchain from +# setup_java_env.sh, compiles the JNI layer against the prebuilt static +# libcudf at CUDF_INSTALL_DIR, and packages the cuDF Java JAR. The classifier +# JAR, sources JAR, javadoc JAR, and POM are copied to OUTPUT_DIR. When +# HOST_UID / HOST_GID are set, OUTPUT_DIR and java/target are chowned on exit +# so the host user owns the outputs. # # Inputs (environment variables): -# RAPIDS_CUDA_VERSION CUDA version, e.g. 12.9 or 12.9.1 (required). +# RAPIDS_CUDA_VERSION CUDA version, e.g. 12.9.2 (required). # PARALLEL_LEVEL Build parallelism (default: nproc). # CMAKE_CUDA_ARCHITECTURES Optional override for -DCMAKE_CUDA_ARCHITECTURES. -# HOST_UID / HOST_GID Chown target for /output and /repo/java/target -# (both required). +# CUDF_INSTALL_DIR Static libcudf install tree (default: /libcudf). +# OUTPUT_DIR Artifact output dir (default: /output). +# REPO_ROOT cuDF checkout (default: /repo). +# HOST_UID / HOST_GID Optional chown target for OUTPUT_DIR + java/target. set -e -OUTPUT_DIR=/output -REPO_ROOT=/repo -CUDF_INSTALL_DIR=/libcudf +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +OUTPUT_DIR="${OUTPUT_DIR:-/output}" +REPO_ROOT="${REPO_ROOT:-/repo}" +CUDF_INSTALL_DIR="${CUDF_INSTALL_DIR:-/libcudf}" -. /opt/conda/etc/profile.d/conda.sh +# shellcheck disable=SC1091 +. "${SCRIPT_DIR}/setup_java_env.sh" -if [[ -z ${RAPIDS_CUDA_VERSION} ]]; then +if [[ -z ${RAPIDS_CUDA_VERSION:-} ]]; then echo "Error: RAPIDS_CUDA_VERSION must be set" >&2 exit 1 fi -if [[ -z ${HOST_UID} || -z ${HOST_GID} ]]; then - echo "Error: HOST_UID and HOST_GID must both be set" >&2 - exit 1 -fi - _chown_outputs_on_exit() { - chown -R "${HOST_UID}:${HOST_GID}" "${OUTPUT_DIR}" "${REPO_ROOT}/java/target" 2>/dev/null || true + if [[ -n ${HOST_UID:-} && -n ${HOST_GID:-} ]]; then + chown -R "${HOST_UID}:${HOST_GID}" "${OUTPUT_DIR}" "${REPO_ROOT}/java/target" 2>/dev/null || true + fi } trap _chown_outputs_on_exit EXIT -if [[ -z ${PARALLEL_LEVEL} ]]; then - PARALLEL_LEVEL=$(nproc) -fi - -CUDA_MAJOR_MINOR=$(echo "${RAPIDS_CUDA_VERSION}" | cut -d. -f1,2) - -rapids-logger "Configuring conda strict channel priority" -conda config --set channel_priority strict - -rapids-logger "Generating build_java conda environment (cuda=${CUDA_MAJOR_MINOR}, arch=$(arch))" -ENV_YAML_DIR="$(mktemp -d)" -rapids-dependency-file-generator \ - --output conda \ - --file-key build_java \ - --matrix "cuda=${CUDA_MAJOR_MINOR};arch=$(arch)" | tee "${ENV_YAML_DIR}/env.yaml" - -rapids-mamba-retry env create --yes -f "${ENV_YAML_DIR}/env.yaml" -n build_java -conda activate build_java - -rapids-print-env - -# The `javadoc-jdk17` profile in java/pom.xml points -# at ${env.JDK17_HOME}/bin/javadoc. The build_java env -# above provides only JDK 8 (mvn's own JVM), so install JDK 17 into a -# dedicated prefix that JDK17_HOME can point to. The primary mvn JVM stays -# on JDK 8; only the javadoc binary is invoked from this prefix. -rapids-logger "Installing JDK 17 into /opt/jdk17 for javadoc-jdk17 profile" -rapids-mamba-retry create --yes --prefix /opt/jdk17 openjdk=17.* -export JDK17_HOME=/opt/jdk17 - -if [[ -z ${CUDACXX} ]]; then - export CUDACXX="${CONDA_PREFIX}/bin/nvcc" -fi -if [[ -z ${LIBCUDF_KERNEL_CACHE_PATH} ]]; then - export LIBCUDF_KERNEL_CACHE_PATH=/tmp/rapids-kernel-cache -fi - BUILD_ARG=( -B - # Prefix every log line with HH:mm:ss.SSS so the elapsed time of individual - # plugin executions is recorded. + # Prefix every log line with HH:mm:ss.SSS to record per-plugin elapsed time. "-Dorg.slf4j.simpleLogger.showDateTime=true" "-Dorg.slf4j.simpleLogger.dateTimeFormat=HH:mm:ss.SSS" - "-Dmaven.repo.local=/tmp/.m2" + "-Dmaven.repo.local=${MAVEN_REPO_LOCAL:-/tmp/.m2}" "-Dparallel.level=${PARALLEL_LEVEL}" "-DskipTests=true" "-DCUDF_USE_PER_THREAD_DEFAULT_STREAM=ON" "-DCUDF_JNI_LIBCUDF_STATIC=ON" "-DUSE_GDS=OFF" - # -Prelease produces the sources.jar file via maven-source-plugin; - # -Pjavadoc-jdk17 produces the javadoc.jar file via maven-javadoc-plugin - # running against ${env.JDK17_HOME}/bin/javadoc. Both are required by - # Maven Central for every published release. + # -Prelease adds the sources.jar; -Pjavadoc-jdk17 adds the javadoc.jar (built + # against ${JDK17_HOME}/bin/javadoc). Both are required by Maven Central. "-Prelease" "-Pjavadoc-jdk17" ) - -if [[ -n ${CMAKE_CUDA_ARCHITECTURES} ]]; then +if [[ -n ${CMAKE_CUDA_ARCHITECTURES:-} ]]; then BUILD_ARG+=("-DCMAKE_CUDA_ARCHITECTURES=${CMAKE_CUDA_ARCHITECTURES}") fi +# Pass the toolchain + sccache launchers from setup_java_env.sh to the JNI cmake +# invocation via the pom's cmake.ccache.opts property. +CMAKE_CCACHE_OPTS=() +if [[ -n ${CMAKE_C_COMPILER_LAUNCHER:-} ]]; then + CMAKE_CCACHE_OPTS+=("-DCMAKE_C_COMPILER_LAUNCHER=${CMAKE_C_COMPILER_LAUNCHER}") +fi +if [[ -n ${CMAKE_CXX_COMPILER_LAUNCHER:-} ]]; then + CMAKE_CCACHE_OPTS+=("-DCMAKE_CXX_COMPILER_LAUNCHER=${CMAKE_CXX_COMPILER_LAUNCHER}") +fi +if [[ -n ${CMAKE_CUDA_COMPILER_LAUNCHER:-} ]]; then + CMAKE_CCACHE_OPTS+=("-DCMAKE_CUDA_COMPILER_LAUNCHER=${CMAKE_CUDA_COMPILER_LAUNCHER}") +fi +CMAKE_CCACHE_OPTS+=("-DCMAKE_C_COMPILER=${CC}" "-DCMAKE_CXX_COMPILER=${CXX}") +CMAKE_CCACHE_OPTS+=("-DCMAKE_CUDA_HOST_COMPILER=${CMAKE_CUDA_HOST_COMPILER}") +if [[ -d ${BOOST_PREFIX} ]]; then + CMAKE_CCACHE_OPTS+=("-DCMAKE_PREFIX_PATH=${BOOST_PREFIX}") +fi +BUILD_ARG+=("-Dcmake.ccache.opts=${CMAKE_CCACHE_OPTS[*]}") + cd "${REPO_ROOT}/java" -CUDF_VERSION="$(mvn help:evaluate -Dexpression=project.version -q -DforceStdout "${BUILD_ARG[@]}")" -rapids-logger "Packaging cuDF Java JAR version ${CUDF_VERSION} (libcudf: ${CUDF_INSTALL_DIR})" +CUDF_VERSION="$(cudf_java_scl mvn help:evaluate -Dexpression=project.version -q -DforceStdout "${BUILD_ARG[@]}")" +rapids-logger "Packaging cuDF Java JAR ${CUDF_VERSION}" -# The `clean` goal is intentionally omitted: /repo/java/target is a -# bind-mount point, so when `mvn clean` attempts to remove the directory, -# it fails with EBUSY. The host wrapper (build_cudf_java_jar.sh) recreates -# the scratch dir before each container launch to guarantee target/ starts empty. -CUDF_INSTALL_DIR="${CUDF_INSTALL_DIR}" mvn package "${BUILD_ARG[@]}" +# Omit the `clean` goal: java/target may be a bind-mount point, so `mvn clean` +# fails with EBUSY. The host wrapper recreates the scratch dir before each +# launch to guarantee target/ starts empty. +CUDF_INSTALL_DIR="${CUDF_INSTALL_DIR}" cudf_java_scl mvn package "${BUILD_ARG[@]}" mkdir -p "${OUTPUT_DIR}" @@ -132,8 +108,7 @@ for candidate in target/cudf-"${CUDF_VERSION}"-*.jar; do *) if [[ -f ${candidate} ]]; then if [[ -n ${MAIN_JAR} ]]; then - echo "Error: multiple main classifier JARs matched under target/" >&2 - ls -l target/ >&2 || true + echo "Error: multiple main classifier JARs under target/" >&2 exit 1 fi MAIN_JAR=${candidate} @@ -143,23 +118,25 @@ for candidate in target/cudf-"${CUDF_VERSION}"-*.jar; do done if [[ -z ${MAIN_JAR} ]]; then - echo "Error: no cuDF classifier JAR produced under target/" >&2 - ls -l target/ >&2 || true + echo "Error: no classifier JAR under target/" >&2 + ls -l target/ >&2 exit 1 fi # Assert the release-profile artifacts landed. A missing file here means -# -Prelease or -Pjavadoc-jdk17 did not activate, or JDK17_HOME did not -# resolve to a usable javadoc binary. +# -Prelease or -Pjavadoc-jdk17 did not activate, or JDK17_HOME did not resolve +# to a usable javadoc binary. for required in "${OUTPUT_DIR}/cudf-${CUDF_VERSION}-sources.jar" \ "${OUTPUT_DIR}/cudf-${CUDF_VERSION}-javadoc.jar"; do if [[ ! -f ${required} ]]; then - echo "Error: expected ${required} not found (mvn -Prelease -Pjavadoc-jdk17 did not produce it)" >&2 + echo "Error: missing ${required}" >&2 exit 1 fi done cp -f "${MAIN_JAR}" "${OUTPUT_DIR}/" cp -f pom.xml "${OUTPUT_DIR}/cudf-${CUDF_VERSION}.pom" - -rapids-logger "Emitted $(basename "${MAIN_JAR}"), cudf-${CUDF_VERSION}-sources.jar, cudf-${CUDF_VERSION}-javadoc.jar, and cudf-${CUDF_VERSION}.pom to ${OUTPUT_DIR}" +rapids-logger "Emitted artifacts to ${OUTPUT_DIR}" +if command -v sccache >/dev/null 2>&1; then + sccache --show-adv-stats || true +fi diff --git a/java/ci/build_static_libcudf.sh b/java/ci/build_static_libcudf.sh index 2fa2e435471c..2ffac68b3c8d 100755 --- a/java/ci/build_static_libcudf.sh +++ b/java/ci/build_static_libcudf.sh @@ -4,7 +4,7 @@ # Self-contained build of a static libcudf install tree. # -# Pulls the RAPIDS ci-conda image, builds libcudf with BUILD_SHARED_LIBS=OFF +# Pulls the RAPIDS ci-wheel image, builds libcudf with BUILD_SHARED_LIBS=OFF # inside a throwaway container, and installs the static libcudf tree (libcudf.a # plus its static dependencies) into a directory on the host. No GPU is required # to build. @@ -16,6 +16,8 @@ REPO_ROOT="$(git -C "${SCRIPT_DIR}" rev-parse --show-toplevel)" # shellcheck disable=SC1091 . "${SCRIPT_DIR}/argparse.sh" +# shellcheck disable=SC1091 +. "${SCRIPT_DIR}/ci_wheel_image.sh" OUTPUT_DIR="" CUDA_VERSION="" @@ -27,15 +29,15 @@ print_help() { Usage: build_static_libcudf.sh --output-dir --cuda-version [OPTIONS] -Builds a static libcudf install tree inside a RAPIDS ci-conda container and +Builds a static libcudf install tree inside a RAPIDS ci-wheel container and writes it to a directory on the host. Always builds for the host architecture -(uname -m). The build image is fixed to rapidsai/ci-conda:-latest -(version derived from the VERSION file). +(uname -m). The build image is derived from --cuda-version and the VERSION file +(see java/ci/ci_wheel_image.sh). REQUIRED: -o, --output-dir Host directory to receive the static install tree (libcudf.a and its static dependencies). - -c, --cuda-version CUDA version to build for (e.g. "12.9" or "12.9.1"). + -c, --cuda-version CUDA version to build for (e.g. "12.9" or "12.9.2"). OPTIONS: -A, --cmake-cuda-architectures @@ -97,15 +99,15 @@ parse_args "$@" require_arg --output-dir "${OUTPUT_DIR}" require_arg --cuda-version "${CUDA_VERSION}" -RAPIDS_VERSION="$(head -1 "${REPO_ROOT}/VERSION" | cut -d. -f1,2)" -IMAGE="rapidsai/ci-conda:${RAPIDS_VERSION}-latest" +IMAGE="$(cudf_java_ci_wheel_image "${CUDA_VERSION}")" +CUDA_VERSION_FULL="$(cudf_java_normalize_cuda_version "${CUDA_VERSION}")" mkdir -p "${OUTPUT_DIR}" OUTPUT_DIR="$(cd "${OUTPUT_DIR}" && pwd)" echo "Building static libcudf" echo " image: ${IMAGE}" -echo " cuda version: ${CUDA_VERSION}" +echo " cuda version: ${CUDA_VERSION_FULL}" echo " parallel: ${PARALLEL_LEVEL}" echo " output dir: ${OUTPUT_DIR}" if [[ -n ${CMAKE_CUDA_ARCHITECTURES} ]]; then @@ -117,10 +119,12 @@ DOCKER_ARGS=( --volume "${REPO_ROOT}:/repo" --volume "${OUTPUT_DIR}:/output" --workdir /repo - --env RAPIDS_CUDA_VERSION="${CUDA_VERSION}" + --env RAPIDS_CUDA_VERSION="${CUDA_VERSION_FULL}" --env PARALLEL_LEVEL="${PARALLEL_LEVEL}" --env HOST_UID="$(id -u)" --env HOST_GID="$(id -g)" + --env INSTALL_PREFIX=/output + --env REPO_ROOT=/repo ) if [[ -n ${CMAKE_CUDA_ARCHITECTURES} ]]; then diff --git a/java/ci/build_static_libcudf_in_container.sh b/java/ci/build_static_libcudf_in_container.sh index e77637e747b2..b911b131aad3 100755 --- a/java/ci/build_static_libcudf_in_container.sh +++ b/java/ci/build_static_libcudf_in_container.sh @@ -4,69 +4,44 @@ # In-container build of a static libcudf install tree. # -# This script runs inside the rapidsai/ci-conda container launched by -# java/ci/build_static_libcudf.sh. It generates the build_java conda toolchain -# environment, builds libcudf with BUILD_SHARED_LIBS=OFF, and installs the -# resulting static libcudf (plus its static dependencies) into /output. Then -# chowns /output to HOST_UID:HOST_GID so the host user owns the outputs. +# Expects a RAPIDS ci-wheel environment. Sources the toolchain from +# setup_java_env.sh, builds libcudf with BUILD_SHARED_LIBS=OFF, and installs +# the resulting static libcudf (plus its static dependencies) into +# INSTALL_PREFIX. When HOST_UID / HOST_GID are set, chowns the install tree so +# the host user owns the outputs. # # Inputs (environment variables): -# RAPIDS_CUDA_VERSION CUDA version, e.g. 12.9 or 12.9.1 (required). +# RAPIDS_CUDA_VERSION CUDA version, e.g. 12.9.2 (required). # PARALLEL_LEVEL Build parallelism (default: nproc). # CMAKE_CUDA_ARCHITECTURES Optional override for -DCMAKE_CUDA_ARCHITECTURES. -# HOST_UID / HOST_GID Chown target for /output (both required). +# INSTALL_PREFIX Static libcudf install dir (default: /output). +# REPO_ROOT cuDF checkout (default: /repo). +# BUILD_DIR CMake build dir (default: /tmp/libcudf-build). +# HOST_UID / HOST_GID Optional chown target for INSTALL_PREFIX. set -e -INSTALL_PREFIX=/output -REPO_ROOT=/repo -BUILD_DIR=/tmp/libcudf-build +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +INSTALL_PREFIX="${INSTALL_PREFIX:-/output}" +REPO_ROOT="${REPO_ROOT:-/repo}" +BUILD_DIR="${BUILD_DIR:-/tmp/libcudf-build}" -. /opt/conda/etc/profile.d/conda.sh +# shellcheck disable=SC1091 +. "${SCRIPT_DIR}/setup_java_env.sh" -if [[ -z ${RAPIDS_CUDA_VERSION} ]]; then +if [[ -z ${RAPIDS_CUDA_VERSION:-} ]]; then echo "Error: RAPIDS_CUDA_VERSION must be set" >&2 exit 1 fi -if [[ -z ${HOST_UID} || -z ${HOST_GID} ]]; then - echo "Error: HOST_UID and HOST_GID must both be set" >&2 - exit 1 -fi - -if [[ -z ${PARALLEL_LEVEL} ]]; then - PARALLEL_LEVEL=$(nproc) -fi - -CUDA_MAJOR_MINOR=$(echo "${RAPIDS_CUDA_VERSION}" | cut -d. -f1,2) - -rapids-logger "Configuring conda strict channel priority" -conda config --set channel_priority strict - -rapids-logger "Generating build_java conda environment (cuda=${CUDA_MAJOR_MINOR}, arch=$(arch))" -ENV_YAML_DIR="$(mktemp -d)" -rapids-dependency-file-generator \ - --output conda \ - --file-key build_java \ - --matrix "cuda=${CUDA_MAJOR_MINOR};arch=$(arch)" | tee "${ENV_YAML_DIR}/env.yaml" - -rapids-mamba-retry env create --yes -f "${ENV_YAML_DIR}/env.yaml" -n build_java -conda activate build_java - -rapids-print-env - -if [[ -z ${CUDACXX} ]]; then - export CUDACXX="${CONDA_PREFIX}/bin/nvcc" -fi -if [[ -z ${LIBCUDF_KERNEL_CACHE_PATH} ]]; then - export LIBCUDF_KERNEL_CACHE_PATH=/tmp/rapids-kernel-cache -fi - CMAKE_ARGS=( -S "${REPO_ROOT}/cpp" -B "${BUILD_DIR}" -GNinja -DCMAKE_INSTALL_PREFIX="${INSTALL_PREFIX}" + -DCMAKE_C_COMPILER="${CC}" + -DCMAKE_CXX_COMPILER="${CXX}" + -DCMAKE_CUDA_HOST_COMPILER="${CMAKE_CUDA_HOST_COMPILER}" -DBUILD_SHARED_LIBS=OFF -DBUILD_TESTS=OFF -DUSE_NVTX=ON @@ -77,19 +52,33 @@ CMAKE_ARGS=( -DRMM_LOGGING_LEVEL=OFF -DCUDF_KVIKIO_REMOTE_IO=OFF ) - -if [[ -n ${CMAKE_CUDA_ARCHITECTURES} ]]; then +if [[ -n ${CMAKE_CUDA_ARCHITECTURES:-} ]]; then CMAKE_ARGS+=("-DCMAKE_CUDA_ARCHITECTURES=${CMAKE_CUDA_ARCHITECTURES}") fi +# Forward the sccache launchers exported by setup_java_env.sh when present. +if [[ -n ${CMAKE_C_COMPILER_LAUNCHER:-} ]]; then + CMAKE_ARGS+=("-DCMAKE_C_COMPILER_LAUNCHER=${CMAKE_C_COMPILER_LAUNCHER}") +fi +if [[ -n ${CMAKE_CXX_COMPILER_LAUNCHER:-} ]]; then + CMAKE_ARGS+=("-DCMAKE_CXX_COMPILER_LAUNCHER=${CMAKE_CXX_COMPILER_LAUNCHER}") +fi +if [[ -n ${CMAKE_CUDA_COMPILER_LAUNCHER:-} ]]; then + CMAKE_ARGS+=("-DCMAKE_CUDA_COMPILER_LAUNCHER=${CMAKE_CUDA_COMPILER_LAUNCHER}") +fi +# setup_java_env.sh builds the static Boost archives into BOOST_PREFIX. +if [[ -d ${BOOST_PREFIX} ]]; then + CMAKE_ARGS+=("-DCMAKE_PREFIX_PATH=${BOOST_PREFIX}") +fi -rapids-logger "Configuring static libcudf" -cmake "${CMAKE_ARGS[@]}" - -rapids-logger "Building static libcudf with ${PARALLEL_LEVEL} jobs" +rapids-logger "Configuring/building static libcudf (cuda=${RAPIDS_CUDA_VERSION})" +cudf_java_scl cmake "${CMAKE_ARGS[@]}" cmake --build "${BUILD_DIR}" --parallel "${PARALLEL_LEVEL}" - -rapids-logger "Installing static libcudf to ${INSTALL_PREFIX}" cmake --install "${BUILD_DIR}" -rapids-logger "Chowning ${INSTALL_PREFIX} to ${HOST_UID}:${HOST_GID}" -chown -R "${HOST_UID}:${HOST_GID}" "${INSTALL_PREFIX}" +# Hand the install tree back to the host user (host wrapper passes HOST_UID/GID). +if [[ -n ${HOST_UID:-} && -n ${HOST_GID:-} ]]; then + chown -R "${HOST_UID}:${HOST_GID}" "${INSTALL_PREFIX}" +fi +if command -v sccache >/dev/null 2>&1; then + sccache --show-adv-stats || true +fi diff --git a/java/ci/ci_wheel_image.sh b/java/ci/ci_wheel_image.sh new file mode 100755 index 000000000000..8879406106ef --- /dev/null +++ b/java/ci/ci_wheel_image.sh @@ -0,0 +1,36 @@ +#!/bin/bash +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +# Image-tag helpers for Java packaging containers. Source, do not execute. +# +# cudf_java_normalize_cuda_version maps short forms (12.9, 13.3) to the full +# tags used by rapidsai/ci-wheel; cudf_java_ci_wheel_image builds the image +# name from VERSION + CUDA + RAPIDS_PY_VERSION. + +_java_ci_image_script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +_java_ci_image_repo_root="${REPO_ROOT:-$(git -C "${_java_ci_image_script_dir}" rev-parse --show-toplevel)}" + +cudf_java_normalize_cuda_version() { + local ver=$1 + case "${ver}" in + 12.9) + echo "12.9.2" + ;; + 13.3) + echo "13.3.0" + ;; + *) + echo "${ver}" + ;; + esac +} + +cudf_java_ci_wheel_image() { + local cuda_ver=${1:?cuda version required} + local rapids_ver cuda_full py_ver + rapids_ver="$(head -1 "${_java_ci_image_repo_root}/VERSION" | cut -d. -f1,2)" + cuda_full="$(cudf_java_normalize_cuda_version "${cuda_ver}")" + py_ver="${RAPIDS_PY_VERSION:-3.11}" + echo "rapidsai/ci-wheel:${rapids_ver}-cuda${cuda_full}-rockylinux8-py${py_ver}" +} diff --git a/java/ci/java_classifier.sh b/java/ci/java_classifier.sh new file mode 100644 index 000000000000..bd967f105fdf --- /dev/null +++ b/java/ci/java_classifier.sh @@ -0,0 +1,116 @@ +#!/bin/bash +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +# Maven classifier helpers for cuDF Java JARs. Source, do not execute. +# +# Derives the cuda[-arm64] classifier, locates the unique classifier JAR +# under a build output dir, and asserts the expected JAR + POM pair. + +# Print Maven classifier for a CUDA version + host arch (e.g. cuda12, cuda12-arm64). +cudf_java_maven_classifier() { + local cuda_ver=${1:?cuda version required} + local major host_arch + major="$(echo "${cuda_ver}" | cut -d. -f1)" + host_arch="$(uname -m)" + case "${host_arch}" in + x86_64) + echo "cuda${major}" + ;; + aarch64|arm64) + echo "cuda${major}-arm64" + ;; + *) + echo "Error: unsupported host arch '${host_arch}'" >&2 + return 1 + ;; + esac +} + +# Echo the unique cudf-*-${classifier}.jar under dir; error if 0 or >1. +cudf_java_find_classifier_jar() { + local dir=${1:?classifier out dir required} + local classifier=${2:?classifier required} + local jar="" candidate + for candidate in "${dir}"/cudf-*-"${classifier}".jar; do + if [[ ! -f ${candidate} ]]; then + continue + fi + if [[ -n ${jar} ]]; then + echo "Error: multiple JARs matching cudf-*-${classifier}.jar in ${dir}" >&2 + ls -1 "${dir}" >&2 + return 1 + fi + jar=${candidate} + done + if [[ -z ${jar} ]]; then + echo "Error: no cudf-*-${classifier}.jar in ${dir}" >&2 + ls -1 "${dir}" >&2 + return 1 + fi + echo "${jar}" +} + +# Assert unique classifier JAR + unique cudf-*.pom; print basenames. +cudf_java_assert_classifier_artifacts() { + local dir=${1:?classifier out dir required} + local classifier=${2:?classifier required} + local jar pom="" candidate + if ! jar="$(cudf_java_find_classifier_jar "${dir}" "${classifier}")"; then + return 1 + fi + for candidate in "${dir}"/cudf-*.pom; do + if [[ ! -f ${candidate} ]]; then + continue + fi + if [[ -n ${pom} ]]; then + echo "Error: multiple POMs in ${dir}" >&2 + ls -1 "${dir}" >&2 + return 1 + fi + pom=${candidate} + done + if [[ -z ${pom} ]]; then + echo "Error: no cudf-*.pom in ${dir}" >&2 + ls -1 "${dir}" >&2 + return 1 + fi + echo "cuDF Java JAR build succeeded:" + echo " $(basename "${jar}")" + echo " $(basename "${pom}")" +} + +# Echo the absolute path of the unique classifier JAR under a java-build +# artifact tree. Classifier from $2 or RAPIDS_CUDA_VERSION; errors if 0 or >1. +cudf_java_resolve_artifact_jar() { + local root=${1:?artifact root required} + local cuda_ver=${2:-${RAPIDS_CUDA_VERSION:-}} + local classifier + local -a matches + + if [[ -z ${cuda_ver} ]]; then + echo "Error: set RAPIDS_CUDA_VERSION or pass a cuda version" >&2 + return 1 + fi + if [[ ! -d ${root} ]]; then + echo "Error: artifact root '${root}' is not a directory" >&2 + return 1 + fi + if ! classifier="$(cudf_java_maven_classifier "${cuda_ver}")"; then + return 1 + fi + + # -${classifier}.jar excludes the sources/javadoc/tests JARs. + mapfile -t matches < <(find "${root}" -type f -name "cudf-*-${classifier}.jar" | sort) + if [[ ${#matches[@]} -eq 0 ]]; then + echo "Error: no cudf-*-${classifier}.jar under ${root}" >&2 + find "${root}" -type f >&2 + return 1 + fi + if [[ ${#matches[@]} -gt 1 ]]; then + echo "Error: multiple cudf-*-${classifier}.jar under ${root}" >&2 + printf '%s\n' "${matches[@]}" >&2 + return 1 + fi + realpath "${matches[0]}" +} diff --git a/java/ci/setup_java_env.sh b/java/ci/setup_java_env.sh new file mode 100644 index 000000000000..ab1bac253bc1 --- /dev/null +++ b/java/ci/setup_java_env.sh @@ -0,0 +1,128 @@ +#!/bin/bash +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +# Toolchain for Java builds inside a RAPIDS ci-wheel container. +# +# Installs/configures JDK 8 + 17, ninja, cmake, Boost.Filesystem/System static +# archives, the RAPIDS GCC toolset, and sccache. Safe to source repeatedly +# (JAVA_ENV_READY short-circuits after the first load). + +TOOLSET_VERSION="${TOOLSET_VERSION:-14}" +NINJA_VERSION="${NINJA_VERSION:-v1.13.1}" +BOOST_VERSION="${BOOST_VERSION:-1.79.0}" +BOOST_PREFIX="${BOOST_PREFIX:-/usr/local}" + +if [[ -n "${JAVA_ENV_READY:-}" ]]; then + return 0 +fi + +if ! type rapids-logger >/dev/null 2>&1; then + rapids-logger() { + echo ">>>> $*" >&2 + } +fi + +# JDK 8 for javac; JDK 17 for the -Pjavadoc-jdk17 Maven profile. +dnf install -y maven java-1.8.0-openjdk-devel java-17-openjdk-devel + +# ci-wheel images do not ship ninja; CMake uses it as the preferred generator. +if ! command -V ninja >/dev/null 2>&1; then + case "$(uname -m)" in + x86_64) + wget --no-hsts -q -O /tmp/ninja-linux.zip \ + "https://github.com/ninja-build/ninja/releases/download/${NINJA_VERSION}/ninja-linux.zip" + ;; + aarch64) + wget --no-hsts -q -O /tmp/ninja-linux.zip \ + "https://github.com/ninja-build/ninja/releases/download/${NINJA_VERSION}/ninja-linux-aarch64.zip" + ;; + *) + echo "Unrecognized platform '$(uname -m)'" >&2 + exit 1 + ;; + esac + unzip -d /usr/bin /tmp/ninja-linux.zip + chmod +x /usr/bin/ninja + rm -f /tmp/ninja-linux.zip +fi + +if command -v rapids-pip-retry >/dev/null 2>&1; then + rapids-pip-retry install cmake +else + pip install cmake +fi + +# Refresh pyenv shims so the cmake we just installed is on PATH. +if command -v pyenv >/dev/null 2>&1; then + pyenv rehash || true +fi + +# Static libcudf needs Boost.Filesystem / Boost.System. ci-wheel does not ship +# those static archives, so build them from source into BOOST_PREFIX when missing. +if [[ ! -f "${BOOST_PREFIX}/lib/libboost_filesystem.a" && \ + ! -f "${BOOST_PREFIX}/lib64/libboost_filesystem.a" ]]; then + BOOST_DIR="boost_${BOOST_VERSION//./_}" + wget -q -O /tmp/boost.tgz \ + "https://archives.boost.io/release/${BOOST_VERSION}/source/${BOOST_DIR}.tar.gz" + tar -xzf /tmp/boost.tgz -C /tmp + ( + if ! cd "/tmp/${BOOST_DIR}"; then + exit 1 + fi + ./bootstrap.sh --prefix="${BOOST_PREFIX}" + ./b2 install --prefix="${BOOST_PREFIX}" --with-filesystem --with-system -j"$(nproc)" + ) + rm -rf "/tmp/${BOOST_DIR}" /tmp/boost.tgz +fi + +if [[ -z "${CUDACXX:-}" ]]; then + if [[ -x /usr/local/cuda/bin/nvcc ]]; then + export CUDACXX=/usr/local/cuda/bin/nvcc + elif command -v nvcc >/dev/null 2>&1; then + CUDACXX="$(command -v nvcc)" + export CUDACXX + fi +fi + +export JAVA_HOME="${JAVA_HOME:-/usr/lib/jvm/java-1.8.0-openjdk}" +if [[ -z "${JDK17_HOME:-}" ]]; then + if [[ -d /usr/lib/jvm/java-17-openjdk ]]; then + export JDK17_HOME=/usr/lib/jvm/java-17-openjdk + else + JDK17_HOME="$(ls -d /usr/lib/jvm/java-17-openjdk* 2>/dev/null | head -1)" + export JDK17_HOME + fi +fi + +export LIBCUDF_KERNEL_CACHE_PATH="${LIBCUDF_KERNEL_CACHE_PATH:-/tmp/rapids-kernel-cache}" +mkdir -p "${LIBCUDF_KERNEL_CACHE_PATH}" +export CMAKE_GENERATOR="${CMAKE_GENERATOR:-Ninja}" +export PARALLEL_LEVEL="${PARALLEL_LEVEL:-$(nproc)}" + +# Prefer gcc-toolset over Rocky's default system compiler. +GCC_TOOLSET_ROOT="/opt/rh/gcc-toolset-${TOOLSET_VERSION}/root" +export CC="${CC:-${GCC_TOOLSET_ROOT}/usr/bin/gcc}" +export CXX="${CXX:-${GCC_TOOLSET_ROOT}/usr/bin/g++}" +export CMAKE_CUDA_HOST_COMPILER="${CMAKE_CUDA_HOST_COMPILER:-${CC}}" + +if command -v rapids-configure-sccache >/dev/null 2>&1; then + # shellcheck disable=SC1091 + source rapids-configure-sccache || true +fi + +if command -v sccache >/dev/null 2>&1; then + export CMAKE_C_COMPILER_LAUNCHER="${CMAKE_C_COMPILER_LAUNCHER:-sccache}" + export CMAKE_CXX_COMPILER_LAUNCHER="${CMAKE_CXX_COMPILER_LAUNCHER:-sccache}" + export CMAKE_CUDA_COMPILER_LAUNCHER="${CMAKE_CUDA_COMPILER_LAUNCHER:-sccache}" + # Restart so the next compile picks up the launcher env we just set. + sccache --stop-server 2>/dev/null || true +fi + +# Rocky SCL wrapper: child build steps run under gcc-toolset-${TOOLSET_VERSION}. +cudf_java_scl() { + scl enable "gcc-toolset-${TOOLSET_VERSION}" -- "$@" +} +export -f cudf_java_scl +export TOOLSET_VERSION BOOST_PREFIX +export JAVA_ENV_READY=1 diff --git a/java/ci/test_packaged_java_local.sh b/java/ci/test_packaged_java_local.sh new file mode 100755 index 000000000000..ee6a283d7d14 --- /dev/null +++ b/java/ci/test_packaged_java_local.sh @@ -0,0 +1,63 @@ +#!/bin/bash +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +# Local GPU wrapper around ci/test_packaged_java.sh. +# +# Resolves the classifier JAR from a test_java_build_local.sh --work-dir and +# runs the packaged-JAR tests in a GPU-enabled ci-wheel container. Set +# RAPIDS_CUDA_VERSION to select which classifier to test (default: 12.9). + +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +REPO_ROOT="$(git -C "${SCRIPT_DIR}" rev-parse --show-toplevel)" + +# shellcheck disable=SC1091 +. "${SCRIPT_DIR}/ci_wheel_image.sh" +# shellcheck disable=SC1091 +. "${SCRIPT_DIR}/java_classifier.sh" + +print_help() { + cat << EOF +Usage: test_packaged_java_local.sh --work-dir + +Uses the output from test_java_build_local.sh. Set RAPIDS_CUDA_VERSION to +select the classifier to test (default: 12.9). +EOF +} + +if [[ ${1:-} == "-h" || ${1:-} == "--help" ]]; then + print_help + exit 0 +fi +if [[ $# -ne 2 ]]; then + print_help + exit 1 +fi +if [[ $1 != "-w" && $1 != "--work-dir" ]]; then + print_help + exit 1 +fi + +WORK_DIR="$(cd "$2" && pwd)" +CUDA_VERSION="${RAPIDS_CUDA_VERSION:-12.9}" +CLASSIFIER="$(cudf_java_maven_classifier "${CUDA_VERSION}")" +JAR_PATH="$(realpath "$(cudf_java_find_classifier_jar "${WORK_DIR}/jars/${CLASSIFIER}" "${CLASSIFIER}")")" +IMAGE="$(cudf_java_ci_wheel_image "${CUDA_VERSION}")" +CUDA_VERSION_FULL="$(cudf_java_normalize_cuda_version "${CUDA_VERSION}")" + +echo "Running cuDF Java tests" +echo " image: ${IMAGE}" +echo " cuda version: ${CUDA_VERSION_FULL}" +echo " classifier: ${CLASSIFIER}" +echo " jar: ${JAR_PATH}" + +docker run --rm --gpus all \ + --volume "${REPO_ROOT}:/repo" \ + --volume "${JAR_PATH}:/product/cudf.jar:ro" \ + --workdir /repo \ + --env RAPIDS_CUDA_VERSION="${CUDA_VERSION_FULL}" \ + --env JAVA_JAR=/product/cudf.jar \ + --env LIBCUDF_LARGE_STRINGS_ENABLED=0 \ + "${IMAGE}" bash /repo/ci/test_packaged_java.sh diff --git a/java/pom.xml b/java/pom.xml index 8521de5567e4..6a9668c5067a 100644 --- a/java/pom.xml +++ b/java/pom.xml @@ -225,6 +225,92 @@ -Wno-deprecated-declarations + + + packaged-jar-tests + + true + + + + ai.rapids + cudf-packaged + ${project.version} + system + ${cudf.jar.path} + + + + + + maven-antrun-plugin + + + cmake + none + + + + + org.codehaus.gmavenplus + gmavenplus-plugin + + + setproperty + none + + + + + maven-compiler-plugin + + true + + + + + maven-resources-plugin + + + copy-native-libs + none + + + + + maven-surefire-plugin + + + packaged-jar-origin-check + + test + + + + + **/PackagedJarOriginCheck.java + + true + + ${cudf.jar.path} + + + + + + + + default-tests @@ -236,6 +322,7 @@ **/CudaFatalTest.java **/ColumnViewNonEmptyNullsTest.java **/NativeDepsLoaderTest.java + **/PackagedJarOriginCheck.java @@ -302,6 +389,7 @@ **/CuFileTest.java **/CudaFatalTest.java **/NativeDepsLoaderTest.java + **/PackagedJarOriginCheck.java @@ -705,9 +793,12 @@ + JVM via the dedicated execution defined in the test profiles. + PackagedJarOriginCheck needs a packaged classifier JAR and is + only run by the packaged-jar-tests profile. --> **/NativeDepsLoaderTest.java + **/PackagedJarOriginCheck.java diff --git a/java/src/main/java/ai/rapids/cudf/NativeDepsLoader.java b/java/src/main/java/ai/rapids/cudf/NativeDepsLoader.java index c1b91844e1b0..54fd1513d113 100755 --- a/java/src/main/java/ai/rapids/cudf/NativeDepsLoader.java +++ b/java/src/main/java/ai/rapids/cudf/NativeDepsLoader.java @@ -80,7 +80,8 @@ public class NativeDepsLoader { * stages where all the dependencies in a stage are not interdependent and * therefore can be loaded in parallel. All dependencies within an earlier * stage are guaranteed to have finished loading before any dependencies in - * subsequent stages are loaded. + * subsequent stages are loaded. nvcomp is skipped if its resource is absent + * because static libcudf builds include it. */ private static final String[][] loadOrder = new String[][]{ new String[]{ @@ -103,7 +104,14 @@ public class NativeDepsLoader { public static synchronized void loadNativeDeps() { if (!loaded) { try { - loadNativeDeps(loadOrder, preserveDepsAfterLoad); + String[][] deps = loadOrder; + if (!hasNativeResource("nvcomp")) { + log.info("Skipping optional native dependency {}", System.mapLibraryName("nvcomp")); + deps = Arrays.stream(loadOrder) + .filter(stage -> Arrays.stream(stage).noneMatch("nvcomp"::equals)) + .toArray(String[][]::new); + } + loadNativeDeps(deps, preserveDepsAfterLoad); loaded = true; } catch (Throwable t) { log.error("Could not load cudf jni library...", t); @@ -111,6 +119,20 @@ public static synchronized void loadNativeDeps() { } } + private static boolean hasNativeResource(String baseName) { + String mapped = System.mapLibraryName(baseName); + if (libNativeDir != null) { + return new File(libNativeDir, mapped).isFile(); + } + String os = System.getProperty("os.name"); + String arch = System.getProperty("os.arch"); + String path = arch + "/" + os + "/" + mapped; + if (loader.getResource(path + CHUNK_MANIFEST_SUFFIX) != null) { + return true; + } + return loader.getResource(path) != null; + } + /** * Allows other libraries to reuse the same native deps loading logic. Libraries will be searched * for under ${os.arch}/${os.name}/ in the class path using the class loader for this class. diff --git a/java/src/test/java/ai/rapids/cudf/PackagedJarOriginCheck.java b/java/src/test/java/ai/rapids/cudf/PackagedJarOriginCheck.java new file mode 100644 index 000000000000..28a4793552ef --- /dev/null +++ b/java/src/test/java/ai/rapids/cudf/PackagedJarOriginCheck.java @@ -0,0 +1,30 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +package ai.rapids.cudf; + +import org.junit.jupiter.api.Test; + +import java.nio.file.Path; +import java.nio.file.Paths; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +/** + * Guard for {@code -Ppackaged-jar-tests}: fail fast if cuDF classes were loaded + * from {@code target/classes} (or any other path) instead of the packaged + * classifier JAR supplied via {@code -Dcudf.jar.path} / the + * {@code cudf.packaged.jar} system property. + */ +class PackagedJarOriginCheck { + @Test + void cudfClassesAreLoadedFromPackagedJar() throws Exception { + Path expected = Paths.get(System.getProperty("cudf.packaged.jar")).toRealPath(); + Path actual = Paths.get( + Cuda.class.getProtectionDomain().getCodeSource().getLocation().toURI()) + .toRealPath(); + assertEquals(expected, actual); + } +}