diff --git a/.github/workflows/release_java.yml b/.github/workflows/release_java.yml index 7f17493302e5..bbc4dd163d2d 100644 --- a/.github/workflows/release_java.yml +++ b/.github/workflows/release_java.yml @@ -42,8 +42,12 @@ jobs: include: - os: ubuntu-latest classifier: linux-x86_64 + - os: ubuntu-latest + classifier: linux-x86_64-musl - os: ubuntu-24.04-arm classifier: linux-aarch_64 + - os: ubuntu-24.04-arm + classifier: linux-aarch_64-musl - os: windows-latest classifier: windows-x86_64 - os: macos-latest @@ -76,16 +80,55 @@ jobs: if: ${{ contains(matrix.os, 'ubuntu') }} run: pip install cargo-zigbuild + - name: Build linux musl JNI library in Alpine + if: ${{ contains(matrix.os, 'ubuntu') && contains(matrix.classifier, 'musl') }} + shell: bash + run: | + set -euo pipefail + HOST_UID="$(id -u)" + HOST_GID="$(id -g)" + PLATFORM="linux/amd64" + if [[ "${{ matrix.classifier }}" == *"aarch_64"* ]]; then + PLATFORM="linux/arm64" + fi + + docker run --rm \ + --platform "$PLATFORM" \ + -e HOST_UID="$HOST_UID" \ + -e HOST_GID="$HOST_GID" \ + -v "$GITHUB_WORKSPACE":/work \ + -w /work/bindings/java \ + rust:1.85-alpine3.20 \ + sh -lc ' + set -eu + apk add --no-cache python3 build-base cmake pkgconfig git protobuf sqlite-dev zlib-dev + export PATH=/usr/local/cargo/bin:$PATH + + python3 tools/build.py \ + --classifier "${{ matrix.classifier }}" \ + --profile release \ + --features services-all \ + --enable-zigbuild false + + chown -R "$HOST_UID:$HOST_GID" /work/bindings/java/target + ' + - name: Local staging working-directory: bindings/java shell: bash run: | + EXTRA_MVN_ARGS="" + if [[ "${{ matrix.classifier }}" == *"musl" ]]; then + EXTRA_MVN_ARGS="-Dexec.skip=true" + fi + ./mvnw -Papache-release package verify org.sonatype.plugins:nexus-staging-maven-plugin:deploy \ -DskipTests=true \ -Djni.classifier=${{ matrix.classifier }} \ -Dcargo-build.profile=release \ -Dcargo-build.features=services-all \ -Dcargo-build.enableZigbuild=${{ env.CARGO_BUILD_ENABLE_ZIGBUILD }} \ + $EXTRA_MVN_ARGS \ -DaltStagingDirectory=local-staging \ -DskipRemoteStaging=true \ -DserverId=apache.releases.https \ @@ -135,11 +178,21 @@ jobs: with: name: linux-x86_64-local-staging path: ~/linux-x86_64-local-staging + - name: Download linux x86_64 (musl) staging directory + uses: actions/download-artifact@v5 + with: + name: linux-x86_64-musl-local-staging + path: ~/linux-x86_64-musl-local-staging - name: Download linux aarch_64 staging directory uses: actions/download-artifact@v7 with: name: linux-aarch_64-local-staging path: ~/linux-aarch_64-local-staging + - name: Download linux aarch_64 (musl) staging directory + uses: actions/download-artifact@v5 + with: + name: linux-aarch_64-musl-local-staging + path: ~/linux-aarch_64-musl-local-staging - name: Download darwin staging directory uses: actions/download-artifact@v7 with: @@ -160,7 +213,9 @@ jobs: python ./scripts/merge_local_staging.py $LOCAL_STAGING_DIR/staging \ ~/windows-x86_64-local-staging/staging \ ~/linux-x86_64-local-staging/staging \ + ~/linux-x86_64-musl-local-staging/staging \ ~/linux-aarch_64-local-staging/staging \ + ~/linux-aarch_64-musl-local-staging/staging \ ~/osx-x86_64-local-staging/staging \ ~/osx-aarch_64-local-staging/staging diff --git a/bindings/java/Cargo.toml b/bindings/java/Cargo.toml index a671f6c6136f..ff723ace4a5d 100644 --- a/bindings/java/Cargo.toml +++ b/bindings/java/Cargo.toml @@ -27,7 +27,7 @@ repository = "https://github.com/apache/opendal" rust-version = "1.85" [lib] -crate-type = ["cdylib"] +crate-type = ["cdylib", "staticlib"] doc = false [features] diff --git a/bindings/java/src/main/java/org/apache/opendal/NativeLibrary.java b/bindings/java/src/main/java/org/apache/opendal/NativeLibrary.java index bc67e3237610..f062ad1aa758 100644 --- a/bindings/java/src/main/java/org/apache/opendal/NativeLibrary.java +++ b/bindings/java/src/main/java/org/apache/opendal/NativeLibrary.java @@ -77,6 +77,9 @@ public static void loadLibrary() { } catch (IOException e) { libraryLoaded.set(LibraryState.NOT_LOADED); throw new UncheckedIOException("Unable to load the OpenDAL shared library", e); + } catch (UnsatisfiedLinkError e) { + libraryLoaded.set(LibraryState.NOT_LOADED); + throw e; } libraryLoaded.set(LibraryState.LOADED); return; @@ -104,12 +107,39 @@ private static void doLoadLibrary() throws IOException { private static void doLoadBundledLibrary() throws IOException { final String libraryPath = bundledLibraryPath(); + UnsatisfiedLinkError linkError = null; try (final InputStream is = NativeObject.class.getResourceAsStream(libraryPath)) { + if (is != null) { + final int dot = libraryPath.indexOf('.'); + final File tmpFile = File.createTempFile(libraryPath.substring(0, dot), libraryPath.substring(dot)); + tmpFile.deleteOnExit(); + Files.copy(is, tmpFile.toPath(), StandardCopyOption.REPLACE_EXISTING); + try { + System.load(tmpFile.getAbsolutePath()); + return; + } catch (UnsatisfiedLinkError e) { + linkError = e; + } + } + } + + final String fallbackLibraryPath = fallbackBundledLibraryPath(); + if (fallbackLibraryPath == null) { + if (linkError != null) { + throw linkError; + } + throw new IOException("cannot find " + libraryPath); + } + try (final InputStream is = NativeObject.class.getResourceAsStream(fallbackLibraryPath)) { if (is == null) { + if (linkError != null) { + throw linkError; + } throw new IOException("cannot find " + libraryPath); } - final int dot = libraryPath.indexOf('.'); - final File tmpFile = File.createTempFile(libraryPath.substring(0, dot), libraryPath.substring(dot)); + final int dot = fallbackLibraryPath.indexOf('.'); + final File tmpFile = + File.createTempFile(fallbackLibraryPath.substring(0, dot), fallbackLibraryPath.substring(dot)); tmpFile.deleteOnExit(); Files.copy(is, tmpFile.toPath(), StandardCopyOption.REPLACE_EXISTING); System.load(tmpFile.getAbsolutePath()); @@ -121,4 +151,17 @@ private static String bundledLibraryPath() { final String libraryName = System.mapLibraryName("opendal_java"); return "/native/" + classifier + "/" + libraryName; } + + private static String fallbackBundledLibraryPath() { + final String classifier = Environment.getClassifier(); + if (!classifier.startsWith("linux-")) { + return null; + } + final String libraryName = System.mapLibraryName("opendal_java"); + if (classifier.endsWith("-musl")) { + final String gnu = classifier.substring(0, classifier.length() - "-musl".length()); + return "/native/" + gnu + "/" + libraryName; + } + return "/native/" + classifier + "-musl/" + libraryName; + } } diff --git a/bindings/java/tools/build.py b/bindings/java/tools/build.py index 0eedaa65ea4c..fdf4ece2cfbd 100755 --- a/bindings/java/tools/build.py +++ b/bindings/java/tools/build.py @@ -19,6 +19,7 @@ from argparse import ArgumentDefaultsHelpFormatter, ArgumentParser from pathlib import Path +import glob import shutil import subprocess @@ -30,8 +31,12 @@ def classifier_to_target(classifier: str) -> str: return "x86_64-apple-darwin" if classifier == "linux-aarch_64": return "aarch64-unknown-linux-gnu" + if classifier == "linux-aarch_64-musl": + return "aarch64-unknown-linux-musl" if classifier == "linux-x86_64": return "x86_64-unknown-linux-gnu" + if classifier == "linux-x86_64-musl": + return "x86_64-unknown-linux-musl" if classifier == "windows-x86_64": return "x86_64-pc-windows-msvc" raise Exception(f"Unsupported classifier: {classifier}") @@ -46,6 +51,9 @@ def get_cargo_artifact_name(classifier: str) -> str: return "opendal_java.dll" raise Exception(f"Unsupported classifier: {classifier}") +def is_musl_runtime() -> bool: + return len(glob.glob("/lib/ld-musl-*.so.1")) > 0 or len(glob.glob("/usr/lib/ld-musl-*.so.1")) > 0 + if __name__ == "__main__": basedir = Path(__file__).parent.parent @@ -68,8 +76,11 @@ def get_cargo_artifact_name(classifier: str) -> str: print("$ " + subprocess.list2cmdline(command)) subprocess.run(command, cwd=basedir, check=True) - # Enable zigbuild if flag enabled and we are building linux target - enable_zigbuild = args.enable_zigbuild == "true" and "linux" in target + # Enable zigbuild if flag enabled and we are building linux gnu target. + # + # For musl targets, prefer using the system musl toolchain (e.g. `musl-tools` on Ubuntu) + # instead of zigbuild. + enable_zigbuild = args.enable_zigbuild == "true" and "linux" in target and not target.endswith("-musl") cmd = [ "cargo", @@ -81,7 +92,7 @@ def get_cargo_artifact_name(classifier: str) -> str: if args.features: cmd += ["--features", args.features] - if enable_zigbuild: + if enable_zigbuild and target.endswith("-gnu"): # Pin glibc to 2.17 if zigbuild has been enabled. cmd += ["--target", f"{target}.2.17"] else: @@ -96,8 +107,15 @@ def get_cargo_artifact_name(classifier: str) -> str: # History reason of cargo profiles. profile = "debug" if args.profile in ["dev", "test", "bench"] else args.profile + artifact = get_cargo_artifact_name(args.classifier) src = output / target / profile / artifact dst = basedir / "target" / "classes" / "native" / args.classifier / artifact dst.parent.mkdir(exist_ok=True, parents=True) + + if target.endswith("-musl") and not is_musl_runtime(): + raise Exception( + "Building musl artifacts requires running inside a musl environment (e.g. Alpine)." + ) + shutil.copy2(src, dst)