diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 439b921..be18e27 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -185,6 +185,86 @@ jobs: env -u ZU_LIBRARY java --enable-native-access=ALL-UNNAMED \ -cp "$cp$RUNNER_TEMP/user" Main + # An image has no linker in it. Every downcall stub is machine code + # written while the image is built, from a file that says which + # signatures to write, and a file that is wrong produces an image that + # builds clean and dies on the first query. The unit test checks that + # file against what the binding binds; only an image can check that the + # file is the file the builder wanted, so one gets built here and run. + native: + strategy: + fail-fast: false + matrix: + os: [ubuntu-latest, macos-latest] + runs-on: ${{ matrix.os }} + steps: + - uses: actions/checkout@v5 + + - uses: actions/checkout@v5 + with: + repository: tamnd/zu + path: engine + + - uses: graalvm/setup-graalvm@v1 + with: + java-version: "25" + distribution: graalvm + cache: maven + github-token: ${{ secrets.GITHUB_TOKEN }} + + - uses: Swatinem/rust-cache@v2 + with: + workspaces: engine + + - name: Build libzu + working-directory: engine + run: cargo build --release -p zu-capi + + - name: Stage the one platform this runner is + run: | + set -eu + case "$RUNNER_OS" in + Linux) flavour=linux-amd64; library=libzu.so ;; + macOS) flavour=darwin-arm64; library=libzu.dylib ;; + *) echo "no row for $RUNNER_OS"; exit 1 ;; + esac + mkdir -p "zudb-native/lib/$flavour" + cp "engine/target/release/$library" "zudb-native/lib/$flavour/$library" + + - run: mvn $MAVEN_ARGS -Pnatives -DskipTests -Denforcer.skip=true package + + # The same program the natives job runs on a JVM, compiled to a + # binary instead. No property, no environment variable, and the + # library comes out of the image rather than off the disk. + - name: An image, and nothing else + run: | + set -eu + mkdir -p "$RUNNER_TEMP/user" + cat > "$RUNNER_TEMP/user/Main.java" <<'EOF' + import dev.zudb.Connection; + import dev.zudb.Result; + import dev.zudb.Zu; + + public class Main { + public static void main(String[] args) { + System.out.println("found " + Zu.library() + " through " + Zu.source()); + try (Connection conn = Connection.memory(); + Result r = conn.query("UNWIND [1, 2, 3] AS n RETURN sum(n) AS total")) { + if (r.row(0).getLong(0) != 6L) { + throw new AssertionError("the image answered something else"); + } + } + System.out.println("the engine came out of the image and answered"); + } + } + EOF + cp=$(ls zudb/target/zudb-*.jar zudb-ffm/target/zudb-ffm-*.jar \ + zudb-native/target/zudb-native-*.jar | grep -v sources | tr '\n' ':') + javac -cp "$cp" -d "$RUNNER_TEMP/user" "$RUNNER_TEMP/user/Main.java" + native-image -cp "$cp$RUNNER_TEMP/user" \ + --no-fallback -o "$RUNNER_TEMP/user/main" Main + env -u ZU_LIBRARY "$RUNNER_TEMP/user/main" + # What Maven Central will run over the artifacts, run here instead so # that a release is not the first time anyone sees it. javadoc: diff --git a/README.md b/README.md index 4b39b1b..1cf2446 100644 --- a/README.md +++ b/README.md @@ -214,6 +214,8 @@ Alpine is a separate row rather than a smaller Linux, because a shared object bu The library inside the jar is a resource, and no loader on any platform can map one of those, so it is copied to a temp file the first time anything needs it and the copy is what gets loaded. That happens once per JVM. +A GraalVM native image needs no configuration for any of this. Both artifacts carry their own reachability metadata: `zudb-ffm` lists every signature it binds, because an image has no linker in it and each downcall stub is machine code the builder has to be told to write, and `zudb-native` registers the libraries so that one ends up inside the image rather than being looked for on a machine that does not have it. Use a classifier rather than the platform-complete jar, or the image carries seven libraries and uses one. CI builds an image on Linux and macOS every run and makes it answer a query, because a metadata file that is wrong produces an image that builds clean and dies on the first call. + On the module path the artifact needs `--add-modules dev.zudb.natives`. Nothing `requires` it, since there is no code in it to require, and a jar nothing requires is a jar that is never resolved and whose resources are therefore invisible. The search says so itself when it comes up empty on a module path, so the failure names the flag rather than leaving a user to work out why the same classpath run worked. ## How it binds diff --git a/zudb-ffm/src/main/java/dev/zudb/ffm/Abi.java b/zudb-ffm/src/main/java/dev/zudb/ffm/Abi.java index ae303d9..34950d4 100644 --- a/zudb-ffm/src/main/java/dev/zudb/ffm/Abi.java +++ b/zudb-ffm/src/main/java/dev/zudb/ffm/Abi.java @@ -434,11 +434,13 @@ Path library() { @SuppressWarnings("restricted") private MethodHandle h(String name, FunctionDescriptor descriptor) { + Shapes.down(descriptor, false); return linker.downcallHandle(find(name), descriptor); } @SuppressWarnings("restricted") private MethodHandle critical(String name, FunctionDescriptor descriptor) { + Shapes.down(descriptor, true); return linker.downcallHandle(find(name), descriptor, Linker.Option.critical(false)); } diff --git a/zudb-ffm/src/main/java/dev/zudb/ffm/Shapes.java b/zudb-ffm/src/main/java/dev/zudb/ffm/Shapes.java new file mode 100644 index 0000000..1e3cd28 --- /dev/null +++ b/zudb-ffm/src/main/java/dev/zudb/ffm/Shapes.java @@ -0,0 +1,61 @@ +package dev.zudb.ffm; + +import java.lang.foreign.FunctionDescriptor; +import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; + +/** + * Every shape of call that crosses the boundary, remembered as it is bound. + * + *

This exists for one reader, and it is not a person. Ahead-of-time + * compilation cannot see a downcall coming: a stub for a given signature is + * machine code that has to be generated while the image is being built, and + * the builder has no way of knowing which signatures a program will ask for, + * because a {@link FunctionDescriptor} is assembled at run time out of + * ordinary objects. So the signatures are written down in a file the image + * builder reads, and the file has to say exactly what this binding does. + * + *

Writing that file by hand is how it goes wrong. Forty-odd entries nobody + * looks at, one of them stale, and the failure is a native image that builds + * clean and dies on a call that a JVM run makes every time. So it is not + * written by hand: every binding registers its shape here as it is made, a + * test builds the file from what was registered, and the build fails if the + * file in the repository is not that. The registry is a few dozen records + * filled once per process, which is a cheap way to make a class of bug + * impossible. + */ +final class Shapes { + + /** + * One signature, and which direction it goes. + * + * @param descriptor the C signature + * @param critical whether the downcall skips the thread state transition + * @param up whether the engine calls Java rather than the other way round + */ + record Shape(FunctionDescriptor descriptor, boolean critical, boolean up) {} + + private static final Set SEEN = ConcurrentHashMap.newKeySet(); + + private Shapes() {} + + /** Remembers a call into the library. */ + static void down(FunctionDescriptor descriptor, boolean critical) { + SEEN.add(new Shape(descriptor, critical, false)); + } + + /** Remembers a call out of it. */ + static void up(FunctionDescriptor descriptor) { + SEEN.add(new Shape(descriptor, false, true)); + } + + /** + * Everything bound so far. + * + * @return a snapshot, which is every shape this binding uses once the + * library has been loaded + */ + static Set seen() { + return Set.copyOf(SEEN); + } +} diff --git a/zudb-ffm/src/main/java/dev/zudb/ffm/Upcall.java b/zudb-ffm/src/main/java/dev/zudb/ffm/Upcall.java index 4958b8c..9da580b 100644 --- a/zudb-ffm/src/main/java/dev/zudb/ffm/Upcall.java +++ b/zudb-ffm/src/main/java/dev/zudb/ffm/Upcall.java @@ -46,6 +46,7 @@ private Upcall(Arena arena, MemorySegment stub) { @SuppressWarnings("restricted") static Upcall of(MethodHandle target, FunctionDescriptor descriptor) { sweep(); + Shapes.up(descriptor); // Shared rather than confined: the callback arrives on a thread of the // library's, and a confined arena would refuse the call it is there to // serve. diff --git a/zudb-ffm/src/main/resources/META-INF/native-image/dev.zudb/zudb-ffm/native-image.properties b/zudb-ffm/src/main/resources/META-INF/native-image/dev.zudb/zudb-ffm/native-image.properties new file mode 100644 index 0000000..c69a83d --- /dev/null +++ b/zudb-ffm/src/main/resources/META-INF/native-image/dev.zudb/zudb-ffm/native-image.properties @@ -0,0 +1,6 @@ +# The Foreign Function and Memory API is off in an image unless it is +# asked for, and this artifact is nothing but calls through it, so an +# image with it on the class path wants it on. Native access is granted +# to this module for the same reason it is granted on a JVM: the calls +# are the point. +Args = --enable-native-access=ALL-UNNAMED diff --git a/zudb-ffm/src/main/resources/META-INF/native-image/dev.zudb/zudb-ffm/reachability-metadata.json b/zudb-ffm/src/main/resources/META-INF/native-image/dev.zudb/zudb-ffm/reachability-metadata.json new file mode 100644 index 0000000..5853219 --- /dev/null +++ b/zudb-ffm/src/main/resources/META-INF/native-image/dev.zudb/zudb-ffm/reachability-metadata.json @@ -0,0 +1,49 @@ +{ + "foreign": { + "downcalls": [ + {"returnType": "int", "parameterTypes": ["void*", "double", "void*"]}, + {"returnType": "int", "parameterTypes": ["void*", "int", "long long", "void*"]}, + {"returnType": "int", "parameterTypes": ["void*", "int", "void*", "void*"]}, + {"returnType": "int", "parameterTypes": ["void*", "int", "void*"]}, + {"returnType": "int", "parameterTypes": ["void*", "long long", "int", "void*", "void*"]}, + {"returnType": "int", "parameterTypes": ["void*", "long long", "int", "void*"]}, + {"returnType": "int", "parameterTypes": ["void*", "long long", "long long", "void*", "void*", "void*", "void*"]}, + {"returnType": "int", "parameterTypes": ["void*", "long long", "void*", "void*", "void*"]}, + {"returnType": "int", "parameterTypes": ["void*", "long long", "void*", "void*"]}, + {"returnType": "int", "parameterTypes": ["void*", "long long", "void*"]}, + {"returnType": "int", "parameterTypes": ["void*", "void*", "long long", "double"]}, + {"returnType": "int", "parameterTypes": ["void*", "void*", "long long", "int", "long long", "int"]}, + {"returnType": "int", "parameterTypes": ["void*", "void*", "long long", "int", "void*", "long long", "void*"]}, + {"returnType": "int", "parameterTypes": ["void*", "void*", "long long", "int"]}, + {"returnType": "int", "parameterTypes": ["void*", "void*", "long long", "long long"]}, + {"returnType": "int", "parameterTypes": ["void*", "void*", "long long", "void*", "int", "void*", "long long", "long long", "void*"]}, + {"returnType": "int", "parameterTypes": ["void*", "void*", "long long", "void*", "long long", "int", "int", "long long", "int", "void*"]}, + {"returnType": "int", "parameterTypes": ["void*", "void*", "long long", "void*", "long long", "int", "void*"]}, + {"returnType": "int", "parameterTypes": ["void*", "void*", "long long", "void*", "long long", "long long", "void*"]}, + {"returnType": "int", "parameterTypes": ["void*", "void*", "long long", "void*", "long long", "void*"]}, + {"returnType": "int", "parameterTypes": ["void*", "void*", "long long", "void*", "long long"]}, + {"returnType": "int", "parameterTypes": ["void*", "void*", "long long", "void*", "void*", "long long", "void*"]}, + {"returnType": "int", "parameterTypes": ["void*", "void*", "long long", "void*", "void*", "void*", "long long", "long long", "void*"]}, + {"returnType": "int", "parameterTypes": ["void*", "void*", "long long", "void*", "void*"]}, + {"returnType": "int", "parameterTypes": ["void*", "void*", "long long", "void*"]}, + {"returnType": "int", "parameterTypes": ["void*", "void*", "long long"]}, + {"returnType": "int", "parameterTypes": ["void*", "void*", "void*", "long long", "void*"]}, + {"returnType": "int", "parameterTypes": ["void*", "void*", "void*", "long long"]}, + {"returnType": "int", "parameterTypes": ["void*", "void*", "void*", "void*"]}, + {"returnType": "int", "parameterTypes": ["void*", "void*", "void*"]}, + {"returnType": "int", "parameterTypes": ["void*", "void*"]}, + {"returnType": "int", "parameterTypes": ["void*"], "options": {"critical": {"allowHeapAccess": false}}}, + {"returnType": "int", "parameterTypes": ["void*"]}, + {"returnType": "long long", "parameterTypes": ["void*"], "options": {"critical": {"allowHeapAccess": false}}}, + {"returnType": "void", "parameterTypes": ["void*"]}, + {"returnType": "void*", "parameterTypes": ["void*", "int", "void*"]}, + {"returnType": "void*", "parameterTypes": ["void*", "long long", "void*"]}, + {"returnType": "void*", "parameterTypes": ["void*", "void*"]}, + {"returnType": "void*", "parameterTypes": []} + ], + "upcalls": [ + {"returnType": "int", "parameterTypes": ["void*", "long long", "long long"]}, + {"returnType": "void", "parameterTypes": ["void*"]} + ] + } +} diff --git a/zudb-ffm/src/test/java/dev/zudb/ffm/ReachabilityTest.java b/zudb-ffm/src/test/java/dev/zudb/ffm/ReachabilityTest.java new file mode 100644 index 0000000..06eee67 --- /dev/null +++ b/zudb-ffm/src/test/java/dev/zudb/ffm/ReachabilityTest.java @@ -0,0 +1,162 @@ +package dev.zudb.ffm; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +import dev.zudb.Connection; +import dev.zudb.Frame; +import dev.zudb.Result; +import java.lang.foreign.AddressLayout; +import java.lang.foreign.FunctionDescriptor; +import java.lang.foreign.MemoryLayout; +import java.lang.foreign.ValueLayout; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.time.Duration; +import java.util.ArrayList; +import java.util.List; +import java.util.TreeSet; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; + +/** + * The file an ahead-of-time image builder reads, checked against what this + * binding actually binds. + * + *

A native image has no linker at run time. Every downcall stub and every + * upcall stub is machine code generated while the image is built, and the + * builder can only generate the ones it was told about, because a signature + * here is assembled out of ordinary objects at run time and there is nothing + * in the bytecode to read it off. Told wrong, the image builds clean and dies + * on a call a JVM run makes every time. + * + *

So the file is not maintained, it is derived. {@link Shapes} remembers + * every shape as it is bound, this writes out what that comes to, and the + * build fails if the file in the repository says something else. Adding a + * function to the C ABI therefore either changes nothing here, because its + * shape is one of the forty already listed, or fails this test with the file + * that would have been right sitting in {@code target/}. + */ +class ReachabilityTest { + + private static final Path CHECKED_IN = + Paths.get("src/main/resources/META-INF/native-image/dev.zudb/zudb-ffm") + .resolve("reachability-metadata.json"); + + @BeforeAll + static void engine() { + Libzu.require(); + } + + @Test + void theImageBuilderIsToldEveryShapeThisBindingBinds() throws Exception { + // Loading the library binds every downcall. The two upcalls are bound + // when something wants one, so something has to want one: a registered + // frame owns the release callback and a watcher owns the progress + // callback, and between them that is every direction this binding goes. + try (Connection conn = Connection.memory()) { + conn.onProgress(Duration.ofMillis(1), (rows, millis) -> true); + try (Frame frame = Frame.of("Reachable", 1)) { + frame.column( + "id", + java.nio.ByteBuffer.allocateDirect(8) + .order(java.nio.ByteOrder.nativeOrder()) + .asLongBuffer() + .put(0, 1)); + conn.register(frame); + } + try (Result r = conn.query("RETURN 1 AS one")) { + assertEquals(1L, r.row(0).getLong(0)); + } + } + + String want = json(); + Path fallback = Paths.get("target", "reachability-metadata.json"); + String have = Files.exists(CHECKED_IN) ? Files.readString(CHECKED_IN) : ""; + if (!want.equals(have)) { + Files.createDirectories(fallback.getParent()); + Files.writeString(fallback, want, StandardCharsets.UTF_8); + } + assertEquals( + want, + have, + CHECKED_IN + + " is not what this binding binds. What it should say is in " + + fallback.toAbsolutePath() + + ", so copy that over it"); + } + + /** + * What was bound, in the shape the image builder reads. + * + *

Sorted rather than in binding order, so that the file is the same file + * however the constructor is rearranged and a diff on it is about what + * changed rather than about what moved. + */ + private static String json() { + var downcalls = new TreeSet(); + var upcalls = new TreeSet(); + for (Shapes.Shape shape : Shapes.seen()) { + (shape.up() ? upcalls : downcalls).add(entry(shape)); + } + StringBuilder sb = new StringBuilder(); + sb.append("{\n \"foreign\": {\n"); + sb.append(" \"downcalls\": [\n").append(String.join(",\n", downcalls)).append("\n ],\n"); + sb.append(" \"upcalls\": [\n").append(String.join(",\n", upcalls)).append("\n ]\n"); + sb.append(" }\n}\n"); + return sb.toString(); + } + + private static String entry(Shapes.Shape shape) { + FunctionDescriptor d = shape.descriptor(); + List parameters = new ArrayList<>(); + for (MemoryLayout layout : d.argumentLayouts()) { + parameters.add("\"" + type(layout) + "\""); + } + StringBuilder sb = new StringBuilder(" {"); + sb.append("\"returnType\": \"") + .append(d.returnLayout().map(ReachabilityTest::type).orElse("void")) + .append("\", "); + sb.append("\"parameterTypes\": [").append(String.join(", ", parameters)).append("]"); + if (shape.critical()) { + // Bound with Linker.Option.critical(false), and a stub for a critical + // call is not the same machine code as a stub for an ordinary one, so + // the two are two entries even where the signature is the same. + sb.append(", \"options\": {\"critical\": {\"allowHeapAccess\": false}}"); + } + return sb.append("}").toString(); + } + + /** + * A layout as the image builder spells it, which is C's spelling rather + * than Java's: the file names canonical layouts, the ones + * {@link java.lang.foreign.Linker#canonicalLayouts()} answers to. + * + *

{@code long long} rather than {@code long} for a 64-bit integer, and + * that matters: C's {@code long} is four bytes on Windows and eight + * everywhere else, and this file is written once and read on all of them. + * {@code size_t} is deliberately not used for the same reason, since a + * descriptor built from it is the same descriptor as one built from a + * {@code long long} on every platform this ships to and naming the concrete + * width keeps the file from meaning two things. + */ + private static String type(MemoryLayout layout) { + if (layout instanceof AddressLayout) { + return "void*"; + } + if (layout instanceof ValueLayout value) { + return switch (value.carrier().getSimpleName()) { + case "int" -> "int"; + case "long" -> "long long"; + case "double" -> "double"; + case "float" -> "float"; + case "short" -> "short"; + case "byte" -> "char"; + case "boolean" -> "bool"; + default -> throw new IllegalStateException("no spelling for " + layout); + }; + } + throw new IllegalStateException("no spelling for " + layout); + } +} diff --git a/zudb-native/pom.xml b/zudb-native/pom.xml index 28c272e..6894978 100644 --- a/zudb-native/pom.xml +++ b/zudb-native/pom.xml @@ -46,6 +46,15 @@ ${zu.natives.dir} dev/zudb/native + + + ${project.basedir}/src/metadata + @@ -105,7 +114,10 @@ jar linux-amd64 - dev/zudb/native/linux-amd64/** + + dev/zudb/native/linux-amd64/** + META-INF/** + @@ -113,7 +125,10 @@ jar linux-arm64 - dev/zudb/native/linux-arm64/** + + dev/zudb/native/linux-arm64/** + META-INF/** + @@ -121,7 +136,10 @@ jar linux-amd64-musl - dev/zudb/native/linux-amd64-musl/** + + dev/zudb/native/linux-amd64-musl/** + META-INF/** + @@ -129,7 +147,10 @@ jar linux-arm64-musl - dev/zudb/native/linux-arm64-musl/** + + dev/zudb/native/linux-arm64-musl/** + META-INF/** + @@ -137,7 +158,10 @@ jar darwin-amd64 - dev/zudb/native/darwin-amd64/** + + dev/zudb/native/darwin-amd64/** + META-INF/** + @@ -145,7 +169,10 @@ jar darwin-arm64 - dev/zudb/native/darwin-arm64/** + + dev/zudb/native/darwin-arm64/** + META-INF/** + @@ -153,7 +180,10 @@ jar windows-amd64 - dev/zudb/native/windows-amd64/** + + dev/zudb/native/windows-amd64/** + META-INF/** + diff --git a/zudb-native/src/metadata/META-INF/native-image/dev.zudb/zudb-native/reachability-metadata.json b/zudb-native/src/metadata/META-INF/native-image/dev.zudb/zudb-native/reachability-metadata.json new file mode 100644 index 0000000..1552d75 --- /dev/null +++ b/zudb-native/src/metadata/META-INF/native-image/dev.zudb/zudb-native/reachability-metadata.json @@ -0,0 +1,7 @@ +{ + "resources": [ + {"glob": "dev/zudb/native/*/libzu.so"}, + {"glob": "dev/zudb/native/*/libzu.dylib"}, + {"glob": "dev/zudb/native/*/zu.dll"} + ] +}