An image builds and answers

.github/workflows/ci.yml

@@ -185,6 +185,86 @@ jobs:
           env -u ZU_LIBRARY java --enable-native-access=ALL-UNNAMED \
               -cp "$cp$RUNNER_TEMP/user" Main
 
+  # An image has no linker in it. Every downcall stub is machine code
+  # written while the image is built, from a file that says which
+  # signatures to write, and a file that is wrong produces an image that
+  # builds clean and dies on the first query. The unit test checks that
+  # file against what the binding binds; only an image can check that the
+  # file is the file the builder wanted, so one gets built here and run.
+  native:
+    strategy:
+      fail-fast: false
+      matrix:
+        os: [ubuntu-latest, macos-latest]
+    runs-on: ${{ matrix.os }}
+    steps:
+      - uses: actions/checkout@v5
+
+      - uses: actions/checkout@v5
+        with:
+          repository: tamnd/zu
+          path: engine
+
+      - uses: graalvm/setup-graalvm@v1
+        with:
+          java-version: "25"
+          distribution: graalvm
+          cache: maven
+          github-token: ${{ secrets.GITHUB_TOKEN }}
+
+      - uses: Swatinem/rust-cache@v2
+        with:
+          workspaces: engine
+
+      - name: Build libzu
+        working-directory: engine
+        run: cargo build --release -p zu-capi
+
+      - name: Stage the one platform this runner is
+        run: |
+          set -eu
+          case "$RUNNER_OS" in
+            Linux) flavour=linux-amd64; library=libzu.so ;;
+            macOS) flavour=darwin-arm64; library=libzu.dylib ;;
+            *) echo "no row for $RUNNER_OS"; exit 1 ;;
+          esac
+          mkdir -p "zudb-native/lib/$flavour"
+          cp "engine/target/release/$library" "zudb-native/lib/$flavour/$library"
+
+      - run: mvn $MAVEN_ARGS -Pnatives -DskipTests -Denforcer.skip=true package
+
+      # The same program the natives job runs on a JVM, compiled to a
+      # binary instead. No property, no environment variable, and the
+      # library comes out of the image rather than off the disk.
+      - name: An image, and nothing else
+        run: |
+          set -eu
+          mkdir -p "$RUNNER_TEMP/user"
+          cat > "$RUNNER_TEMP/user/Main.java" <<'EOF'
+          import dev.zudb.Connection;
+          import dev.zudb.Result;
+          import dev.zudb.Zu;
+
+          public class Main {
+              public static void main(String[] args) {
+                  System.out.println("found " + Zu.library() + " through " + Zu.source());
+                  try (Connection conn = Connection.memory();
+                       Result r = conn.query("UNWIND [1, 2, 3] AS n RETURN sum(n) AS total")) {
+                      if (r.row(0).getLong(0) != 6L) {
+                          throw new AssertionError("the image answered something else");
+                      }
+                  }
+                  System.out.println("the engine came out of the image and answered");
+              }
+          }
+          EOF
+          cp=$(ls zudb/target/zudb-*.jar zudb-ffm/target/zudb-ffm-*.jar \
+              zudb-native/target/zudb-native-*.jar | grep -v sources | tr '\n' ':')
+          javac -cp "$cp" -d "$RUNNER_TEMP/user" "$RUNNER_TEMP/user/Main.java"
+          native-image -cp "$cp$RUNNER_TEMP/user" \
+              --no-fallback -o "$RUNNER_TEMP/user/main" Main
+          env -u ZU_LIBRARY "$RUNNER_TEMP/user/main"
+
   # What Maven Central will run over the artifacts, run here instead so
   # that a release is not the first time anyone sees it.
   javadoc:

README.md

@@ -214,6 +214,8 @@ Alpine is a separate row rather than a smaller Linux, because a shared object bu
 
 The library inside the jar is a resource, and no loader on any platform can map one of those, so it is copied to a temp file the first time anything needs it and the copy is what gets loaded. That happens once per JVM.
 
+A GraalVM native image needs no configuration for any of this. Both artifacts carry their own reachability metadata: `zudb-ffm` lists every signature it binds, because an image has no linker in it and each downcall stub is machine code the builder has to be told to write, and `zudb-native` registers the libraries so that one ends up inside the image rather than being looked for on a machine that does not have it. Use a classifier rather than the platform-complete jar, or the image carries seven libraries and uses one. CI builds an image on Linux and macOS every run and makes it answer a query, because a metadata file that is wrong produces an image that builds clean and dies on the first call.
+
 On the module path the artifact needs `--add-modules dev.zudb.natives`. Nothing `requires` it, since there is no code in it to require, and a jar nothing requires is a jar that is never resolved and whose resources are therefore invisible. The search says so itself when it comes up empty on a module path, so the failure names the flag rather than leaving a user to work out why the same classpath run worked.
 
 ## How it binds

zudb-ffm/src/main/java/dev/zudb/ffm/Abi.java

@@ -434,11 +434,13 @@ Path library() {
 
   @SuppressWarnings("restricted")
   private MethodHandle h(String name, FunctionDescriptor descriptor) {
+    Shapes.down(descriptor, false);
     return linker.downcallHandle(find(name), descriptor);
   }
 
   @SuppressWarnings("restricted")
   private MethodHandle critical(String name, FunctionDescriptor descriptor) {
+    Shapes.down(descriptor, true);
     return linker.downcallHandle(find(name), descriptor, Linker.Option.critical(false));
   }
 

zudb-ffm/src/main/java/dev/zudb/ffm/Shapes.java

@@ -0,0 +1,61 @@
+package dev.zudb.ffm;
+
+import java.lang.foreign.FunctionDescriptor;
+import java.util.Set;
+import java.util.concurrent.ConcurrentHashMap;
+
+/**
+ * Every shape of call that crosses the boundary, remembered as it is bound.
+ *
+ * <p>This exists for one reader, and it is not a person. Ahead-of-time
+ * compilation cannot see a downcall coming: a stub for a given signature is
+ * machine code that has to be generated while the image is being built, and
+ * the builder has no way of knowing which signatures a program will ask for,
+ * because a {@link FunctionDescriptor} is assembled at run time out of
+ * ordinary objects. So the signatures are written down in a file the image
+ * builder reads, and the file has to say exactly what this binding does.
+ *
+ * <p>Writing that file by hand is how it goes wrong. Forty-odd entries nobody
+ * looks at, one of them stale, and the failure is a native image that builds
+ * clean and dies on a call that a JVM run makes every time. So it is not
+ * written by hand: every binding registers its shape here as it is made, a
+ * test builds the file from what was registered, and the build fails if the
+ * file in the repository is not that. The registry is a few dozen records
+ * filled once per process, which is a cheap way to make a class of bug
+ * impossible.
+ */
+final class Shapes {
+
+  /**
+   * One signature, and which direction it goes.
+   *
+   * @param descriptor the C signature
+   * @param critical whether the downcall skips the thread state transition
+   * @param up whether the engine calls Java rather than the other way round
+   */
+  record Shape(FunctionDescriptor descriptor, boolean critical, boolean up) {}
+
+  private static final Set<Shape> SEEN = ConcurrentHashMap.newKeySet();
+
+  private Shapes() {}
+
+  /** Remembers a call into the library. */
+  static void down(FunctionDescriptor descriptor, boolean critical) {
+    SEEN.add(new Shape(descriptor, critical, false));
+  }
+
+  /** Remembers a call out of it. */
+  static void up(FunctionDescriptor descriptor) {
+    SEEN.add(new Shape(descriptor, false, true));
+  }
+
+  /**
+   * Everything bound so far.
+   *
+   * @return a snapshot, which is every shape this binding uses once the
+   *     library has been loaded
+   */
+  static Set<Shape> seen() {
+    return Set.copyOf(SEEN);
+  }
+}

zudb-ffm/src/main/java/dev/zudb/ffm/Upcall.java

@@ -46,6 +46,7 @@ private Upcall(Arena arena, MemorySegment stub) {
   @SuppressWarnings("restricted")
   static Upcall of(MethodHandle target, FunctionDescriptor descriptor) {
     sweep();
+    Shapes.up(descriptor);
     // Shared rather than confined: the callback arrives on a thread of the
     // library's, and a confined arena would refuse the call it is there to
     // serve.

zudb-ffm/src/main/resources/META-INF/native-image/dev.zudb/zudb-ffm/native-image.properties

@@ -0,0 +1,6 @@
+# The Foreign Function and Memory API is off in an image unless it is
+# asked for, and this artifact is nothing but calls through it, so an
+# image with it on the class path wants it on. Native access is granted
+# to this module for the same reason it is granted on a JVM: the calls
+# are the point.
+Args = --enable-native-access=ALL-UNNAMED

zudb-ffm/src/main/resources/META-INF/native-image/dev.zudb/zudb-ffm/reachability-metadata.json

@@ -0,0 +1,49 @@
+{
+  "foreign": {
+    "downcalls": [
+      {"returnType": "int", "parameterTypes": ["void*", "double", "void*"]},
+      {"returnType": "int", "parameterTypes": ["void*", "int", "long long", "void*"]},
+      {"returnType": "int", "parameterTypes": ["void*", "int", "void*", "void*"]},
+      {"returnType": "int", "parameterTypes": ["void*", "int", "void*"]},
+      {"returnType": "int", "parameterTypes": ["void*", "long long", "int", "void*", "void*"]},
+      {"returnType": "int", "parameterTypes": ["void*", "long long", "int", "void*"]},
+      {"returnType": "int", "parameterTypes": ["void*", "long long", "long long", "void*", "void*", "void*", "void*"]},
+      {"returnType": "int", "parameterTypes": ["void*", "long long", "void*", "void*", "void*"]},
+      {"returnType": "int", "parameterTypes": ["void*", "long long", "void*", "void*"]},
+      {"returnType": "int", "parameterTypes": ["void*", "long long", "void*"]},
+      {"returnType": "int", "parameterTypes": ["void*", "void*", "long long", "double"]},
+      {"returnType": "int", "parameterTypes": ["void*", "void*", "long long", "int", "long long", "int"]},
+      {"returnType": "int", "parameterTypes": ["void*", "void*", "long long", "int", "void*", "long long", "void*"]},
+      {"returnType": "int", "parameterTypes": ["void*", "void*", "long long", "int"]},
+      {"returnType": "int", "parameterTypes": ["void*", "void*", "long long", "long long"]},
+      {"returnType": "int", "parameterTypes": ["void*", "void*", "long long", "void*", "int", "void*", "long long", "long long", "void*"]},
+      {"returnType": "int", "parameterTypes": ["void*", "void*", "long long", "void*", "long long", "int", "int", "long long", "int", "void*"]},
+      {"returnType": "int", "parameterTypes": ["void*", "void*", "long long", "void*", "long long", "int", "void*"]},
+      {"returnType": "int", "parameterTypes": ["void*", "void*", "long long", "void*", "long long", "long long", "void*"]},
+      {"returnType": "int", "parameterTypes": ["void*", "void*", "long long", "void*", "long long", "void*"]},
+      {"returnType": "int", "parameterTypes": ["void*", "void*", "long long", "void*", "long long"]},
+      {"returnType": "int", "parameterTypes": ["void*", "void*", "long long", "void*", "void*", "long long", "void*"]},
+      {"returnType": "int", "parameterTypes": ["void*", "void*", "long long", "void*", "void*", "void*", "long long", "long long", "void*"]},
+      {"returnType": "int", "parameterTypes": ["void*", "void*", "long long", "void*", "void*"]},
+      {"returnType": "int", "parameterTypes": ["void*", "void*", "long long", "void*"]},
+      {"returnType": "int", "parameterTypes": ["void*", "void*", "long long"]},
+      {"returnType": "int", "parameterTypes": ["void*", "void*", "void*", "long long", "void*"]},
+      {"returnType": "int", "parameterTypes": ["void*", "void*", "void*", "long long"]},
+      {"returnType": "int", "parameterTypes": ["void*", "void*", "void*", "void*"]},
+      {"returnType": "int", "parameterTypes": ["void*", "void*", "void*"]},
+      {"returnType": "int", "parameterTypes": ["void*", "void*"]},
+      {"returnType": "int", "parameterTypes": ["void*"], "options": {"critical": {"allowHeapAccess": false}}},
+      {"returnType": "int", "parameterTypes": ["void*"]},
+      {"returnType": "long long", "parameterTypes": ["void*"], "options": {"critical": {"allowHeapAccess": false}}},
+      {"returnType": "void", "parameterTypes": ["void*"]},
+      {"returnType": "void*", "parameterTypes": ["void*", "int", "void*"]},
+      {"returnType": "void*", "parameterTypes": ["void*", "long long", "void*"]},
+      {"returnType": "void*", "parameterTypes": ["void*", "void*"]},
+      {"returnType": "void*", "parameterTypes": []}
+    ],
+    "upcalls": [
+      {"returnType": "int", "parameterTypes": ["void*", "long long", "long long"]},
+      {"returnType": "void", "parameterTypes": ["void*"]}
+    ]
+  }
+}

zudb-ffm/src/test/java/dev/zudb/ffm/ReachabilityTest.java

@@ -0,0 +1,162 @@
+package dev.zudb.ffm;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+
+import dev.zudb.Connection;
+import dev.zudb.Frame;
+import dev.zudb.Result;
+import java.lang.foreign.AddressLayout;
+import java.lang.foreign.FunctionDescriptor;
+import java.lang.foreign.MemoryLayout;
+import java.lang.foreign.ValueLayout;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.time.Duration;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.TreeSet;
+import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.Test;
+
+/**
+ * The file an ahead-of-time image builder reads, checked against what this
+ * binding actually binds.
+ *
+ * <p>A native image has no linker at run time. Every downcall stub and every
+ * upcall stub is machine code generated while the image is built, and the
+ * builder can only generate the ones it was told about, because a signature
+ * here is assembled out of ordinary objects at run time and there is nothing
+ * in the bytecode to read it off. Told wrong, the image builds clean and dies
+ * on a call a JVM run makes every time.
+ *
+ * <p>So the file is not maintained, it is derived. {@link Shapes} remembers
+ * every shape as it is bound, this writes out what that comes to, and the
+ * build fails if the file in the repository says something else. Adding a
+ * function to the C ABI therefore either changes nothing here, because its
+ * shape is one of the forty already listed, or fails this test with the file
+ * that would have been right sitting in {@code target/}.
+ */
+class ReachabilityTest {
+
+  private static final Path CHECKED_IN =
+      Paths.get("src/main/resources/META-INF/native-image/dev.zudb/zudb-ffm")
+          .resolve("reachability-metadata.json");
+
+  @BeforeAll
+  static void engine() {
+    Libzu.require();
+  }
+
+  @Test
+  void theImageBuilderIsToldEveryShapeThisBindingBinds() throws Exception {
+    // Loading the library binds every downcall. The two upcalls are bound
+    // when something wants one, so something has to want one: a registered
+    // frame owns the release callback and a watcher owns the progress
+    // callback, and between them that is every direction this binding goes.
+    try (Connection conn = Connection.memory()) {
+      conn.onProgress(Duration.ofMillis(1), (rows, millis) -> true);
+      try (Frame frame = Frame.of("Reachable", 1)) {
+        frame.column(
+            "id",
+            java.nio.ByteBuffer.allocateDirect(8)
+                .order(java.nio.ByteOrder.nativeOrder())
+                .asLongBuffer()
+                .put(0, 1));
+        conn.register(frame);
+      }
+      try (Result r = conn.query("RETURN 1 AS one")) {
+        assertEquals(1L, r.row(0).getLong(0));
+      }
+    }
+
+    String want = json();
+    Path fallback = Paths.get("target", "reachability-metadata.json");
+    String have = Files.exists(CHECKED_IN) ? Files.readString(CHECKED_IN) : "";
+    if (!want.equals(have)) {
+      Files.createDirectories(fallback.getParent());
+      Files.writeString(fallback, want, StandardCharsets.UTF_8);
+    }
+    assertEquals(
+        want,
+        have,
+        CHECKED_IN
+            + " is not what this binding binds. What it should say is in "
+            + fallback.toAbsolutePath()
+            + ", so copy that over it");
+  }
+
+  /**
+   * What was bound, in the shape the image builder reads.
+   *
+   * <p>Sorted rather than in binding order, so that the file is the same file
+   * however the constructor is rearranged and a diff on it is about what
+   * changed rather than about what moved.
+   */
+  private static String json() {
+    var downcalls = new TreeSet<String>();
+    var upcalls = new TreeSet<String>();
+    for (Shapes.Shape shape : Shapes.seen()) {
+      (shape.up() ? upcalls : downcalls).add(entry(shape));
+    }
+    StringBuilder sb = new StringBuilder();
+    sb.append("{\n  \"foreign\": {\n");
+    sb.append("    \"downcalls\": [\n").append(String.join(",\n", downcalls)).append("\n    ],\n");
+    sb.append("    \"upcalls\": [\n").append(String.join(",\n", upcalls)).append("\n    ]\n");
+    sb.append("  }\n}\n");
+    return sb.toString();
+  }
+
+  private static String entry(Shapes.Shape shape) {
+    FunctionDescriptor d = shape.descriptor();
+    List<String> parameters = new ArrayList<>();
+    for (MemoryLayout layout : d.argumentLayouts()) {
+      parameters.add("\"" + type(layout) + "\"");
+    }
+    StringBuilder sb = new StringBuilder("      {");
+    sb.append("\"returnType\": \"")
+        .append(d.returnLayout().map(ReachabilityTest::type).orElse("void"))
+        .append("\", ");
+    sb.append("\"parameterTypes\": [").append(String.join(", ", parameters)).append("]");
+    if (shape.critical()) {
+      // Bound with Linker.Option.critical(false), and a stub for a critical
+      // call is not the same machine code as a stub for an ordinary one, so
+      // the two are two entries even where the signature is the same.
+      sb.append(", \"options\": {\"critical\": {\"allowHeapAccess\": false}}");
+    }
+    return sb.append("}").toString();
+  }
+
+  /**
+   * A layout as the image builder spells it, which is C's spelling rather
+   * than Java's: the file names canonical layouts, the ones
+   * {@link java.lang.foreign.Linker#canonicalLayouts()} answers to.
+   *
+   * <p>{@code long long} rather than {@code long} for a 64-bit integer, and
+   * that matters: C's {@code long} is four bytes on Windows and eight
+   * everywhere else, and this file is written once and read on all of them.
+   * {@code size_t} is deliberately not used for the same reason, since a
+   * descriptor built from it is the same descriptor as one built from a
+   * {@code long long} on every platform this ships to and naming the concrete
+   * width keeps the file from meaning two things.
+   */
+  private static String type(MemoryLayout layout) {
+    if (layout instanceof AddressLayout) {
+      return "void*";
+    }
+    if (layout instanceof ValueLayout value) {
+      return switch (value.carrier().getSimpleName()) {
+        case "int" -> "int";
+        case "long" -> "long long";
+        case "double" -> "double";
+        case "float" -> "float";
+        case "short" -> "short";
+        case "byte" -> "char";
+        case "boolean" -> "bool";
+        default -> throw new IllegalStateException("no spelling for " + layout);
+      };
+    }
+    throw new IllegalStateException("no spelling for " + layout);
+  }
+}