perf(lite): bound read allocations and fix shared-library exports

.github/workflows/antfly-artifact-build.yml

@@ -136,7 +136,7 @@ jobs:
         if: ${{ matrix.os == 'linux' }}
         run: |
           sudo apt-get update
-          sudo apt-get install -y build-essential clang lld curl xz-utils git perl python3 linux-libc-dev ca-certificates file
+          sudo apt-get install -y build-essential clang lld llvm curl xz-utils git perl python3 linux-libc-dev ca-certificates file
 
       - name: Install Nix
         if: ${{ matrix.os == 'linux' }}

.github/workflows/zig-tests.yml

@@ -587,6 +587,7 @@ jobs:
           tools/test_audit_test_selection.py
           tools/test_audit_unit_test_ownership.py
           tools/test_check_storage_compilation.py
+          tools/test_capi_exports.py
           tools/test_run_build_cache_contracts.py
           tools/test_run_test_partitions.py
           tools/test_retain_test_process.py
@@ -999,6 +1000,7 @@ jobs:
           tools/test_audit_test_selection.py
           tools/test_audit_unit_test_ownership.py
           tools/test_check_storage_compilation.py
+          tools/test_capi_exports.py
           tools/test_run_build_cache_contracts.py
           tools/test_run_test_partitions.py
           tools/test_retain_test_process.py

zig/bench/baselines/lite-decoded-cache-budget.json

@@ -0,0 +1,99 @@
+{
+  "schema_version": 1,
+  "base_commit": "84dfbf5a953443cd89d2ddec477b8cda3ba922ce",
+  "reference_commit": "4cd4e471e7d82d9574e03a4a0f1b350ea1f1c8d7",
+  "platform": "aarch64-macos",
+  "optimization": "Debug",
+  "scope": "Deterministic allocation and lifetime fixtures; not a query-throughput or 2 GiB soak benchmark",
+  "hot_point_reads": {
+    "queries_after_initial_cold_read": 100,
+    "prefix": {
+      "reference_backend_allocator_calls": 500,
+      "updated_backend_allocator_calls": 5
+    },
+    "prefix_snappy": {
+      "reference_backend_allocator_calls": 604,
+      "updated_backend_allocator_calls": 5
+    },
+    "reference_encoded_block_loads": 100,
+    "updated_encoded_block_loads": 1,
+    "updated_retained_decoded_blocks": 1,
+    "subsequent_100_warm_reads_backend_allocator_calls": 0,
+    "subsequent_100_warm_reads_encoded_block_loads": 0
+  },
+  "repeated_batch_results": {
+    "two_key_batches": 101,
+    "reference_retained_pins": 64,
+    "reference_distinct_pinned_blocks": 1,
+    "reference_owned_values": 74,
+    "updated_retained_pins": 1,
+    "updated_distinct_pinned_blocks": 1,
+    "updated_owned_values": 0
+  },
+  "cache_policy": {
+    "default_payload_residency_bytes": 2097152,
+    "default_max_admitted_block_bytes": 262144,
+    "max_cache_entries": 64,
+    "max_result_owner_pinned_bytes": 1048576,
+    "max_result_owner_distinct_pins": 64,
+    "promotion_misses": 2,
+    "denied_promotion_cooldown_misses": 64,
+    "resource_credit_covers_payload_and_shared_header": true,
+    "evicted_pinned_payload_credit_survives_until_final_release": true,
+    "host_pressure_reclaims_unpinned_cache_without_blocking_backend": true,
+    "denied_admission_preserves_read_results": true,
+    "shutdown_fences_pending_reclaimer_registration": true,
+    "impossible_admission_preserves_existing_cache": true
+  },
+  "notes": [
+    "Allocation counts cover the backend allocator; owned point results still allocate in the caller's result allocator.",
+    "Encoded block loads may hit the provider memory cache; these counts are not physical disk reads.",
+    "Cold decoding and first promotion still allocate. Zero allocation applies to the measured warm backend path.",
+    "Resource accounting follows decoded scratch and uncached payload leases in the read-working-set slice, then transfers admitted payload credit to the cache slice through eviction and result leases. Resource-managed result owners copy unadmitted values.",
+    "The on-disk format and public C ABI are unchanged.",
+    "Workspace-retention capacity excludes arena bookkeeping. Backend allocation counts exclude runtime metadata supplied by its separate allocator; writer scratch reuse is checked separately."
+  ],
+  "previous_pr_commit": "1ad9a16b18be51ac35fd855b4a82e96a99b99513",
+  "warm_write_batches": {
+    "reference_commit": "1ad9a16b18be51ac35fd855b4a82e96a99b99513",
+    "two_key_batches_after_warmup": 100,
+    "reference_backend_allocator_calls": 206,
+    "updated_backend_allocator_calls": 0,
+    "reference_owned_values": 200,
+    "updated_owned_values": 0,
+    "updated_retained_pins": 1,
+    "writer_metadata_allocations_after_first_warm_batch": 0,
+    "mutable_results_preserved_after_later_publication": true,
+    "borrowed_results_preserved_after_cache_eviction": true
+  },
+  "concurrent_cold_batches": {
+    "readers": 8,
+    "joined_readers": 7,
+    "encoded_block_loads": 1,
+    "shared_failure_and_success_cleanup_verified": true
+  },
+  "transient_compressed_point_reads": {
+    "queries_after_workspace_warmup": 100,
+    "prefix_backend_allocator_calls": 100,
+    "prefix_snappy_backend_allocator_calls": 100,
+    "new_cache_admissions": 0,
+    "hot_cached_block_preserved": true
+  },
+  "compressed_negative_lookup": {
+    "bloom_false_positive": true,
+    "reference_encoded_block_loads": 2,
+    "updated_encoded_block_loads": 1,
+    "updated_cache_admissions": 0
+  },
+  "decoder_policy": {
+    "workspaces": 4,
+    "in_flight_records": 16,
+    "default_active_working_byte_gate": 8388608,
+    "max_retained_arena_capacity_per_workspace": 65536,
+    "oversized_operations_run_alone": true,
+    "scratch_backing_allocation_byte_cap_enforced": true,
+    "scratch_and_uncached_lease_resource_slice": "lsm.read_working_set",
+    "cache_admission_transfers_host_credit_without_double_charge": true,
+    "max_retained_writer_batch_keys": 1024
+  }
+}

zig/build.zig

@@ -250,7 +250,7 @@ pub fn create(b: *std.Build) ?Artifacts {
     const capi_mod = embedded.capi_mod;
     const libantfly_link_mod = embedded.libantfly_link_mod;
     const install_libantfly = embedded.install_libantfly;
-    if (install_apple_bridge) |install| install_libantfly.step.dependOn(&install.step);
+    if (install_apple_bridge) |install| install_libantfly.dependOn(&install.step);
     const install_capi_header = embedded.install_capi_header;
     const run_capi_smoke = embedded.run_capi_smoke;
     const run_capi_conformance = embedded.run_capi_conformance;
@@ -1020,7 +1020,7 @@ pub fn create(b: *std.Build) ?Artifacts {
     const lite_step = b.step("lite", "Build and install the Antfly Lite CLI and libantfly C ABI");
     lite_step.dependOn(&b.top_level_steps.get("licenses-antfly-lite").?.step);
     lite_step.dependOn(&install_lite_main.step);
-    lite_step.dependOn(&install_libantfly.step);
+    lite_step.dependOn(install_libantfly);
     lite_step.dependOn(&install_capi_header.step);
 
     const lite_test_step = b.step("lite-test", "Run Lite backend, CLI, bindings, examples, and C ABI packaging checks");

zig/build_support/embedded/embedded.zig

@@ -106,7 +106,7 @@ pub const AddEmbeddedResult = struct {
     capi_root_mod: *std.Build.Module,
     capi_mod: *std.Build.Module,
     libantfly_link_mod: *std.Build.Module,
-    install_libantfly: *std.Build.Step.InstallArtifact,
+    install_libantfly: *std.Build.Step,
     install_capi_header: *std.Build.Step.InstallFile,
     run_capi_smoke: *std.Build.Step.Run,
     run_capi_conformance: *std.Build.Step.Run,
@@ -289,25 +289,15 @@ pub fn addEmbedded(b: *std.Build, options: AddEmbeddedOptions) AddEmbeddedResult
     libantfly_link_mod.linkLibrary(native_inference);
     libantfly_link_mod.linkLibrary(native_enrichment);
     addMacosSdkPaths(b, libantfly_link_mod, target);
-    const libantfly = b.addLibrary(.{
-        .linkage = .dynamic,
-        .name = "antfly",
-        .root_module = libantfly_link_mod,
-        .max_rss = 12 * 1024 * 1024 * 1024,
-    });
-    libantfly.link_gc_sections = true;
-    // Homebrew rewrites the dylib ID to its absolute opt/lib path on install.
-    if (target.result.os.tag == .macos) {
-        libantfly.headerpad_max_install_names = true;
-    }
-    const install_libantfly = b.addInstallArtifact(libantfly, .{});
+    const libantfly = @import("shared_library.zig").add(b, libantfly_link_mod);
+    const install_libantfly = libantfly.install;
     b.dependOnFileContents(b.path("pkg/antfly-embedded/libantfly.pc.in"));
     const pc_path = b.root.join(b.allocator, "pkg/antfly-embedded/libantfly.pc.in") catch @panic("OOM");
     const pc_template = pc_path.root_dir.handle.readFileAlloc(b.graph.io, pc_path.sub_path, b.allocator, .limited(16 * 1024)) catch @panic("unable to read libantfly.pc.in");
     const pc_contents = std.mem.replaceOwned(u8, b.allocator, pc_template, "@VERSION@", options.version) catch @panic("OOM");
     const pc_file = b.addWriteFiles().add("libantfly.pc", pc_contents);
     const install_pkg_config = b.addInstallFileWithDir(pc_file, .lib, "pkgconfig/libantfly.pc");
-    install_libantfly.step.dependOn(&install_pkg_config.step);
+    install_libantfly.dependOn(&install_pkg_config.step);
     b.step("pkgconfig", "Install relocatable libantfly pkg-config metadata").dependOn(&install_pkg_config.step);
 
     const install_capi_header = b.addInstallFileWithDir(
@@ -332,7 +322,7 @@ pub fn addEmbedded(b: *std.Build, options: AddEmbeddedOptions) AddEmbeddedResult
     b.getInstallStep().dependOn(install_licenses);
     const capi_step = b.step("capi", "Build the public libantfly C ABI shared library");
     capi_step.dependOn(install_licenses);
-    capi_step.dependOn(&install_libantfly.step);
+    capi_step.dependOn(install_libantfly);
     capi_step.dependOn(&install_capi_header.step);
 
     const capi_smoke_mod = b.createModule(.{
@@ -349,10 +339,13 @@ pub fn addEmbedded(b: *std.Build, options: AddEmbeddedOptions) AddEmbeddedResult
         .name = "antfly-c-smoke",
         .root_module = capi_smoke_mod,
     });
-    capi_smoke.root_module.linkLibrary(libantfly);
+    libantfly.link(capi_smoke.root_module);
     const run_capi_smoke = b.addRunArtifact(capi_smoke);
+    if (target.result.os.tag == .macos or target.result.ofmt == .elf) b.step("capi-exports-check", "Verify that libantfly exports exactly the public C header functions").dependOn(&libantfly.check.step);
     const capi_smoke_step = b.step("capi-smoke", "Compile and run a C consumer smoke test for libantfly");
     capi_smoke_step.dependOn(&run_capi_smoke.step);
+    if (target.result.os.tag == .macos or target.result.ofmt == .elf) capi_smoke_step.dependOn(&libantfly.check.step);
+    if (b.top_level_steps.get("capi-linkage-test")) |linkage| capi_smoke_step.dependOn(&linkage.step);
 
     // Reference runner for the shared conformance cases every binding runs
     // (pkg/antfly-embedded/capi-conformance/README.md). It calls libantfly only
@@ -376,7 +369,7 @@ pub fn addEmbedded(b: *std.Build, options: AddEmbeddedOptions) AddEmbeddedResult
         .name = "antfly-capi-conformance",
         .root_module = capi_conformance_mod,
     });
-    capi_conformance.root_module.linkLibrary(libantfly);
+    libantfly.link(capi_conformance.root_module);
     const run_capi_conformance = b.addRunArtifact(capi_conformance);
     run_capi_conformance.addDirectoryArg2(b.path("pkg/antfly-embedded/capi-conformance/cases"), .{ .make_absolute = true });
     _ = run_capi_conformance.addOutputDirectoryArg2("capi-conformance-work", .{ .make_absolute = true });
@@ -395,7 +388,7 @@ pub fn addEmbedded(b: *std.Build, options: AddEmbeddedOptions) AddEmbeddedResult
     });
     run_lite_go_tests.argv.insert(b.allocator, 2, .{ .decorated_directory = .{ .lazy_path = b.graph.path(.install_lib, "pkgconfig"), .prefix = "PKG_CONFIG_PATH=", .suffix = "", .make_absolute = true } }) catch @panic("OOM");
     run_lite_go_tests.setCwd(b.path("../go/pkg/embedded"));
-    run_lite_go_tests.step.dependOn(&install_libantfly.step);
+    run_lite_go_tests.step.dependOn(install_libantfly);
     run_lite_go_tests.step.dependOn(&install_capi_header.step);
     const lite_go_test_step = b.step("lite-go-test", "Run Go Antfly Lite binding tests against libantfly");
     lite_go_test_step.dependOn(&run_lite_go_tests.step);
@@ -414,7 +407,7 @@ pub fn addEmbedded(b: *std.Build, options: AddEmbeddedOptions) AddEmbeddedResult
     });
     run_lite_py_tests.argv.insert(b.allocator, 2, .{ .decorated_directory = .{ .lazy_path = b.graph.path(.install_lib, ""), .prefix = "ANTFLY_LIB_DIR=", .suffix = "", .make_absolute = true } }) catch @panic("OOM");
     run_lite_py_tests.setCwd(b.path("../py/packages/embedded"));
-    run_lite_py_tests.step.dependOn(&install_libantfly.step);
+    run_lite_py_tests.step.dependOn(install_libantfly);
     const lite_py_test_step = b.step("lite-py-test", "Run Python Antfly Lite binding tests against libantfly");
     lite_py_test_step.dependOn(&run_lite_py_tests.step);
 
@@ -432,7 +425,7 @@ pub fn addEmbedded(b: *std.Build, options: AddEmbeddedOptions) AddEmbeddedResult
     });
     run_lite_rs_tests.argv.insert(b.allocator, 1, .{ .decorated_directory = .{ .lazy_path = b.graph.path(.install_lib, ""), .prefix = "ANTFLY_LIB_DIR=", .suffix = "", .make_absolute = true } }) catch @panic("OOM");
     run_lite_rs_tests.setCwd(b.path("."));
-    run_lite_rs_tests.step.dependOn(&install_libantfly.step);
+    run_lite_rs_tests.step.dependOn(install_libantfly);
     const lite_rs_test_step = b.step("lite-rs-test", "Run Rust Antfly Lite binding tests against libantfly");
     lite_rs_test_step.dependOn(&run_lite_rs_tests.step);
 
@@ -445,7 +438,7 @@ pub fn addEmbedded(b: *std.Build, options: AddEmbeddedOptions) AddEmbeddedResult
     });
     run_lite_ts_tests.argv.insert(b.allocator, 2, .{ .decorated_directory = .{ .lazy_path = b.graph.path(.install_lib, ""), .prefix = "ANTFLY_LIB_DIR=", .suffix = "", .make_absolute = true } }) catch @panic("OOM");
     run_lite_ts_tests.setCwd(b.path("../ts/packages/embedded"));
-    run_lite_ts_tests.step.dependOn(&install_libantfly.step);
+    run_lite_ts_tests.step.dependOn(install_libantfly);
     const lite_ts_test_step = b.step("lite-ts-test", "Run TypeScript Antfly Lite binding tests against libantfly (needs pnpm install in ts/)");
     lite_ts_test_step.dependOn(&run_lite_ts_tests.step);
 
@@ -463,7 +456,7 @@ pub fn addEmbedded(b: *std.Build, options: AddEmbeddedOptions) AddEmbeddedResult
     });
     run_lite_go_example.argv.insert(b.allocator, 2, .{ .decorated_directory = .{ .lazy_path = b.graph.path(.install_lib, "pkgconfig"), .prefix = "PKG_CONFIG_PATH=", .suffix = "", .make_absolute = true } }) catch @panic("OOM");
     run_lite_go_example.setCwd(b.path("../examples/antfly-lite-go"));
-    run_lite_go_example.step.dependOn(&install_libantfly.step);
+    run_lite_go_example.step.dependOn(install_libantfly);
     run_lite_go_example.step.dependOn(&install_capi_header.step);
     const lite_go_example_step = b.step("lite-go-example", "Run the embedded Go Antfly Lite example app");
     lite_go_example_step.dependOn(&run_lite_go_example.step);
@@ -482,7 +475,7 @@ pub fn addEmbedded(b: *std.Build, options: AddEmbeddedOptions) AddEmbeddedResult
     });
     run_lite_go_retrieval_template.argv.insert(b.allocator, 2, .{ .decorated_directory = .{ .lazy_path = b.graph.path(.install_lib, "pkgconfig"), .prefix = "PKG_CONFIG_PATH=", .suffix = "", .make_absolute = true } }) catch @panic("OOM");
     run_lite_go_retrieval_template.setCwd(b.path("../examples/antfly-lite-retrieval-go"));
-    run_lite_go_retrieval_template.step.dependOn(&install_libantfly.step);
+    run_lite_go_retrieval_template.step.dependOn(install_libantfly);
     run_lite_go_retrieval_template.step.dependOn(&install_capi_header.step);
     const lite_go_retrieval_template_step = b.step("lite-go-retrieval-template", "Run the embedded Go Antfly Lite retrieval template");
     lite_go_retrieval_template_step.dependOn(&run_lite_go_retrieval_template.step);