diff --git a/Cargo.lock b/Cargo.lock index bb82176..757f593 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1031,6 +1031,8 @@ dependencies = [ "duckdb", "ducklink-runtime", "libduckdb-sys", + "serde_json", + "sha2", "wasmtime", "wasmtime-wasi", ] diff --git a/Cargo.toml b/Cargo.toml index c6a78ce..963ca95 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -44,6 +44,11 @@ ducklink-runtime = { path = "runtime" } wasmtime = { version = "46.0.1", features = ["component-model"] } wasmtime-wasi = "46.0.1" anyhow = "1" +# Manifest parsing for the lifted resolver (de-risk spike for design "D"). +serde_json = "1" +# Conformance suite content-digest (the canonical suite_digest the resolver gate +# checks against the manifest's recorded digest). +sha2 = "0.10" # DuckDB C Extension API (loadable-extension build). Pinned to the DuckDB version # the community-extensions CI builds against (v1.5.4); bumped per release. diff --git a/runtime/src/extension.rs b/runtime/src/extension.rs index 6f6cf90..68da05e 100644 --- a/runtime/src/extension.rs +++ b/runtime/src/extension.rs @@ -1409,6 +1409,12 @@ pub struct ExtensionInstance { // Lazily-built storage bindings (None until first storage-dispatch call or // for non-storage extensions). storage_bindings: Option, + // 2.1.0 additive: lazily-built writable-storage bindings (None until first + // storage-write-dispatch call or for read-only storage backends). The types + // are remapped via `with:` to the storage bindings' generated types, so the + // existing `storage_*_to_ext` helpers apply on the return path. + storage_write_bindings: + Option, // Item 3 / M2a: lazily-built index bindings (None until first index-dispatch // call or for non-index extensions). index_bindings: Option, @@ -1561,6 +1567,7 @@ impl ExtensionInstance { bindings, instance, storage_bindings: None, + storage_write_bindings: None, index_bindings: None, files_bindings: None, } @@ -1824,6 +1831,143 @@ impl ExtensionInstance { .map_err(storage_duckerror_to_ext) } + // --- 2.1.0 (additive): storage-write-dispatch re-entry --- + // + // Mirrors the storage-dispatch trampolines above but drives the WRITABLE + // half (transactions + DDL + DML) of a storage backend. Types are remapped + // via `with:` in the lib-level bindings module to the SAME + // `extension_types::*` the rest of the runtime uses, so no per-world + // conversion is needed on either the argument or the return path. + + /// Builds (once) the writable-storage bindings from the raw instance. + /// Errors if this component does not export storage-write-dispatch (i.e. + /// is not a writable storage backend). + fn storage_write_bindings( + &mut self, + ) -> Result< + &crate::duckdb_extension_storage_write_bindings::DuckdbExtensionStorageWrite, + extension_types::Duckerror, + > { + if self.storage_write_bindings.is_none() { + let built = + crate::duckdb_extension_storage_write_bindings::DuckdbExtensionStorageWrite::new( + self.store.as_context_mut(), + &self.instance, + ) + .map_err(map_extension_trap)?; + self.storage_write_bindings = Some(built); + } + Ok(self.storage_write_bindings.as_ref().unwrap()) + } + + /// Begin a write transaction on `catalog`; returns a transaction handle. + pub fn storage_begin_transaction( + &mut self, + handle: u32, + catalog: u32, + ) -> Result { + self.storage_write_bindings()?; + let bindings = self.storage_write_bindings.as_ref().unwrap(); + let guest = bindings.duckdb_extension_storage_write_dispatch(); + let store = &mut self.store; + guest + .call_begin_transaction(store.as_context_mut(), handle, catalog) + .map_err(map_extension_trap)? + } + + pub fn storage_commit_transaction( + &mut self, + handle: u32, + txn: u32, + ) -> Result<(), extension_types::Duckerror> { + self.storage_write_bindings()?; + let bindings = self.storage_write_bindings.as_ref().unwrap(); + let guest = bindings.duckdb_extension_storage_write_dispatch(); + let store = &mut self.store; + guest + .call_commit_transaction(store.as_context_mut(), handle, txn) + .map_err(map_extension_trap)? + } + + pub fn storage_rollback_transaction( + &mut self, + handle: u32, + txn: u32, + ) -> Result<(), extension_types::Duckerror> { + self.storage_write_bindings()?; + let bindings = self.storage_write_bindings.as_ref().unwrap(); + let guest = bindings.duckdb_extension_storage_write_dispatch(); + let store = &mut self.store; + guest + .call_rollback_transaction(store.as_context_mut(), handle, txn) + .map_err(map_extension_trap)? + } + + pub fn storage_create_table( + &mut self, + handle: u32, + txn: u32, + table: &str, + columns: &[extension_types::Columndef], + ) -> Result<(), extension_types::Duckerror> { + self.storage_write_bindings()?; + let bindings = self.storage_write_bindings.as_ref().unwrap(); + let guest = bindings.duckdb_extension_storage_write_dispatch(); + let store = &mut self.store; + guest + .call_create_table(store.as_context_mut(), handle, txn, table, columns) + .map_err(map_extension_trap)? + } + + pub fn storage_insert_rows( + &mut self, + handle: u32, + txn: u32, + table: &str, + rows: &[Vec], + ) -> Result { + self.storage_write_bindings()?; + let bindings = self.storage_write_bindings.as_ref().unwrap(); + let guest = bindings.duckdb_extension_storage_write_dispatch(); + let store = &mut self.store; + guest + .call_insert_rows(store.as_context_mut(), handle, txn, table, rows) + .map_err(map_extension_trap)? + } + + pub fn storage_delete_rows( + &mut self, + handle: u32, + txn: u32, + table: &str, + rowids: &[i64], + ) -> Result { + self.storage_write_bindings()?; + let bindings = self.storage_write_bindings.as_ref().unwrap(); + let guest = bindings.duckdb_extension_storage_write_dispatch(); + let store = &mut self.store; + guest + .call_delete_rows(store.as_context_mut(), handle, txn, table, rowids) + .map_err(map_extension_trap)? + } + + pub fn storage_update_rows( + &mut self, + handle: u32, + txn: u32, + table: &str, + rowids: &[i64], + rows: &[Vec], + ) -> Result { + self.storage_write_bindings()?; + let bindings = self.storage_write_bindings.as_ref().unwrap(); + let guest = bindings.duckdb_extension_storage_write_dispatch(); + let store = &mut self.store; + guest + .call_update_rows(store.as_context_mut(), handle, txn, table, rowids, rows) + .map_err(map_extension_trap)? + } + // --- Item 3 / M2a: index-dispatch (custom index build + search) re-entry --- // Mirrors the storage-dispatch `storage_*` methods but drives the component's // exported `index-dispatch` interface. The HNSW (or other ANN) build happens diff --git a/runtime/src/lib.rs b/runtime/src/lib.rs index 0ac66a5..561657e 100644 --- a/runtime/src/lib.rs +++ b/runtime/src/lib.rs @@ -167,6 +167,29 @@ pub mod duckdb_extension_storage_bindings { }); } +/// Bindings for the writable-storage world (`duckdb-extension-storage-write`, +/// additive within contract-major 2), which additionally exports +/// `storage-write-dispatch` on top of the read-only `storage-dispatch`. Only +/// writable storage backends satisfy this; the runtime builds these bindings +/// lazily from an already-loaded component instance so non-writable storage +/// backends (which don't export storage-write-dispatch) still load against the +/// read-only storage world above. +/// +/// Types (`duckerror`, `duckvalue`, `columndef`) are remapped to the BASE +/// bindings' generated types via `with:`, so the write trampolines exchange +/// the same `extension_types::*` the rest of the runtime uses -- no per-world +/// conversion on either the argument or the return path. +pub mod duckdb_extension_storage_write_bindings { + wasmtime::component::bindgen!({ + path: "./wit", + world: "duckdb:extension-host/duckdb-extension-storage-write", + require_store_data_send: true, + with: { + "duckdb:extension/types@2.0.0": crate::duckdb_extension_bindings::duckdb::extension::types, + }, + }); +} + /// Bindings for the index-capable world (`duckdb-extension-index`), which /// additionally exports `index-dispatch` (Item 3 / M2a custom index). Only /// custom-index backend components (e.g. hnswfns) satisfy this; the runtime diff --git a/runtime/wit-canonical/duckdb-extension/storage-write-dispatch.wit b/runtime/wit-canonical/duckdb-extension/storage-write-dispatch.wit new file mode 100644 index 0000000..5fa6a1f --- /dev/null +++ b/runtime/wit-canonical/duckdb-extension/storage-write-dispatch.wit @@ -0,0 +1,48 @@ +package duckdb:extension@2.0.0; + +use types; + +// Host -> component callbacks for the WRITABLE half of a storage backend +// (additive, 2.1.0). The read-only `storage-dispatch` (attach/list/scan) is left +// INTACT; this adds the mutating path -- transactions, DDL, and DML -- for +// storage backends that can write through. Exported by writable storage +// components via the separate `duckdb-extension-storage-write` world. +// +// Every call carries `handle` = the callback-handle the component passed to +// storage.register-storage; `catalog` is the catalog handle from +// storage-dispatch.storage-attach. +interface storage-write-dispatch { + use types.{duckerror, duckvalue, columndef}; + + // Begin a transaction on `catalog`; returns a transaction handle. + begin-transaction: func(handle: u32, catalog: u32) -> result; + + commit-transaction: func(handle: u32, txn: u32) -> result<_, duckerror>; + rollback-transaction: func(handle: u32, txn: u32) -> result<_, duckerror>; + + // CREATE TABLE within `txn`. + create-table: func(handle: u32, + txn: u32, + table: string, + columns: list) -> result<_, duckerror>; + + // Append rows; returns the number inserted. + insert-rows: func(handle: u32, + txn: u32, + table: string, + rows: list>) -> result; + + // Delete rows by row-id; returns the number deleted. + delete-rows: func(handle: u32, + txn: u32, + table: string, + rowids: list) -> result; + + // Update rows by row-id (parallel `rowids` / `rows`); returns the number + // updated. + update-rows: func(handle: u32, + txn: u32, + table: string, + rowids: list, + rows: list>) -> result; +} diff --git a/runtime/wit/deps/duckdb-extension/storage-write-dispatch.wit b/runtime/wit/deps/duckdb-extension/storage-write-dispatch.wit new file mode 100644 index 0000000..5fa6a1f --- /dev/null +++ b/runtime/wit/deps/duckdb-extension/storage-write-dispatch.wit @@ -0,0 +1,48 @@ +package duckdb:extension@2.0.0; + +use types; + +// Host -> component callbacks for the WRITABLE half of a storage backend +// (additive, 2.1.0). The read-only `storage-dispatch` (attach/list/scan) is left +// INTACT; this adds the mutating path -- transactions, DDL, and DML -- for +// storage backends that can write through. Exported by writable storage +// components via the separate `duckdb-extension-storage-write` world. +// +// Every call carries `handle` = the callback-handle the component passed to +// storage.register-storage; `catalog` is the catalog handle from +// storage-dispatch.storage-attach. +interface storage-write-dispatch { + use types.{duckerror, duckvalue, columndef}; + + // Begin a transaction on `catalog`; returns a transaction handle. + begin-transaction: func(handle: u32, catalog: u32) -> result; + + commit-transaction: func(handle: u32, txn: u32) -> result<_, duckerror>; + rollback-transaction: func(handle: u32, txn: u32) -> result<_, duckerror>; + + // CREATE TABLE within `txn`. + create-table: func(handle: u32, + txn: u32, + table: string, + columns: list) -> result<_, duckerror>; + + // Append rows; returns the number inserted. + insert-rows: func(handle: u32, + txn: u32, + table: string, + rows: list>) -> result; + + // Delete rows by row-id; returns the number deleted. + delete-rows: func(handle: u32, + txn: u32, + table: string, + rowids: list) -> result; + + // Update rows by row-id (parallel `rowids` / `rows`); returns the number + // updated. + update-rows: func(handle: u32, + txn: u32, + table: string, + rowids: list, + rows: list>) -> result; +} diff --git a/runtime/wit/duckdb-extension-host.wit b/runtime/wit/duckdb-extension-host.wit index 0d0f417..4bc68df 100644 --- a/runtime/wit/duckdb-extension-host.wit +++ b/runtime/wit/duckdb-extension-host.wit @@ -8,6 +8,7 @@ use duckdb:extension/catalog@2.0.0; use duckdb:extension/files@2.0.0; use duckdb:extension/storage@2.0.0; use duckdb:extension/storage-dispatch@2.0.0; +use duckdb:extension/storage-write-dispatch@2.0.0; use duckdb:extension/index@2.0.0; use duckdb:extension/index-dispatch@2.0.0; use duckdb:extension/collation@2.0.0; @@ -76,6 +77,26 @@ world duckdb-extension-index { export index-dispatch; } +// Writable-storage world (additive, 2.1.0 within contract-major 2): additionally +// exports storage-write-dispatch on top of the read-only storage-dispatch. Only +// storage backends that can write through satisfy this; the runtime instantiates +// its bindings lazily from an already-loaded ExtensionInstance. Existing @2.0.0 +// components do NOT export storage-write-dispatch and keep loading un-rebuilt. +world duckdb-extension-storage-write { + import duckdb:extension/runtime@2.0.0; + import duckdb:extension/config@2.0.0; + import duckdb:extension/logging@2.0.0; + import duckdb:extension/catalog@2.0.0; + import duckdb:extension/files@2.0.0; + import duckdb:extension/storage@2.0.0; + import duckdb:extension/index@2.0.0; + import duckdb:extension/collation@2.0.0; + export duckdb:extension/guest@2.0.0; + export callback-dispatch; + export storage-dispatch; + export storage-write-dispatch; +} + // Files-capable world: additionally exports file-dispatch. Only files backend // components (e.g. webfs) satisfy this; the runtime instantiates its bindings // lazily from an already-loaded ExtensionInstance. diff --git a/src/engine.rs b/src/engine.rs index 02bb2e4..ae926db 100644 --- a/src/engine.rs +++ b/src/engine.rs @@ -134,6 +134,30 @@ pub struct AggregateFunc { pub callback_handle: u32, } +/// A logical-type registration a component requested during load(). +/// #79 scaffold: drain path carries these through; the DuckDB sink currently +/// logs but does not yet invoke `duckdb_create_logical_type` + alias — that's +/// a follow-up C-API impl. Once wired, GEOMETRY/GEOGRAPHY etc. become +/// bind-time aliases over BLOB, closing the "st_astext(BLOB) no-match" gap. +#[derive(Clone, Debug)] +pub struct LogicalTypeAlias { + pub extension: String, + pub name: String, + pub physical: String, +} + +/// A cast registration a component requested during load(). #79 scaffold — +/// paired with `LogicalTypeAlias` for the eventual DuckDB C-API implicit-cast +/// wiring. `callback_handle` refers to the guest-side identity/marshal fn +/// (currently ignored by the stub sink). +#[derive(Clone, Debug)] +pub struct CastReg { + pub extension: String, + pub source: String, + pub target: String, + pub callback_handle: u32, +} + /// What a component registered: the functions a direction-specific sink bridges /// into the database. #[derive(Clone, Debug, Default)] @@ -141,6 +165,10 @@ pub struct LoadedComponent { pub scalars: Vec, pub tables: Vec, pub aggregates: Vec, + /// #79 scaffold — see `LogicalTypeAlias`. + pub logical_types: Vec, + /// #79 scaffold — see `CastReg`. + pub casts: Vec, } /// Process-wide Direction-2 engine: loads components and dispatches DuckDB @@ -218,11 +246,39 @@ impl Engine2 { callback_handle: a.callback_handle, }) .collect(); + // #79 scaffold — drain logical_types + casts so a follow-up sink can + // wire them into DuckDB's C-API. Currently the direction-2 sink (see + // reg_duckdb::register_logical_types_stub) logs but does not create + // aliases; every GEOMETRY-typed scalar therefore still surfaces to + // the binder as BLOB. When the C-API impl lands (duckdb_create_logical_type + // + duckdb_logical_type_set_alias + duckdb_register_cast), the binder + // resolves GEOMETRY → BLOB at bind time. + let logical_types = pending + .logical_types + .into_iter() + .map(|lt| LogicalTypeAlias { + extension: lt.extension, + name: lt.name, + physical: lt.physical, + }) + .collect(); + let casts = pending + .casts + .into_iter() + .map(|c| CastReg { + extension: c.extension, + source: c.source, + target: c.target, + callback_handle: c.callback_handle, + }) + .collect(); self.instances.insert(extension.to_string(), instance); Ok(LoadedComponent { scalars, tables, aggregates, + logical_types, + casts, }) } @@ -243,7 +299,7 @@ impl Engine2 { }; let instance = self .instances - .get_mut(&entry.extension) + .get_mut(&*entry.extension) .ok_or_else(|| anyhow!("extension '{}' is not loaded", entry.extension))?; let wit_args: Vec = args.into_iter().map(neutral_to_wit).collect(); @@ -282,7 +338,7 @@ impl Engine2 { }; let instance = self .instances - .get_mut(&entry.extension) + .get_mut(&*entry.extension) .ok_or_else(|| anyhow!("extension '{}' is not loaded", entry.extension))?; let ctx = extension_runtime::Invokeinfo { rowindex: Some(base_row_index), @@ -309,7 +365,7 @@ impl Engine2 { }; let instance = self .instances - .get_mut(&entry.extension) + .get_mut(&*entry.extension) .ok_or_else(|| anyhow!("extension '{}' is not loaded", entry.extension))?; let wit_args: Vec = args.into_iter().map(neutral_to_wit).collect(); @@ -339,7 +395,7 @@ impl Engine2 { }; let instance = self .instances - .get_mut(&entry.extension) + .get_mut(&*entry.extension) .ok_or_else(|| anyhow!("extension '{}' is not loaded", entry.extension))?; let wit_rows: Vec> = rows .into_iter() diff --git a/src/lib.rs b/src/lib.rs index 18d1983..9bd9da0 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -16,12 +16,24 @@ pub mod engine; +/// The multi-provider resolver spine (design A), lifted VERBATIM from +/// `crates/ducklink-host/src/resolver.rs` to prove it drops into a native +/// extension crate unchanged (de-risk spike for design "D"). Self-contained: no +/// wasmtime / no engine types. +pub mod resolver; + /// The Direction-2 DuckDB sink (registration + dispatch). Present whenever the /// duckdb crate is available (the `loadable` and `bundled` features both enable /// it); the `bundled` end-to-end test lives in this module. #[cfg(feature = "duckdb-api")] pub mod reg_duckdb; +/// The native-passthrough hook (de-risk spike): `ducklink_load('aba')` resolves a +/// provider via [`resolver`] and dual-loads it (wasm arm via the Route-A bridge / +/// native arm via DuckDB's own `LOAD`). +#[cfg(feature = "duckdb-api")] +pub mod passthrough; + #[cfg(feature = "loadable")] mod loadable { use std::error::Error; @@ -31,6 +43,7 @@ mod loadable { use duckdb::core::{DataChunkHandle, Inserter, LogicalTypeId}; use duckdb::ffi; use duckdb::vscalar::{ScalarFunctionSignature, VScalar}; + use duckdb::types::DuckString; use duckdb::vtab::arrow::WritableVector; use duckdb::Connection; @@ -171,4 +184,205 @@ mod loadable { fn stringify(err: impl std::fmt::Display) -> Box { err.to_string().into() } + + // ----------------------------------------------------------------------- + // The DuckLink SHIM entrypoint (transparent LOAD, design "D" first-cut). + // + // A DuckLink "shim" is a `.duckdb_extension` named after a LOGICAL extension + // (e.g. `aba`). Stock DuckDB derives the init symbol from the FILENAME + // (`_init_c_api`), so `aba.duckdb_extension` must export + // `aba_init_c_api`. After a one-time `ducklink install aba` (INSTALL FROM the + // DuckLink repo), plain `LOAD aba` on STOCK duckdb calls this symbol; the + // shim runs the multi-provider resolver IN-PROCESS (reading the manifest) and + // dual-loads the chosen provider (native passthrough, or the wasm bridge). + // + // The shim is identical for every name -- the NAME is the only variable -- so + // the generator emits one `ducklink_shim!("", _init_c_api);` line + // per managed extension (all share one cdylib; each copy is renamed + + // footer-stamped to `.duckdb_extension`). + // ----------------------------------------------------------------------- + + /// Shared shim entry. `name` is the logical extension; provider selection is + /// manifest-driven (`DUCKLINK_HOME/index.json`) via [`crate::passthrough::shim_load`]. + unsafe fn shim_entry( + name: &str, + info: ffi::duckdb_extension_info, + access: *const ffi::duckdb_extension_access, + ) -> bool { + match shim_entry_inner(name, info, access) { + Ok(loaded) => loaded, + Err(e) => { + if let Some(set_error) = (*access).set_error { + if let Ok(c) = CString::new(e.to_string()) { + set_error(info, c.as_ptr()); + } + } + false + } + } + } + + unsafe fn shim_entry_inner( + name: &str, + info: ffi::duckdb_extension_info, + access: *const ffi::duckdb_extension_access, + ) -> Result> { + let con = match open_connection(info, access)? { + Some(c) => c, + None => return Ok(false), + }; + let report = crate::passthrough::shim_load(&con, name).map_err(stringify)?; + eprintln!( + "[ducklink-shim:{name}] resolved provider '{}' [{}]; registered {} fn(s); reasoning: {}", + report.chosen_id, report.chosen_kind, report.registered, report.reasoning + ); + Ok(true) + } + + /// Common entrypoint boilerplate: init the C-API table, open a duckdb-rs + /// `Connection` on the database DuckDB handed us. `None` => abort cleanly. + unsafe fn open_connection( + info: ffi::duckdb_extension_info, + access: *const ffi::duckdb_extension_access, + ) -> Result, Box> { + if !ffi::duckdb_rs_extension_api_init(info, access, "v1.5.4").map_err(stringify)? { + return Ok(None); + } + let get_database = (*access) + .get_database + .ok_or_else(|| stringify("get_database is null in duckdb_extension_access"))?; + let db_ptr = get_database(info); + if db_ptr.is_null() { + return Ok(None); + } + let db: ffi::duckdb_database = *db_ptr; + Ok(Some(Connection::open_from_raw(db.cast())?)) + } + + /// Emit a `_init_c_api` shim entrypoint. The generator adds one line + /// per managed logical extension; the body is the shared, manifest-driven + /// [`shim_entry`]. + macro_rules! ducklink_shim { + ($name:literal, $init:ident) => { + // DuckLink shim entrypoint (auto-generated by `ducklink_shim!`). + // Safety: called by DuckDB during `LOAD` with a valid info/access pair. + #[no_mangle] + pub unsafe extern "C" fn $init( + info: ffi::duckdb_extension_info, + access: *const ffi::duckdb_extension_access, + ) -> bool { + shim_entry($name, info, access) + } + }; + } + + // ---- The managed-extension shim table (the generator owns this list) ---- + ducklink_shim!("aba", aba_init_c_api); + + // ----------------------------------------------------------------------- + // The NATIVE provider implementation for `aba` (a real native + // `.duckdb_extension`). Served as `aba_native.duckdb_extension` (exports + // `aba_native_init_c_api`); the shim's native arm `LOAD`s it. This is the + // native-speed passthrough: `aba_validate` computed in compiled Rust, no wasm. + // ----------------------------------------------------------------------- + + /// ABA routing-number checksum, native Rust (mirrors the wasm component's + /// semantics: 9 digits, weights 3,7,1,..., sum % 10 == 0; spaces/hyphens + /// ignored). The provider-neutral conformance suite certifies the two agree. + fn aba_checksum_valid(s: &str) -> bool { + let mut digits: Vec = Vec::with_capacity(s.len()); + for c in s.chars() { + if c.is_whitespace() || c == '-' { + continue; + } + match c.to_digit(10) { + Some(d) => digits.push(d), + None => return false, + } + } + if digits.len() != 9 { + return false; + } + let w = [3u32, 7, 1, 3, 7, 1, 3, 7, 1]; + let sum: u32 = digits.iter().zip(w).map(|(&d, k)| d * k).sum(); + sum % 10 == 0 + } + + struct AbaValidateNative; + + impl VScalar for AbaValidateNative { + type State = (); + fn invoke( + _: &Self::State, + input: &mut DataChunkHandle, + output: &mut dyn WritableVector, + ) -> Result<(), Box> { + let len = input.len(); + let in_vec = input.flat_vector(0); + let strs = unsafe { in_vec.as_slice_with_len::(len) }; + let mut out = output.flat_vector(); + { + let out_slice = unsafe { out.as_mut_slice_with_len::(len) }; + for i in 0..len { + // NULL propagation (the semantic contract: NULL in -> NULL out). + // Never read a NULL row's string_t (it may be garbage); write a + // placeholder, then set_null below. + out_slice[i] = if in_vec.row_is_null(i as u64) { + false + } else { + let mut t = strs[i]; + aba_checksum_valid(&DuckString::new(&mut t).as_str()) + }; + } + } + for i in 0..len { + if in_vec.row_is_null(i as u64) { + out.set_null(i); + } + } + Ok(()) + } + fn signatures() -> Vec { + vec![ScalarFunctionSignature::exact( + vec![LogicalTypeId::Varchar.into()], + LogicalTypeId::Boolean.into(), + )] + } + } + + /// NATIVE provider entrypoint for `aba` (file `aba_native.duckdb_extension`). + /// + /// # Safety + /// Called by DuckDB during `LOAD 'aba_native...'` with a valid pair. + #[no_mangle] + pub unsafe extern "C" fn aba_native_init_c_api( + info: ffi::duckdb_extension_info, + access: *const ffi::duckdb_extension_access, + ) -> bool { + match aba_native_init(info, access) { + Ok(b) => b, + Err(e) => { + if let Some(set_error) = (*access).set_error { + if let Ok(c) = CString::new(e.to_string()) { + set_error(info, c.as_ptr()); + } + } + false + } + } + } + + unsafe fn aba_native_init( + info: ffi::duckdb_extension_info, + access: *const ffi::duckdb_extension_access, + ) -> Result> { + let con = match open_connection(info, access)? { + Some(c) => c, + None => return Ok(false), + }; + con.register_scalar_function::("aba_validate") + .map_err(stringify)?; + eprintln!("[ducklink-native:aba] registered native aba_validate"); + Ok(true) + } } diff --git a/src/passthrough.rs b/src/passthrough.rs new file mode 100644 index 0000000..b011951 --- /dev/null +++ b/src/passthrough.rs @@ -0,0 +1,371 @@ +//! The native-passthrough hook (de-risk spike for design "D"). +//! +//! This is the seam that answers Q1/Q2 of the spike: a *function* the user +//! invokes (`ducklink_load('aba')`) that runs the lifted [`resolver`] and then +//! DUAL-LOADS the chosen provider through the matching arm: +//! +//! * **wasm** arm -> register the component's functions into the connection +//! via the proven Route-A bridge ([`crate::reg_duckdb::register_components`]), +//! embedded wasmtime, native speed. +//! * **native** arm -> hand off to DuckDB's OWN extension loader by issuing a +//! `LOAD ''` on the connection. This is the C-API-tractable form of +//! the native passthrough: the loaded ducklink extension drives DuckDB's +//! `dlopen` + `_init` path for the resolved `.duckdb_extension`. +//! * **remote** arm -> out of scope for this spike. +//! +//! Q1 finding (see the spike report): native DuckDB has no registerable hook that +//! can intercept a transparent `LOAD spatial` for *another* extension's name -- +//! `PhysicalLoad::GetDataInternal` calls `ExtensionHelper::LoadExternalExtension` +//! unconditionally and `OnBeginExtensionLoad` returns `void`. So the pragmatic +//! mechanism is this explicit entry point, fronted in production by a +//! `PRAGMA ducklink_load('aba')` / table-function `CALL` (which fires at a +//! statement boundary, where catalog mutation is safe). This module is that +//! entry point's body; the bundled test calls it at a statement boundary exactly +//! as the pragma would. + +use std::path::PathBuf; +use std::sync::{Arc, Mutex}; + +use anyhow::{anyhow, Result}; +use duckdb::Connection; + +use crate::engine::Engine2; +use crate::reg_duckdb::{register_components, ComponentSpec}; +use crate::resolver::{ + self, Conformance, ContentRef, Env, ManifestEntry, ProviderDescriptor, ProviderKind, + ResolvePolicy, +}; + +/// Outcome of one `ducklink_load` hook call (the observability the doc's +/// `PRAGMA extension_provider` would surface). +#[derive(Debug)] +pub struct LoadReport { + pub extension: String, + pub chosen_id: String, + pub chosen_kind: &'static str, + /// Per-candidate reasoning (the chosen + why each loser lost). + pub reasoning: String, + /// Functions registered (wasm arm); 0 for the native arm (DuckDB owns it). + pub registered: usize, +} + +/// The hook body: resolve `entry` under `env`/`policy`, then dual-load the chosen +/// provider through its arm. Reuses the resolver (A) verbatim and the Route-A +/// bridge (`register_components`) verbatim; the only new code is the kind->arm +/// dispatch below. +pub fn ducklink_load( + con: &Connection, + engine: Arc>, + entry: &ManifestEntry, + env: &Env, + policy: &ResolvePolicy, +) -> Result { + ducklink_load_gated(con, engine, entry, env, policy, None) +} + +/// As [`ducklink_load`], but with the conformance HARD GATE's canonical +/// suite-digest wired in: a non-reference provider is admitted only if its +/// `conformance.suite_digest` equals `canonical_suite_digest` (anti-forgery), on +/// top of passed + contract match. `None` skips the suite-digest sub-check. +pub fn ducklink_load_gated( + con: &Connection, + engine: Arc>, + entry: &ManifestEntry, + env: &Env, + policy: &ResolvePolicy, + canonical_suite_digest: Option<&str>, +) -> Result { + let res = resolver::resolve(entry, env, policy, canonical_suite_digest) + .map_err(|e| anyhow!(e.to_string()))?; + let reasoning = resolver::render_reasoning(&res.reasoning); + + let registered = match res.chosen_kind { + // WASM ARM: the Route-A bridge, reused unchanged. + "wasm" => { + let path = artifact_path(&res.artifact)?; + let spec = ComponentSpec { + name: res.extension.clone(), + path, + }; + register_components(con, None, engine, std::slice::from_ref(&spec))? + } + // NATIVE ARM: hand off to DuckDB's own native extension loader. + "native" => { + let path = artifact_path(&res.artifact)?; + con.execute_batch(&format!("LOAD '{}';", path.display())) + .map_err(|e| anyhow!("native arm: DuckDB LOAD of '{}' failed: {e}", path.display()))?; + 0 + } + other => return Err(anyhow!("provider kind '{other}' not implemented in this spike")), + }; + + Ok(LoadReport { + extension: res.extension, + chosen_id: res.chosen_id, + chosen_kind: res.chosen_kind, + reasoning, + registered, + }) +} + +fn artifact_path(artifact: &ContentRef) -> Result { + match artifact { + ContentRef::Path(p) => Ok(p.clone()), + // Spike: a remote/native entry may carry its location as an Oci string; + // treat it as a path for the local-file native arm. + ContentRef::Oci(s) => Ok(PathBuf::from(s)), + ContentRef::Digest(_) => { + Err(anyhow!("digest artifact resolution not wired in this spike")) + } + } +} + +/// Build a minimal in-code manifest entry for `aba`: a wasm reference provider +/// (the portable baseline) and, optionally, a native provider. In production this +/// comes from `registry/index.json` via `resolver::read_manifest_entry`; built +/// in-code here to keep the spike self-contained. +pub fn aba_manifest(wasm_artifact: PathBuf, native_artifact: Option) -> ManifestEntry { + const CONTRACT: &str = "spike-aba-contract"; + let mut providers = vec![ProviderDescriptor { + id: "wasm-component".into(), + kind: ProviderKind::Wasm { + abi: "duckdb:extension@2.0.0".into(), + artifact: ContentRef::Path(wasm_artifact), + content_digest: None, + browser_safe: false, + }, + reference: true, // certified-by-construction (the reference baseline) + conformance: None, + trust: None, + }]; + if let Some(p) = native_artifact { + providers.push(ProviderDescriptor { + id: "native-local".into(), + kind: ProviderKind::Native { + os: std::env::consts::OS.into(), + arch: std::env::consts::ARCH.into(), + artifact: ContentRef::Path(p), + }, + reference: false, + // Certified at the live contract so the hard gate admits it. + conformance: Some(Conformance { + suite: "aba@1".into(), + suite_digest: String::new(), + contract_digest: CONTRACT.into(), + passed: true, + }), + trust: None, + }); + } + ManifestEntry { + name: "aba".into(), + wit_contract: CONTRACT.into(), + providers, + } +} + +// --------------------------------------------------------------------------- +// The productionized SHIM body (manifest-driven; design "D" first-cut). +// +// A shim named `.duckdb_extension` (exporting `_init_c_api`) calls +// `shim_load(con, "")`. It is the same for every logical extension -- the +// NAME is the only variable -- so one body + one macro generate every shim. +// +// Provider selection comes from the MULTI-PROVIDER MANIFEST (the generalized +// `registry/index.json` `providers[]`), NOT from an env var. The manifest + +// suite live under `DUCKLINK_HOME` (the DuckLink install dir): +// $DUCKLINK_HOME/index.json +// $DUCKLINK_HOME/ +// $DUCKLINK_HOME/suites/.sql (the conformance suite; its content +// digest is the canonical suite_digest) +// +// Policy OVERRIDES are env-driven on STOCK duckdb, because the stable C +// extension API has no runtime-setting registration (no `duckdb_add_extension_ +// option`; only open-time `duckdb_set_config`), so `SET extension_provider=...` +// cannot be honored by a C-API shim. The equivalents: +// DUCKLINK_ALLOW_NATIVE=0 -> deny the native arm (== SET allow_native_providers=false) +// DUCKLINK_PROVIDER= -> force a provider id (== SET extension_provider=) +// DUCKLINK_DENY=, -> exclude provider ids +// (See the report: `SET`-based overrides need a C++ base extension / +// DuckDB exposing add_extension_option in the C API -- roadmap.) +// --------------------------------------------------------------------------- + +/// The shared, manifest-driven shim body. Reads `DUCKLINK_HOME/index.json`, +/// resolves the best certified provider for `name`, and dual-loads it. +pub fn shim_load(con: &Connection, name: &str) -> Result { + let home = std::env::var("DUCKLINK_HOME") + .map(PathBuf::from) + .map_err(|_| anyhow!("DUCKLINK_HOME not set (the DuckLink install dir with index.json)"))?; + + let index_path = home.join("index.json"); + let index_bytes = std::fs::read(&index_path) + .map_err(|e| anyhow!("reading {}: {e}", index_path.display()))?; + let index: serde_json::Value = serde_json::from_slice(&index_bytes) + .map_err(|e| anyhow!("parsing {}: {e}", index_path.display()))?; + + let mut entry = resolver::read_manifest_entry(&index, name) + .ok_or_else(|| anyhow!("no manifest entry for '{name}' in {}", index_path.display()))?; + absolutize_artifacts(&mut entry, &home); + + // Canonical suite digest = content digest of the suite the resolver holds. + let canonical = canonical_suite_digest(&home, name); + + let env = env_from_environment(); + let policy = policy_from_environment(); + let engine = Arc::new(Mutex::new(Engine2::new()?)); + ducklink_load_gated(con, engine, &entry, &env, &policy, canonical.as_deref()) +} + +/// Rewrite each provider's artifact (relative in the manifest) to an absolute +/// path under `home`. Leaves remote URLs (`oci://`, `http`) untouched. +fn absolutize_artifacts(entry: &mut ManifestEntry, home: &std::path::Path) { + fn fix(r: &ContentRef, home: &std::path::Path) -> ContentRef { + match r { + ContentRef::Path(p) if p.is_relative() => ContentRef::Path(home.join(p)), + ContentRef::Oci(s) if !s.contains("://") => { + ContentRef::Oci(home.join(s).to_string_lossy().into_owned()) + } + other => other.clone(), + } + } + for p in &mut entry.providers { + match &mut p.kind { + ProviderKind::Wasm { artifact, .. } => *artifact = fix(artifact, home), + ProviderKind::Native { artifact, .. } => *artifact = fix(artifact, home), + ProviderKind::Remote { .. } => {} + } + } +} + +/// Canonical conformance-suite digest for `name`, byte-identical to build C's +/// `resolver::compute_suite_digest` / `tooling/conformance.py`: a structured +/// digest over `$home/suites/.sql` + `.expected`. `None` if absent +/// -> the gate's suite-digest sub-check is skipped (passed + contract still +/// enforced). The shim and the host MUST agree on this scheme or certified +/// records would be rejected as stale. +fn canonical_suite_digest(home: &std::path::Path, name: &str) -> Option { + let dir = home.join("suites"); + let sql = std::fs::read_to_string(dir.join(format!("{name}.sql"))).ok()?; + let expected = std::fs::read_to_string(dir.join(format!("{name}.expected"))).ok()?; + Some(resolver::compute_suite_digest(&sql, &expected)) +} + +/// Native arm allowed unless `DUCKLINK_ALLOW_NATIVE=0`. Default ON -- the native +/// passthrough is the headline: plain `LOAD aba` prefers native (precedence). +fn env_from_environment() -> Env { + let allow_native = std::env::var("DUCKLINK_ALLOW_NATIVE") + .map(|v| v != "0" && !v.eq_ignore_ascii_case("false")) + .unwrap_or(true); + Env { + wasm_runtime: true, + allow_native, + } +} + +fn policy_from_environment() -> ResolvePolicy { + let forced_provider = std::env::var("DUCKLINK_PROVIDER").ok().filter(|s| !s.is_empty()); + let denied = std::env::var("DUCKLINK_DENY") + .ok() + .map(|s| s.split(',').filter(|x| !x.is_empty()).map(|x| x.to_string()).collect()) + .unwrap_or_default(); + ResolvePolicy { + forced_provider, + denied, + } +} + +#[cfg(all(test, feature = "bundled"))] +mod tests { + use super::*; + + fn aba_wasm() -> PathBuf { + PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("../../artifacts/extensions/aba.wasm") + } + + /// Q2 WASM-ARM PROOF (end-to-end through the hook): `ducklink_load('aba')` + /// resolves the wasm reference provider, registers `aba_validate` via the + /// Route-A bridge, and the function then computes the ABA checksum INSIDE the + /// wasm component for a subsequent query. + #[test] + fn wasm_arm_dual_loads_through_hook() { + let engine = Arc::new(Mutex::new(Engine2::new().expect("engine"))); + let con = Connection::open_in_memory().expect("open duckdb"); + + // wasm-only manifest, default env (allow_native = false). + let entry = aba_manifest(aba_wasm(), None); + let report = ducklink_load( + &con, + engine, + &entry, + &Env::default(), + &ResolvePolicy::default(), + ) + .expect("hook loads"); + + assert_eq!(report.chosen_kind, "wasm", "reasoning: {}", report.reasoning); + assert_eq!(report.chosen_id, "wasm-component"); + assert!(report.registered >= 1, "expected aba_validate registered"); + eprintln!("[spike] wasm arm: {report:?}"); + + // The hook registered the function; a normal query now dispatches into wasm. + let valid: bool = con + .query_row("SELECT aba_validate('021000021')", [], |r| r.get(0)) + .expect("query valid"); + assert!(valid, "021000021 is a valid ABA routing number (checksum 0)"); + + let invalid: bool = con + .query_row("SELECT aba_validate('123456789')", [], |r| r.get(0)) + .expect("query invalid"); + assert!(!invalid, "123456789 fails the ABA checksum"); + } + + /// Q2 NATIVE-ARM PROOF (mechanism, through the hook): with a native provider + /// present and `allow_native = true`, the resolver PICKS native (precedence + /// native > wasm) and the hook hands off to DuckDB's own `LOAD` -- proving the + /// native arm reaches DuckDB's dlopen/init substrate. We point it at a missing + /// artifact, so the error originates from DuckDB's native loader (full e2e + /// needs a built+signed native `aba.duckdb_extension`; see the report). + #[test] + fn native_arm_is_chosen_and_hands_off_to_duckdb_load() { + let engine = Arc::new(Mutex::new(Engine2::new().expect("engine"))); + let con = Connection::open_in_memory().expect("open duckdb"); + + let bogus = PathBuf::from("/nonexistent/aba.duckdb_extension"); + let entry = aba_manifest(aba_wasm(), Some(bogus.clone())); + let env = Env { + wasm_runtime: true, + allow_native: true, + }; + + let err = ducklink_load(&con, engine, &entry, &env, &ResolvePolicy::default()) + .expect_err("native arm should attempt DuckDB LOAD of a missing artifact"); + let msg = err.to_string(); + eprintln!("[spike] native arm error (expected): {msg}"); + // Proof the NATIVE arm was selected and dispatched to DuckDB's loader. + assert!( + msg.contains("native arm") && msg.contains("LOAD"), + "expected native-arm LOAD handoff, got: {msg}" + ); + } + + /// The native arm is gated off by default (allow_native = false): the SAME + /// manifest resolves to the wasm baseline -- graceful degradation, not silent + /// native load. + #[test] + fn native_off_by_default_falls_back_to_wasm() { + let engine = Arc::new(Mutex::new(Engine2::new().expect("engine"))); + let con = Connection::open_in_memory().expect("open duckdb"); + let entry = aba_manifest(aba_wasm(), Some(PathBuf::from("/nonexistent/aba.duckdb_extension"))); + + let report = ducklink_load( + &con, + engine, + &entry, + &Env::default(), // allow_native = false + &ResolvePolicy::default(), + ) + .expect("falls back to wasm"); + assert_eq!(report.chosen_kind, "wasm", "reasoning: {}", report.reasoning); + assert!(report.reasoning.contains("native-local"), "loser recorded"); + } +} diff --git a/src/reg_duckdb.rs b/src/reg_duckdb.rs index 70bef3f..78e00d6 100644 --- a/src/reg_duckdb.rs +++ b/src/reg_duckdb.rs @@ -754,6 +754,177 @@ impl VTab for WasmTable { } } +/// Map a physical-type string (from the guest's LogicalTypeReg.physical) to a +/// DuckDB C-API type code. The physical is the storage-level type; the alias +/// is a bind-time name that resolves through it. +fn duckdb_type_of_physical(physical: &str) -> ffi::duckdb_type { + match physical.to_ascii_uppercase().as_str() { + "BLOB" => duckdb_type_of(T_BLOB), + "TEXT" | "VARCHAR" => duckdb_type_of(T_TEXT), + "INT64" | "BIGINT" => duckdb_type_of(T_I64), + "INT32" | "INTEGER" => duckdb_type_of(T_I32), + "FLOAT64" | "DOUBLE" => duckdb_type_of(T_F64), + "FLOAT32" | "REAL" => duckdb_type_of(T_F32), + "BOOLEAN" | "BOOL" => duckdb_type_of(T_BOOL), + _ => { + // Unknown physical types fall back to BLOB — safe: opaque bytes + // survive round-trip. Follow-up: expand this table when new WIT + // Logicaltype variants land. + duckdb_type_of(T_BLOB) + } + } +} + +/// #79 — Register logical-type aliases at DuckDB via the C-API. For each +/// LogicalTypeAlias: +/// 1. `duckdb_create_logical_type(physical)` -> handle +/// 2. `duckdb_logical_type_set_alias(handle, name)` +/// 3. `duckdb_register_logical_type(con, handle, null)` — registers the +/// alias on the connection so subsequent `SELECT ... ::GEOMETRY` binds +/// to the handle instead of failing at type-resolution. +/// 4. `duckdb_destroy_logical_type` — the C-API copies internally. +/// +/// After this pass, a subsequent scalar registered as `st_area(BLOB)` accepts +/// arguments cast as GEOMETRY because the alias resolves to BLOB physically. +/// +/// # Safety +/// `raw_con` must be a valid `duckdb_connection`. +pub unsafe fn register_logical_types( + raw_con: ffi::duckdb_connection, + logical_types: &[super::engine::LogicalTypeAlias], +) -> duckdb::Result { + let mut registered = 0usize; + for lt in logical_types { + let physical = duckdb_type_of_physical(<.physical); + let mut handle = ffi::duckdb_create_logical_type(physical); + if handle.is_null() { + eprintln!( + "[reg_duckdb] logical_type: create_logical_type returned null for ext='{}' name='{}' physical='{}' — skipping", + lt.extension, lt.name, lt.physical + ); + continue; + } + let cname = CString::new(lt.name.as_str()) + .map_err(|_| duckdb::Error::DuckDBFailure(ffi::Error::new(ffi::DuckDBError), None))?; + ffi::duckdb_logical_type_set_alias(handle, cname.as_ptr()); + // `info` param is opaque/optional per the C-API docs — null is accepted. + let rc = ffi::duckdb_register_logical_type(raw_con, handle, std::ptr::null_mut()); + ffi::duckdb_destroy_logical_type(&mut handle); + if rc != ffi::duckdb_state_DuckDBSuccess { + eprintln!( + "[reg_duckdb] logical_type: register_logical_type failed (rc={}) for ext='{}' name='{}' — likely already registered", + rc, lt.extension, lt.name + ); + continue; + } + eprintln!( + "[reg_duckdb] logical_type: registered ext='{}' name='{}' physical='{}'", + lt.extension, lt.name, lt.physical + ); + registered += 1; + } + Ok(registered) +} + +/// #79 — Identity cast callback. GEOMETRY↔BLOB (and similar same-physical +/// aliases) are byte-identical: the cast is a straight per-row copy of the +/// BLOB payload. DuckDB's cast API does NOT auto-populate the output vector +/// when the callback returns true — a "no-op" trampoline leaves `output` +/// uninitialised, which surfaces as an empty "Conversion Error" at query +/// time (that was the initial buggy impl documented in G3). Iterate rows +/// explicitly and copy validity + BLOB bytes via the same primitives the +/// scalar-return path uses (`duckdb_vector_assign_string_element_len`). +/// +/// Scope: this trampoline is only registered for BLOB↔BLOB aliases (the +/// GEOMETRY case). Extending to other same-physical-type aliases would +/// need a per-physical-type branch here — for now the register_casts +/// caller only produces BLOB pairs. +unsafe extern "C" fn identity_cast_trampoline( + _info: ffi::duckdb_function_info, + row_count: u64, + input: ffi::duckdb_vector, + output: ffi::duckdb_vector, +) -> bool { + let in_validity = ffi::duckdb_vector_get_validity(input); + let in_data = ffi::duckdb_vector_get_data(input) as *const duckdb_string_t; + for i in 0..(row_count as usize) { + let is_valid = in_validity.is_null() + || ffi::duckdb_validity_row_is_valid(in_validity, i as u64); + if !is_valid { + ffi::duckdb_vector_ensure_validity_writable(output); + let out_validity = ffi::duckdb_vector_get_validity(output); + ffi::duckdb_validity_set_row_validity(out_validity, i as u64, false); + continue; + } + let mut s = *in_data.add(i); + let bytes = DuckString::new(&mut s).as_bytes(); + ffi::duckdb_vector_assign_string_element_len( + output, + i as u64, + bytes.as_ptr() as *const c_char, + bytes.len() as u64, + ); + } + true +} + +/// #79 — Register implicit casts between aliased and underlying types. For a +/// GEOMETRY↔BLOB alias, this makes `st_area(GEOMETRY)` and `st_area(BLOB)` +/// interchangeable at the binder level. +/// +/// # Safety +/// `raw_con` must be a valid `duckdb_connection`. Every `source` and `target` +/// name must resolve to a registered logical type (via `register_logical_types` +/// or as a DuckDB built-in). +pub unsafe fn register_casts( + raw_con: ffi::duckdb_connection, + casts: &[super::engine::CastReg], +) -> duckdb::Result { + let mut registered = 0usize; + for c in casts { + // Both source and target resolve through the same physical-type map. + // For GEOMETRY↔BLOB this is BLOB on both sides, and the identity + // trampoline handles the "cast". + let mut source_ty = ffi::duckdb_create_logical_type(duckdb_type_of_physical(&c.source)); + let mut target_ty = ffi::duckdb_create_logical_type(duckdb_type_of_physical(&c.target)); + // Restore the alias for lookup — DuckDB matches casts by (source, + // target) logical-type identity, which includes the alias if set. + if let Ok(cname) = CString::new(c.source.as_str()) { + ffi::duckdb_logical_type_set_alias(source_ty, cname.as_ptr()); + } + if let Ok(cname) = CString::new(c.target.as_str()) { + ffi::duckdb_logical_type_set_alias(target_ty, cname.as_ptr()); + } + let cast = ffi::duckdb_create_cast_function(); + ffi::duckdb_cast_function_set_source_type(cast, source_ty); + ffi::duckdb_cast_function_set_target_type(cast, target_ty); + // Cost 0 => cheapest implicit cast; the binder prefers it over the + // built-in explicit-cast lookup so alias→physical resolution wins. + ffi::duckdb_cast_function_set_implicit_cast_cost(cast, 0); + ffi::duckdb_cast_function_set_function(cast, Some(identity_cast_trampoline)); + let rc = ffi::duckdb_register_cast_function(raw_con, cast); + // The FFI docs say we own the cast function object; destroy it after + // register_cast_function copies whatever it needs. + let mut cast_owned = cast; + ffi::duckdb_destroy_cast_function(&mut cast_owned); + ffi::duckdb_destroy_logical_type(&mut source_ty); + ffi::duckdb_destroy_logical_type(&mut target_ty); + if rc != ffi::duckdb_state_DuckDBSuccess { + eprintln!( + "[reg_duckdb] cast: register_cast_function failed (rc={}) for ext='{}' source='{}' target='{}' — likely already registered", + rc, c.extension, c.source, c.target + ); + continue; + } + eprintln!( + "[reg_duckdb] cast: registered ext='{}' source='{}' target='{}' (implicit-cost=0)", + c.extension, c.source, c.target + ); + registered += 1; + } + Ok(registered) +} + /// Register every component table function on `con`. Returns the count /// registered. Parameter and column types use the same `reg` logical-type set as /// scalars. @@ -1198,6 +1369,25 @@ pub fn register_components( let mut e = engine.lock().expect("engine mutex poisoned"); e.load(&spec.name, &spec.path)? }; + // #79 — register logical-type aliases BEFORE scalars so the binder + // sees GEOMETRY/GEOGRAPHY/... as valid types when the scalars register + // their arg-type expectations. Casts follow, giving the binder an + // implicit path between the alias and its physical type. + if let Some(rc) = raw_con { + if !loaded.logical_types.is_empty() { + total += unsafe { register_logical_types(rc, &loaded.logical_types)? }; + } + if !loaded.casts.is_empty() { + total += unsafe { register_casts(rc, &loaded.casts)? }; + } + } else if !loaded.logical_types.is_empty() || !loaded.casts.is_empty() { + eprintln!( + "[ducklink] skipping {} logical_type + {} cast registration(s) from '{}': no raw connection available", + loaded.logical_types.len(), + loaded.casts.len(), + spec.name + ); + } total += register_scalars(con, engine.clone(), &loaded.scalars)?; total += register_tables(con, engine.clone(), &loaded.tables)?; match raw_con { diff --git a/src/resolver.rs b/src/resolver.rs new file mode 100644 index 0000000..6adf303 --- /dev/null +++ b/src/resolver.rs @@ -0,0 +1,730 @@ +//! Multi-provider extension resolver — the spine (design A) from +//! PLAN-multi-provider-extensions.md ("The Resolver" + Appendix B). +//! +//! A logical extension has one semantic contract (`wit_contract`) and one or +//! more `providers[]`, each an implementation of that contract on some substrate +//! (wasm / native / remote). This module is the substrate-agnostic policy + +//! candidate-filtering spine over those providers. It is deliberately +//! self-contained (no wasmtime / no engine types) so it lifts cleanly into +//! `datalink` later; the host injects the concrete wasm `load()` via a callback. +//! +//! This pass implements the Wasm kind only. The Wasm `load()` IS Route A's +//! resident `duckdb:extension` dispatch (datalink-dynlink + register-capture + +//! the direct `callback-dispatch` import) — wrapped, not reimplemented. Native +//! and Remote are stubbed variants that are filtered out as unavailable. + +use std::path::PathBuf; + +// --------------------------------------------------------------------------- +// B.2 types +// --------------------------------------------------------------------------- + +/// Content-addressed artifact reference. +#[derive(Clone, Debug, PartialEq, Eq)] +pub enum ContentRef { + Path(PathBuf), + Digest(String), + Oci(String), +} + +/// Substrate of one provider. Only `Wasm` is implemented this pass; `Native` and +/// `Remote` are scaffolded so the filtering pipeline is real (they resolve as +/// unavailable). +#[derive(Clone, Debug, PartialEq, Eq)] +pub enum ProviderKind { + Wasm { + abi: String, + artifact: ContentRef, + content_digest: Option, + browser_safe: bool, + }, + Native { + os: String, + arch: String, + artifact: ContentRef, + }, + Remote { + endpoint: String, + }, +} + +impl ProviderKind { + pub fn tag(&self) -> &'static str { + match self { + ProviderKind::Wasm { .. } => "wasm", + ProviderKind::Native { .. } => "native", + ProviderKind::Remote { .. } => "remote", + } + } +} + +/// The SEMANTIC-contract certificate carried by a provider entry. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct Conformance { + pub suite: String, + /// Content digest of the suite itself; must equal the canonical suite the + /// resolver holds for (ext, contract). + pub suite_digest: String, + /// Must equal the logical extension's `wit_contract`. + pub contract_digest: String, + pub passed: bool, +} + +/// Native/remote admission inputs (unused for wasm, which is sandboxed). +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct Trust { + pub signed_by: Option, + pub attestation: Option, +} + +/// Static descriptor parsed from one manifest provider entry. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct ProviderDescriptor { + pub id: String, + pub kind: ProviderKind, + /// Defines semantics (the wasm baseline). A reference wasm provider is + /// certified-by-construction at the entry's contract. + pub reference: bool, + pub conformance: Option, + pub trust: Option, +} + +/// One logical extension entry: identity + semantic contract + providers. +#[derive(Clone, Debug)] +pub struct ManifestEntry { + pub name: String, + /// THE semantic_contract digest (witcanon). + pub wit_contract: String, + pub providers: Vec, +} + +/// Environment inputs to availability/precedence. +#[derive(Clone, Debug)] +pub struct Env { + /// A wasm runtime is present in-process (always true for the ducklink host). + pub wasm_runtime: bool, + /// Native `.duckdb_extension` loading allowed (this pass: always false). + pub allow_native: bool, +} + +impl Default for Env { + fn default() -> Self { + Self { + wasm_runtime: true, + allow_native: false, + } + } +} + +/// Resolution policy (the overridable knobs from the doc). +#[derive(Clone, Debug, Default)] +pub struct ResolvePolicy { + /// `SET extension_provider = ''` — force a specific provider id. + pub forced_provider: Option, + /// `SET extension_provider_deny = ','` — user-excluded providers. + pub denied: Vec, +} + +// --------------------------------------------------------------------------- +// Candidate pipeline outcome (observability) +// --------------------------------------------------------------------------- + +#[derive(Clone, Debug, PartialEq, Eq)] +pub enum Outcome { + Chosen, + Rejected(String), +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct CandidateOutcome { + pub id: String, + pub kind: &'static str, + pub outcome: Outcome, +} + +#[derive(Clone, Debug)] +pub struct Resolution { + pub extension: String, + pub chosen_id: String, + pub chosen_kind: &'static str, + /// The resolved artifact of the chosen provider (for the substrate loader). + pub artifact: ContentRef, + /// Per-candidate outcome, in evaluation order (the chosen + why each loser lost). + pub reasoning: Vec, +} + +#[derive(Clone, Debug)] +pub struct ResolveError { + pub extension: String, + pub reasoning: Vec, +} + +impl std::fmt::Display for ResolveError { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!( + f, + "no admissible provider for '{}': {}", + self.extension, + render_reasoning(&self.reasoning) + ) + } +} + +impl std::error::Error for ResolveError {} + +/// Render per-candidate reasoning as a single line (used by the error + the +/// `extension_provider` observability function). +pub fn render_reasoning(reasoning: &[CandidateOutcome]) -> String { + reasoning + .iter() + .map(|c| match &c.outcome { + Outcome::Chosen => format!("{} [{}] = CHOSEN", c.id, c.kind), + Outcome::Rejected(why) => format!("{} [{}] = rejected ({})", c.id, c.kind, why), + }) + .collect::>() + .join("; ") +} + +// --------------------------------------------------------------------------- +// Conformance suite content digest (the canonical suite_digest the gate checks) +// +// BYTE-IDENTICAL to the monorepo `crates/ducklink-host/src/resolver.rs:: +// compute_suite_digest` and `tooling/conformance.py` (build C): a structured +// digest over the NORMALIZED conformance.sql + conformance.expected so cosmetic +// edits don't churn it but any executable/expected change does. The shim and the +// host MUST agree, or certified records would be rejected as stale. +// +// sha256( b"duckdb:conformance-suite:1\n" +// || normalize_sql(conformance.sql) +// || b"\n\x1e\n" +// || normalize_expected(conformance.expected) ) +// --------------------------------------------------------------------------- + +/// Compute the canonical suite digest from the suite's `.sql` + `.expected`. +pub fn compute_suite_digest(sql: &str, expected: &str) -> String { + use sha2::{Digest, Sha256}; + let mut canon: Vec = Vec::new(); + canon.extend_from_slice(b"duckdb:conformance-suite:1\n"); + canon.extend_from_slice(normalize_suite_sql(sql).as_bytes()); + canon.extend_from_slice(b"\n\x1e\n"); + canon.extend_from_slice(normalize_suite_expected(expected).as_bytes()); + let mut h = Sha256::new(); + h.update(&canon); + h.finalize().iter().map(|b| format!("{b:02x}")).collect() +} + +/// Canonical SQL: drop blank lines and `--` comment lines, rstrip the rest. +fn normalize_suite_sql(sql: &str) -> String { + sql.lines() + .map(str::trim_end) + .filter(|l| !l.trim().is_empty()) + .filter(|l| !l.trim_start().starts_with("--")) + .collect::>() + .join("\n") +} + +/// Canonical expected: drop blank lines and `#`/`# ` comment lines, rstrip. +fn normalize_suite_expected(expected: &str) -> String { + expected + .lines() + .map(str::trim_end) + .filter(|l| !l.trim().is_empty()) + .filter(|l| { + let ls = l.trim_start(); + !(ls == "#" || ls.starts_with("# ")) + }) + .collect::>() + .join("\n") +} + +// --------------------------------------------------------------------------- +// The resolver +// --------------------------------------------------------------------------- + +/// Precedence rank (lower = preferred): native(trusted) > wasm-local > +/// wasm-browser > remote. +fn precedence_rank(kind: &ProviderKind) -> u8 { + match kind { + ProviderKind::Native { .. } => 0, + ProviderKind::Wasm { browser_safe: false, .. } => 1, + ProviderKind::Wasm { browser_safe: true, .. } => 2, + ProviderKind::Remote { .. } => 3, + } +} + +/// The conformance HARD GATE. A provider is certified iff: +/// passed && contract_digest == wit_contract && suite_digest == canonical. +/// A reference wasm provider is certified-by-construction (the baseline WIT *is* +/// the contract that defines the semantics). +/// +/// `canonical_suite_digest` is the suite the resolver holds for (ext, contract); +/// `None` means no suite is registered yet, so the suite_digest sub-check is +/// scaffolded (skipped) — the gate still enforces passed + contract match. +fn conformance_ok( + p: &ProviderDescriptor, + wit_contract: &str, + canonical_suite_digest: Option<&str>, +) -> Result<(), String> { + if p.reference && matches!(p.kind, ProviderKind::Wasm { .. }) { + return Ok(()); + } + match &p.conformance { + None => Err("uncertified: no conformance record".to_string()), + Some(c) => { + if !c.passed { + return Err("uncertified: conformance.passed=false".to_string()); + } + if c.contract_digest != wit_contract { + return Err(format!( + "uncertified: certified at contract {} != live {}", + short(&c.contract_digest), + short(wit_contract) + )); + } + if let Some(canon) = canonical_suite_digest { + if c.suite_digest != canon { + return Err(format!( + "uncertified: suite_digest {} != canonical {}", + short(&c.suite_digest), + short(canon) + )); + } + } + Ok(()) + } + } +} + +fn substrate_available(kind: &ProviderKind, env: &Env) -> Result<(), String> { + match kind { + ProviderKind::Wasm { .. } => { + if env.wasm_runtime { + Ok(()) + } else { + Err("no wasm runtime".to_string()) + } + } + ProviderKind::Native { os: _, arch: _, .. } => { + // SPIKE (design "D"): the native arm IS implemented, in the passthrough + // hook (it hands off to DuckDB's own `LOAD`). So native is "available" + // whenever the trust policy permits it. Production should additionally + // verify platform match + that the artifact dlopen-loads. + if !env.allow_native { + Err("native providers disabled (allow_native=false)".to_string()) + } else { + Ok(()) + } + } + ProviderKind::Remote { endpoint } => { + Err(format!("remote loader not implemented ({endpoint})")) + } + } +} + +fn trusted(kind: &ProviderKind) -> Result<(), String> { + match kind { + // wasm is sandboxed -> always trusted. + ProviderKind::Wasm { .. } => Ok(()), + // SPIKE (design "D"): trust-all for native so the arm can be exercised. + // Production gates this on signature/attestation (datalink-contract / + // std:attest) + `SET allow_native_providers`. + ProviderKind::Native { .. } => Ok(()), + ProviderKind::Remote { .. } => Err("remote trust policy not implemented".to_string()), + } +} + +/// Run the candidate pipeline and pick a provider, or fail with per-candidate +/// reasons. The pipeline order mirrors the doc: +/// conformance gate -> available -> trusted -> !user_excluded -> !forced_out +/// -> order by precedence -> first. +pub fn resolve( + entry: &ManifestEntry, + env: &Env, + policy: &ResolvePolicy, + canonical_suite_digest: Option<&str>, +) -> Result { + let mut reasoning: Vec = Vec::new(); + let mut admitted: Vec<&ProviderDescriptor> = Vec::new(); + + for p in &entry.providers { + let kind = p.kind.tag(); + // 1. conformance HARD GATE + if let Err(why) = conformance_ok(p, &entry.wit_contract, canonical_suite_digest) { + reasoning.push(reject(&p.id, kind, why)); + continue; + } + // 2. substrate available + if let Err(why) = substrate_available(&p.kind, env) { + reasoning.push(reject(&p.id, kind, why)); + continue; + } + // 3. trusted + if let Err(why) = trusted(&p.kind) { + reasoning.push(reject(&p.id, kind, why)); + continue; + } + // 4. not user-excluded + if policy.denied.iter().any(|d| d == &p.id) { + reasoning.push(reject(&p.id, kind, "user-excluded (deny)".to_string())); + continue; + } + // 5. forced-provider override: if set, only that id survives + if let Some(forced) = &policy.forced_provider { + if forced != &p.id { + reasoning.push(reject( + &p.id, + kind, + format!("not the forced provider ('{forced}')"), + )); + continue; + } + } + admitted.push(p); + } + + // 6. order by precedence (stable; manifest order breaks ties) + admitted.sort_by_key(|p| precedence_rank(&p.kind)); + + match admitted.first() { + Some(chosen) => { + // Record the chosen + keep the losers' reasons already collected. + reasoning.insert( + 0, + CandidateOutcome { + id: chosen.id.clone(), + kind: chosen.kind.tag(), + outcome: Outcome::Chosen, + }, + ); + // Any other admitted providers lost on precedence. + for other in admitted.iter().skip(1) { + reasoning.push(reject( + &other.id, + other.kind.tag(), + "lost on precedence".to_string(), + )); + } + let artifact = match &chosen.kind { + ProviderKind::Wasm { artifact, .. } => artifact.clone(), + ProviderKind::Native { artifact, .. } => artifact.clone(), + ProviderKind::Remote { endpoint } => ContentRef::Oci(endpoint.clone()), + }; + Ok(Resolution { + extension: entry.name.clone(), + chosen_id: chosen.id.clone(), + chosen_kind: chosen.kind.tag(), + artifact, + reasoning, + }) + } + None => Err(ResolveError { + extension: entry.name.clone(), + reasoning, + }), + } +} + +fn reject(id: &str, kind: &'static str, why: String) -> CandidateOutcome { + CandidateOutcome { + id: id.to_string(), + kind, + outcome: Outcome::Rejected(why), + } +} + +fn short(digest: &str) -> String { + digest.chars().take(12).collect() +} + +// --------------------------------------------------------------------------- +// B.1 manifest reader (providers[] + backward-compat single-artifact) +// --------------------------------------------------------------------------- + +/// Read one extension's manifest entry from a parsed `registry/index.json` +/// value. Returns `None` if the extension is absent or has no usable artifact. +/// +/// Backward compatibility: a current single-artifact entry (just `artifact` + +/// `content_digest` + `wit_contract`) is read as a one-element `providers[]` with +/// `kind:"wasm", reference:true`, lifting those fields verbatim. An explicit +/// `providers[]` array (the generalized shape) is read directly. +pub fn read_manifest_entry(index: &serde_json::Value, name: &str) -> Option { + let exts = index.get("extensions")?.as_array()?; + let entry = exts + .iter() + .find(|e| e.get("name").and_then(|v| v.as_str()) == Some(name))?; + + let wit_contract = entry + .get("wit_contract") + .and_then(|v| v.as_str()) + .unwrap_or("") + .to_string(); + let abi = format!( + "duckdb:extension@{}", + entry + .get("wit_contract_version") + .and_then(|v| v.as_str()) + .unwrap_or("2.0.0") + ); + + let mut providers = Vec::new(); + if let Some(arr) = entry.get("providers").and_then(|v| v.as_array()) { + // Generalized shape. + for p in arr { + if let Some(desc) = parse_provider(p, &abi) { + providers.push(desc); + } + } + } else if let Some(artifact) = entry.get("artifact").and_then(|v| v.as_str()) { + // Backward-compat: single artifact -> one wasm reference provider. + providers.push(ProviderDescriptor { + id: "wasm-component".to_string(), + kind: ProviderKind::Wasm { + abi: abi.clone(), + artifact: ContentRef::Path(PathBuf::from(artifact)), + content_digest: entry + .get("content_digest") + .and_then(|v| v.as_str()) + .map(|s| s.to_string()), + browser_safe: false, + }, + reference: true, + conformance: None, // reference wasm: certified-by-construction + trust: None, + }); + } + + if providers.is_empty() { + return None; + } + Some(ManifestEntry { + name: name.to_string(), + wit_contract, + providers, + }) +} + +fn parse_provider(p: &serde_json::Value, default_abi: &str) -> Option { + let id = p.get("id").and_then(|v| v.as_str())?.to_string(); + let kind_tag = p.get("kind").and_then(|v| v.as_str()).unwrap_or("wasm"); + let reference = p.get("reference").and_then(|v| v.as_bool()).unwrap_or(false); + + let kind = match kind_tag { + "wasm" => { + let artifact = p.get("artifact").and_then(|v| v.as_str())?; + ProviderKind::Wasm { + abi: p + .get("abi") + .and_then(|v| v.as_str()) + .unwrap_or(default_abi) + .to_string(), + artifact: ContentRef::Path(PathBuf::from(artifact)), + content_digest: p + .get("content_digest") + .and_then(|v| v.as_str()) + .map(|s| s.to_string()), + browser_safe: p + .get("browser_safe") + .and_then(|v| v.as_bool()) + .unwrap_or(false), + } + } + "native" => { + let plat = p.get("platform"); + ProviderKind::Native { + os: plat + .and_then(|v| v.get("os")) + .and_then(|v| v.as_str()) + .unwrap_or("") + .to_string(), + arch: plat + .and_then(|v| v.get("arch")) + .and_then(|v| v.as_str()) + .unwrap_or("") + .to_string(), + artifact: ContentRef::Oci( + p.get("artifact") + .and_then(|v| v.as_str()) + .unwrap_or("") + .to_string(), + ), + } + } + "remote" => ProviderKind::Remote { + endpoint: p + .get("endpoint") + .and_then(|v| v.as_str()) + .unwrap_or("") + .to_string(), + }, + _ => return None, + }; + + let conformance = p.get("conformance").map(|c| Conformance { + suite: c.get("suite").and_then(|v| v.as_str()).unwrap_or("").to_string(), + suite_digest: c + .get("suite_digest") + .and_then(|v| v.as_str()) + .unwrap_or("") + .to_string(), + contract_digest: c.get("at").and_then(|v| v.as_str()).unwrap_or("").to_string(), + passed: c.get("passed").and_then(|v| v.as_bool()).unwrap_or(false), + }); + + let trust = p.get("trust").map(|t| Trust { + signed_by: t.get("signed_by").and_then(|v| v.as_str()).map(|s| s.to_string()), + attestation: t + .get("attestation") + .and_then(|v| v.as_str()) + .map(|s| s.to_string()), + }); + + Some(ProviderDescriptor { + id, + kind, + reference, + conformance, + trust, + }) +} + +// --------------------------------------------------------------------------- +// tests +// --------------------------------------------------------------------------- + +#[cfg(test)] +mod tests { + use super::*; + + fn wasm_ref(id: &str) -> ProviderDescriptor { + ProviderDescriptor { + id: id.to_string(), + kind: ProviderKind::Wasm { + abi: "duckdb:extension@2.0.0".to_string(), + artifact: ContentRef::Path(PathBuf::from("artifacts/extensions/aba.wasm")), + content_digest: Some("366cdf".to_string()), + browser_safe: false, + }, + reference: true, + conformance: None, + trust: None, + } + } + + fn entry(providers: Vec) -> ManifestEntry { + ManifestEntry { + name: "aba".to_string(), + wit_contract: "90fdc46a585c".to_string(), + providers, + } + } + + #[test] + fn reference_wasm_is_chosen_certified_by_construction() { + let r = resolve(&entry(vec![wasm_ref("wasm-component")]), &Env::default(), &ResolvePolicy::default(), None) + .expect("resolves"); + assert_eq!(r.chosen_id, "wasm-component"); + assert_eq!(r.chosen_kind, "wasm"); + assert!(matches!(r.reasoning[0].outcome, Outcome::Chosen)); + } + + #[test] + fn forced_provider_excludes_others() { + let r = resolve( + &entry(vec![wasm_ref("wasm-component")]), + &Env::default(), + &ResolvePolicy { forced_provider: Some("wasm-component".into()), denied: vec![] }, + None, + ) + .expect("forced match resolves"); + assert_eq!(r.chosen_id, "wasm-component"); + } + + #[test] + fn forced_unknown_provider_fails_with_reason() { + let err = resolve( + &entry(vec![wasm_ref("wasm-component")]), + &Env::default(), + &ResolvePolicy { forced_provider: Some("nope".into()), denied: vec![] }, + None, + ) + .unwrap_err(); + assert!(render_reasoning(&err.reasoning).contains("not the forced provider")); + } + + #[test] + fn denied_provider_is_excluded() { + let err = resolve( + &entry(vec![wasm_ref("wasm-component")]), + &Env::default(), + &ResolvePolicy { forced_provider: None, denied: vec!["wasm-component".into()] }, + None, + ) + .unwrap_err(); + assert!(render_reasoning(&err.reasoning).contains("user-excluded")); + } + + #[test] + fn uncertified_nonreference_provider_is_gated_out() { + // A non-reference wasm provider with a conformance record at the WRONG + // contract is rejected by the hard gate. + let mut p = wasm_ref("wasm-stale"); + p.reference = false; + p.conformance = Some(Conformance { + suite: "aba@2".into(), + suite_digest: "7f3c".into(), + contract_digest: "DEADBEEF".into(), // != wit_contract + passed: true, + }); + let err = resolve(&entry(vec![p]), &Env::default(), &ResolvePolicy::default(), None) + .unwrap_err(); + assert!(render_reasoning(&err.reasoning).contains("uncertified")); + } + + #[test] + fn native_and_remote_are_unavailable_this_pass() { + let native = ProviderDescriptor { + id: "native-linux-x86_64".into(), + kind: ProviderKind::Native { os: "linux".into(), arch: "x86_64".into(), artifact: ContentRef::Oci("oci://x".into()) }, + reference: false, + conformance: Some(Conformance { suite: "aba@2".into(), suite_digest: "7f3c".into(), contract_digest: "90fdc46a585c".into(), passed: true }), + trust: None, + }; + // native is certified but unavailable -> wasm reference wins. + let r = resolve(&entry(vec![native, wasm_ref("wasm-component")]), &Env::default(), &ResolvePolicy::default(), None).expect("resolves to wasm"); + assert_eq!(r.chosen_id, "wasm-component"); + assert!(render_reasoning(&r.reasoning).contains("native-linux-x86_64")); + } + + #[test] + fn backward_compat_single_artifact_reads_as_one_wasm_reference() { + let index = serde_json::json!({ + "extensions": [ + { "name": "aba", "artifact": "artifacts/extensions/aba.wasm", + "content_digest": "366cdf", "wit_contract": "90fdc46a585c", + "wit_contract_version": "2.0.0" } + ] + }); + let e = read_manifest_entry(&index, "aba").expect("entry"); + assert_eq!(e.providers.len(), 1); + assert_eq!(e.providers[0].id, "wasm-component"); + assert!(e.providers[0].reference); + assert_eq!(e.wit_contract, "90fdc46a585c"); + } + + #[test] + fn explicit_providers_array_is_read() { + let index = serde_json::json!({ + "extensions": [ + { "name": "aba", "wit_contract": "90fdc46a585c", "wit_contract_version": "2.0.0", + "providers": [ + { "id": "wasm-component", "kind": "wasm", "reference": true, + "artifact": "artifacts/extensions/aba.wasm", "content_digest": "366cdf" } + ] } + ] + }); + let e = read_manifest_entry(&index, "aba").expect("entry"); + assert_eq!(e.providers.len(), 1); + assert_eq!(e.providers[0].id, "wasm-component"); + assert!(e.providers[0].reference); + } +} diff --git a/tooling/ducklink-install.sh b/tooling/ducklink-install.sh new file mode 100755 index 0000000..e72abae --- /dev/null +++ b/tooling/ducklink-install.sh @@ -0,0 +1,18 @@ +#!/usr/bin/env bash +# ducklink-install.sh - pre-populate the local extension dir so plain +# `LOAD ` works on stock DuckDB (the `ducklink install ` step). +# +# Drives stock DuckDB's `INSTALL FROM ''`, which fetches the shim +# from the DuckLink repo and copies it into the extension directory. Thereafter a +# plain `LOAD ` (any session) loads it - no manual INSTALL. +# +# Usage: ducklink-install.sh [duckdb_cli] +set -euo pipefail +NAME=${1:?name}; REPO=${2:?repo_dir}; EXTDIR=${3:?extension_dir}; CLI=${4:-duckdb} +mkdir -p "$EXTDIR" +"$CLI" -unsigned :memory: <_init_c_api` per managed name (added via +# the `ducklink_shim!` table in src/lib.rs) plus the native-provider entrypoints. +# This stamps a copy as `.duckdb_extension` with the DuckDB metadata footer +# and lays it out at /// (the INSTALL FROM layout). +# +# Usage: gen-shim.sh +# e.g. gen-shim.sh aba osx_arm64 v1.5.4 ./ducklink-repo target/debug/libducklink.dylib +set -euo pipefail +NAME=${1:?name}; PLATFORM=${2:?platform}; DV=${3:?duckdb_version}; REPO=${4:?repo_dir}; LIB=${5:?lib_path} +HERE=$(cd "$(dirname "$0")/.." && pwd) +APPEND="$HERE/extension-ci-tools/scripts/append_extension_metadata.py" +OUT="$REPO/$DV/$PLATFORM" +mkdir -p "$OUT" +RAW=$(mktemp -t "$NAME.raw.XXXX") +cp "$LIB" "$RAW" +python3 "$APPEND" -l "$RAW" -o "$OUT/$NAME.duckdb_extension" -n "$NAME" \ + --abi-type C_STRUCT_UNSTABLE -dv "$DV" -p "$PLATFORM" -ev v0.0.1 +gzip -kf "$OUT/$NAME.duckdb_extension" +rm -f "$RAW" +echo "wrote $OUT/$NAME.duckdb_extension (+ .gz)" diff --git a/tooling/native-conformance.sh b/tooling/native-conformance.sh new file mode 100755 index 0000000..9a915df --- /dev/null +++ b/tooling/native-conformance.sh @@ -0,0 +1,65 @@ +#!/usr/bin/env bash +# native-conformance.sh - run the provider-neutral conformance suite against the +# NATIVE provider of on stock DuckDB and, on pass, emit its conformance +# record (suite + content-digest + contract + passed) for registry/index.json. +# +# The suite is provider-blind SQL (extensions/-component/conformance.sql). +# A pass certifies the native provider AT the current contract + suite digest; +# the resolver's hard gate admits only records whose suite_digest equals the +# canonical and whose `at` equals the live wit_contract. +# +# Usage: native-conformance.sh [cli] +set -euo pipefail +NAME=${1:?name}; ART=${2:?native_artifact}; SUITE=${3:?conformance.sql}; CONTRACT=${4:?contract_digest}; CLI=${5:-duckdb} +EXP="${SUITE%.sql}.expected" + +# Canonical suite digest -- build C's STRUCTURED scheme over conformance.{sql,expected} +# (byte-identical to resolver::compute_suite_digest / tooling/conformance.py), +# NOT a plain sha256, so the emitted record's suite_digest matches the canonical +# the resolver recomputes. +SUITE_DIGEST=$(python3 - "$SUITE" "$EXP" <<'PY' +import hashlib,sys +sql=open(sys.argv[1]).read(); exp=open(sys.argv[2]).read() +nsql="\n".join(l.rstrip() for l in sql.splitlines() if l.strip() and not l.lstrip().startswith("--")) +ne=[] +for l in exp.splitlines(): + s=l.rstrip() + if not s.strip(): continue + ls=s.lstrip() + if ls=="#" or ls.startswith("# "): continue + ne.append(s) +canon=b"duckdb:conformance-suite:1\n"+nsql.encode()+b"\n\x1e\n"+"\n".join(ne).encode() +print(hashlib.sha256(canon).hexdigest()) +PY +) + +# Run the suite (provider-blind SQL) against the native provider in `.mode csv`, +# the SAME way the canonical smoke/conformance harness emits, so it compares +# against the EXISTING reviewed fixture (suite's companion `.expected`, minus its +# `#` comment lines). The reference (wasm) provider is certified-by-construction +# against this same fixture; matching it certifies the native provider. +GOT=$("$CLI" -unsigned :memory: 2>/dev/null </dev/null; then + PASSED=false + fi +fi + +if [[ "$PASSED" == true ]]; then + echo "[$NAME native] CONFORMANCE PASSED at contract ${CONTRACT:0:12} suite ${SUITE_DIGEST:0:12}" + cat < NATIVE passthrough (precedence native > wasm) +# 2. DUCKLINK_PROVIDER=wasm... -> force the WASM provider +# 3. DUCKLINK_ALLOW_NATIVE=0 -> native denied, WASM fallback +# 4. tampered suite_digest -> conformance gate rejects native, WASM fallback +# plus the native conformance run (suite vs the native provider). +# +# Requires: a stock DuckDB v1.5.4 CLI (path in $DUCKDB_CLI), built against the +# unstable C API. Run with `-unsigned` (the Stage-0 signing posture). +# +# Usage: DUCKDB_CLI=/path/to/duckdb verify-d.sh [workdir] +set -euo pipefail +HERE=$(cd "$(dirname "$0")/.." && pwd) # native-extension/ducklink +ROOT=$(cd "$HERE/../.." && pwd) # ducklink monorepo +CLI=${DUCKDB_CLI:?set DUCKDB_CLI to a stock duckdb v1.5.4 binary} +WORK=${1:-$(mktemp -d)} +PLATFORM=$("$CLI" -noheader -list -c "PRAGMA platform" :memory:) +REV=$("$CLI" -noheader -list -c "SELECT version()" :memory:) +CONTRACT=$(python3 -c "import json;d=json.load(open('$ROOT/registry/index.json'));print([e for e in d['extensions'] if e['name']=='aba'][0]['wit_contract'])") +SUITE=$ROOT/extensions/aba-component/conformance.sql +SUITE_EXP=$ROOT/extensions/aba-component/conformance.expected +LIB=$HERE/target/debug/libducklink.dylib + +REPO=$WORK/repo; DLHOME=$WORK/home; EXTDIR=$WORK/extdir +rm -rf "$REPO" "$DLHOME" "$EXTDIR"; mkdir -p "$DLHOME/artifacts" "$DLHOME/suites" "$EXTDIR" + +echo "## build"; ( cd "$HERE" && cargo build >/dev/null 2>&1 ) +echo "## generate shim + native provider into the repo ($REV/$PLATFORM)" +bash "$HERE/tooling/gen-shim.sh" aba "$PLATFORM" "$REV" "$REPO" "$LIB" >/dev/null +bash "$HERE/tooling/gen-shim.sh" aba_native "$PLATFORM" "$REV" "$REPO" "$LIB" >/dev/null + +echo "## DuckLink home (manifest + artifacts + suite)" +cp "$ROOT/artifacts/extensions/aba.wasm" "$DLHOME/artifacts/aba.wasm" +cp "$REPO/$REV/$PLATFORM/aba_native.duckdb_extension" "$DLHOME/artifacts/aba_native.duckdb_extension" +cp "$SUITE" "$DLHOME/suites/aba.sql" +cp "$SUITE_EXP" "$DLHOME/suites/aba.expected" +# Canonical suite digest: build C's structured scheme over conformance.{sql,expected} +# (byte-identical to resolver::compute_suite_digest / tooling/conformance.py). +SUITE_DIGEST=$(python3 - "$SUITE" "$SUITE_EXP" <<'PY' +import hashlib,sys +sql=open(sys.argv[1]).read(); exp=open(sys.argv[2]).read() +nsql="\n".join(l.rstrip() for l in sql.splitlines() if l.strip() and not l.lstrip().startswith("--")) +ne=[] +for l in exp.splitlines(): + s=l.rstrip() + if not s.strip(): continue + ls=s.lstrip() + if ls=="#" or ls.startswith("# "): continue + ne.append(s) +canon=b"duckdb:conformance-suite:1\n"+nsql.encode()+b"\n\x1e\n"+"\n".join(ne).encode() +print(hashlib.sha256(canon).hexdigest()) +PY +) +echo " canonical suite_digest = $SUITE_DIGEST" +python3 - "$DLHOME/index.json" "$CONTRACT" "$SUITE_DIGEST" <<'PY' +import json,sys +out,contract,suite=sys.argv[1:4] +json.dump({"extensions":[{"name":"aba","wit_contract":contract,"wit_contract_version":"2.0.0","providers":[ + {"id":"wasm-component","kind":"wasm","reference":True,"abi":"duckdb:extension@2.0.0","artifact":"artifacts/aba.wasm", + "conformance":{"suite":"aba@2","suite_digest":suite,"at":contract,"passed":True}}, + {"id":"native-arm64-macos","kind":"native","platform":{"os":"macos","arch":"arm64"},"artifact":"artifacts/aba_native.duckdb_extension", + "trust":{"signed_by":"ed25519:ducklink-dev","attestation":"none"}, + "conformance":{"suite":"aba@2","suite_digest":suite,"at":contract,"passed":True}}]}]},open(out,"w"),indent=1) +PY + +echo "## ducklink install aba" +bash "$HERE/tooling/ducklink-install.sh" aba "$REPO" "$EXTDIR" "$CLI" >/dev/null + +run(){ DUCKLINK_HOME="$DLHOME" "$@" "$CLI" -unsigned -box :memory:; } +echo "== 1: plain LOAD aba -> NATIVE ==" +DUCKLINK_HOME="$DLHOME" "$CLI" -unsigned :memory: < wasm ==" +DUCKLINK_HOME="$DLHOME" DUCKLINK_ALLOW_NATIVE=0 "$CLI" -unsigned :memory: < gate rejects native -> wasm ==" +DLBAD=$WORK/home-tampered; rm -rf "$DLBAD"; cp -R "$DLHOME" "$DLBAD" +python3 - "$DLBAD/index.json" <<'PY' +import json,sys +p=sys.argv[1]; d=json.load(open(p)) +for pr in d["extensions"][0]["providers"]: + if pr["kind"]=="native": pr["conformance"]["suite_digest"]="DEADBEEF-forged" +json.dump(d,open(p,"w"),indent=1) +PY +DUCKLINK_HOME="$DLBAD" "$CLI" -unsigned :memory: <