From 52dfdceae078bf86c56eb9c45d26b05cdeab2384 Mon Sep 17 00:00:00 2001 From: ZVN DEV <78920650+zvndev@users.noreply.github.com> Date: Tue, 9 Jun 2026 15:02:26 -0400 Subject: [PATCH 1/3] fix: harden server against oversized-row DoS, enforce readonly role, fix NULL wire format and unordered window frames MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - storage: reject rows over MAX_ROW_DATA_SIZE (4070B) with a graceful RowTooLarge error instead of panicking; with panic=abort one oversized insert previously killed the whole server. Size check runs before WAL append so a rejected update can't poison replay. - server: enforce roles at dispatch — readonly principals get 'permission denied' on DML/DDL/transaction control; unknown roles fail closed; shared-password/open/embedded modes unaffected. - server: serialize NULL as bareword 'null' on the wire (was '{}'), matching the TS client's documented sentinel; remote REPL renders NULL. - query: window aggregates with no 'order' clause now use the whole partition as the frame instead of a running frame. Co-Authored-By: Claude Fable 5 --- CHANGELOG.md | 39 ++++ crates/cli/src/main.rs | 38 +++- crates/query/src/executor/plan_exec.rs | 40 ++++ crates/query/src/executor/tests.rs | 128 ++++++++++++ crates/query/tests/oversized_rows.rs | 135 ++++++++++++ crates/server/src/handler.rs | 164 +++++++++++++-- crates/server/tests/integration.rs | 197 ++++++++++++++++++ crates/storage/src/catalog.rs | 20 ++ crates/storage/src/error.rs | 6 + crates/storage/src/heap.rs | 107 +++++++++- crates/storage/src/page.rs | 9 + .../audits/2026-05-26-smoke-audit.md | 0 .../plans/2026-04-16-ts-client-v02-sprint.md | 0 13 files changed, 866 insertions(+), 17 deletions(-) create mode 100644 crates/query/tests/oversized_rows.rs rename SMOKE-AUDIT.md => docs/audits/2026-05-26-smoke-audit.md (100%) rename clients/ts/SPRINT-PLAN.md => docs/superpowers/plans/2026-04-16-ts-client-v02-sprint.md (100%) diff --git a/CHANGELOG.md b/CHANGELOG.md index e42097d..fe769ed 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,45 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +### Fixed +- **Oversized rows no longer kill the server (remote DoS).** Inserting or + updating a row whose encoded size exceeds one 4 KB page (`MAX_ROW_DATA_SIZE`, + 4070 bytes) previously hit a `panic!` in the heap layer, which — combined with + `panic = "abort"` — terminated the entire server process. Any connected client + could take down the database with a single ~5 KB string insert. The heap now + rejects oversized rows with a graceful `row too large: N bytes exceeds max M + bytes` query error before anything is written or WAL-logged (an oversized + update can no longer poison WAL replay), and the connection and server keep + running. There is still no large-object/overflow-page support — values over + ~4 KB are rejected, not stored. +- **`readonly` role is now enforced at the query layer.** Previously the role + was authenticated and stored but never checked: a `readonly` user could + insert, update, delete, and drop tables. The server now classifies each + parsed statement and rejects writes (DML, DDL, view DDL, and transaction + control) from `readonly` principals with `permission denied: role 'readonly' + cannot execute write statements`. Unknown roles fail closed. Shared-password + mode, open mode, and embedded use are unaffected. +- **NULL values now arrive as `null` on the wire instead of `{}`.** The server + serialized SQL NULLs as `{}`, which the remote CLI displayed verbatim and + which broke the TS client's documented `"null"` sentinel for typed-row + decoding. The wire serialization is now the bareword `null`, and the remote + REPL renders it as `NULL`, matching embedded mode. +- **Window aggregates without `order` now compute the whole-partition value.** + `avg(.x) over (partition .d)` previously returned a running aggregate per + row (frame = partition start → current row) even with no `order` clause; per + standard semantics the frame is now the entire partition. Ordered windows + keep the running-frame behavior; ranking functions are unchanged. + +### Documentation +- Ecosystem-wide accuracy sweep: site pages synced to v0.4.5 (banners were + v0.2.0, MSRV corrected to 1.93), crates.io homepage URL fixed (was a 404), + README/CONTRIBUTING/AGENTS no longer claim the bench suite is a CI merge + gate, SECURITY.md documents both auth modes and the ≤0.4.5 readonly caveat, + RELEASES.md covers all six crates + the Docker image, deploy examples fixed + (`fly.toml` was missing `POWDB_BIND=0.0.0.0`), TS client docs document the + multi-user-server incompatibility, and AGENTS.md gained small-model-tested + gotchas (reserved aggregate keywords as aliases; line-oriented REPL). + ## [0.4.5] - 2026-06-09 ### Added diff --git a/crates/cli/src/main.rs b/crates/cli/src/main.rs index cf7c6a7..f99072b 100644 --- a/crates/cli/src/main.rs +++ b/crates/cli/src/main.rs @@ -1205,6 +1205,19 @@ fn print_local_result(result: &QueryResult) { } } +/// Render one wire cell for display. The server serializes NULL as the +/// bareword "null" (the sentinel the TS client's typed decoder matches); +/// remote mode renders it as `NULL`, matching the embedded REPL. A string +/// column whose *value* is literally "null" is indistinguishable on the +/// untyped wire — same tradeoff the TS client documents. +fn render_remote_cell(cell: &str) -> String { + if cell == "null" { + "NULL".into() + } else { + cell.into() + } +} + fn print_remote_result(msg: &Message) { match msg { Message::ResultRows { columns, rows } => { @@ -1212,10 +1225,14 @@ fn print_remote_result(msg: &Message) { println!("(empty set)"); return; } - print_table(columns, rows); + let rendered: Vec> = rows + .iter() + .map(|row| row.iter().map(|c| render_remote_cell(c)).collect()) + .collect(); + print_table(columns, &rendered); } Message::ResultScalar { value } => { - println!("{value}"); + println!("{}", render_remote_cell(value)); } Message::ResultOk { affected } => { println!( @@ -1286,3 +1303,20 @@ fn format_value(v: &Value) -> String { Value::Empty => "NULL".into(), } } + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn remote_null_sentinel_renders_as_null_like_embedded() { + // The wire sends NULL as the bareword "null"; remote display must + // match the embedded REPL's `NULL`. + assert_eq!(render_remote_cell("null"), "NULL"); + assert_eq!(format_value(&Value::Empty), "NULL"); + // Ordinary values pass through untouched. + assert_eq!(render_remote_cell("42"), "42"); + assert_eq!(render_remote_cell(""), ""); + assert_eq!(render_remote_cell("NULL"), "NULL"); + } +} diff --git a/crates/query/src/executor/plan_exec.rs b/crates/query/src/executor/plan_exec.rs index 8aa1076..c0ebea6 100644 --- a/crates/query/src/executor/plan_exec.rs +++ b/crates/query/src/executor/plan_exec.rs @@ -2914,6 +2914,26 @@ pub(super) fn execute_window( std::cmp::Ordering::Equal }); + // SQL window-frame semantics: with no `order` clause the frame for an + // aggregate window is the ENTIRE partition, not the running prefix. + // The loop below computes running values; for the no-order case we + // back-fill every row of a partition with the partition's final + // (i.e. complete) aggregate once its boundary is reached. Ranking + // functions are untouched — row_number/rank/dense_rank are inherently + // positional. + let whole_partition_frame = wdef.order_by.is_empty() + && matches!( + wdef.function, + WindowFunc::Sum + | WindowFunc::Avg + | WindowFunc::Count + | WindowFunc::Min + | WindowFunc::Max + ); + // Original row indices of the partition currently being scanned + // (only tracked when back-filling is needed). + let mut partition_row_indices: Vec = Vec::new(); + // Compute window values in sorted order, tracking partition boundaries. let mut win_values: Vec = vec![Value::Empty; n]; let mut partition_start = 0usize; @@ -2943,6 +2963,15 @@ pub(super) fn execute_window( }; if new_partition { + // No-order aggregate frame: the partition that just ended is + // complete, so its final running value IS the whole-partition + // aggregate. Back-fill it onto every row of that partition. + if whole_partition_frame && sorted_pos > 0 { + let final_v = win_values[indices[sorted_pos - 1]].clone(); + for ri in partition_row_indices.drain(..) { + win_values[ri] = final_v.clone(); + } + } partition_start = sorted_pos; running_count = 0; running_int_sum = 0; @@ -3071,6 +3100,17 @@ pub(super) fn execute_window( prev_order_key = Some(current_order_key); win_values[row_idx] = value; + if whole_partition_frame { + partition_row_indices.push(row_idx); + } + } + + // Back-fill the final partition (the loop only flushes at boundaries). + if whole_partition_frame && n > 0 { + let final_v = win_values[indices[n - 1]].clone(); + for ri in partition_row_indices.drain(..) { + win_values[ri] = final_v.clone(); + } } // Append the computed window column to each row. diff --git a/crates/query/src/executor/tests.rs b/crates/query/src/executor/tests.rs index f5e8abc..bfc1509 100644 --- a/crates/query/src/executor/tests.rs +++ b/crates/query/src/executor/tests.rs @@ -4172,3 +4172,131 @@ fn test_memory_limit_default_allows_normal_query() { _ => panic!("expected rows"), } } + +// --------------------------------------------------------------------------- +// Window aggregates without `order`: frame must be the ENTIRE partition, +// not a running prefix. (`avg(.sal) over (partition .dept)` used to return +// 10/15/20 for salaries 10/20/30 instead of 20/20/20.) +// --------------------------------------------------------------------------- + +fn window_engine() -> Engine { + let id = TEST_COUNTER.fetch_add(1, Ordering::SeqCst); + let dir = std::env::temp_dir().join(format!("powdb_win_{}_{}", std::process::id(), id)); + let mut engine = Engine::new(&dir).unwrap(); + engine + .execute_powql("type Emp { required name: str, required dept: str, required sal: int }") + .unwrap(); + for (name, dept, sal) in [ + ("a", "eng", 10), + ("b", "eng", 20), + ("c", "eng", 30), + ("d", "ops", 100), + ("e", "ops", 300), + ] { + engine + .execute_powql(&format!( + r#"insert Emp {{ name := "{name}", dept := "{dept}", sal := {sal} }}"# + )) + .unwrap(); + } + engine +} + +/// Extract (name → window value) pairs from a two-column result. +fn window_col_by_name(result: QueryResult) -> std::collections::HashMap { + match result { + QueryResult::Rows { columns, rows } => { + let name_idx = columns.iter().position(|c| c == "name").unwrap(); + let win_idx = columns.len() - 1; + rows.into_iter() + .map(|r| { + let name = match &r[name_idx] { + Value::Str(s) => s.clone(), + other => panic!("expected name string, got {other:?}"), + }; + (name, r[win_idx].clone()) + }) + .collect() + } + other => panic!("expected rows, got {other:?}"), + } +} + +#[test] +fn test_window_agg_without_order_uses_whole_partition() { + let mut engine = window_engine(); + let result = engine + .execute_powql("Emp { .name, davg: avg(.sal) over (partition .dept) }") + .unwrap(); + let by_name = window_col_by_name(result); + // eng partition avg = (10+20+30)/3 = 20 for EVERY row. + for n in ["a", "b", "c"] { + assert_eq!(by_name[n], Value::Float(20.0), "row {n}"); + } + // ops partition avg = (100+300)/2 = 200 for both rows. + for n in ["d", "e"] { + assert_eq!(by_name[n], Value::Float(200.0), "row {n}"); + } +} + +#[test] +fn test_window_sum_count_min_max_without_order_whole_partition() { + let mut engine = window_engine(); + + let by_name = window_col_by_name( + engine + .execute_powql("Emp { .name, dsum: sum(.sal) over (partition .dept) }") + .unwrap(), + ); + for n in ["a", "b", "c"] { + assert_eq!(by_name[n], Value::Int(60), "sum row {n}"); + } + for n in ["d", "e"] { + assert_eq!(by_name[n], Value::Int(400), "sum row {n}"); + } + + let by_name = window_col_by_name( + engine + .execute_powql("Emp { .name, dcnt: count(.sal) over (partition .dept) }") + .unwrap(), + ); + for n in ["a", "b", "c"] { + assert_eq!(by_name[n], Value::Int(3), "count row {n}"); + } + + let by_name = window_col_by_name( + engine + .execute_powql("Emp { .name, dmin: min(.sal) over (partition .dept) }") + .unwrap(), + ); + for n in ["a", "b", "c"] { + assert_eq!(by_name[n], Value::Int(10), "min row {n}"); + } + + let by_name = window_col_by_name( + engine + .execute_powql("Emp { .name, dmax: max(.sal) over (partition .dept) }") + .unwrap(), + ); + for n in ["a", "b", "c"] { + assert_eq!(by_name[n], Value::Int(30), "max row {n}"); + } + for n in ["d", "e"] { + assert_eq!(by_name[n], Value::Int(300), "max row {n}"); + } +} + +#[test] +fn test_window_agg_with_order_keeps_running_frame() { + // WITH an explicit `order`, the running (rows-so-far) frame is the + // existing documented behavior — it must not change. + let mut engine = window_engine(); + let by_name = window_col_by_name( + engine + .execute_powql("Emp { .name, ravg: avg(.sal) over (partition .dept order .sal) }") + .unwrap(), + ); + assert_eq!(by_name["a"], Value::Float(10.0)); + assert_eq!(by_name["b"], Value::Float(15.0)); + assert_eq!(by_name["c"], Value::Float(20.0)); +} diff --git a/crates/query/tests/oversized_rows.rs b/crates/query/tests/oversized_rows.rs new file mode 100644 index 0000000..fe7064d --- /dev/null +++ b/crates/query/tests/oversized_rows.rs @@ -0,0 +1,135 @@ +//! Oversized-row regression tests (remote-DoS fix). +//! +//! With `panic = "abort"` in the release profile, a panicking insert path +//! kills the whole server process. An encoded row larger than one 4KB +//! slotted page used to hit `expect("row too large for empty page")` in +//! `HeapFile::insert`. These tests pin the fixed behavior: oversized rows +//! are a clean query error and the engine stays fully usable. + +use powdb_query::executor::Engine; +use powdb_query::result::QueryResult; + +fn temp_engine(name: &str) -> (Engine, std::path::PathBuf) { + let dir = std::env::temp_dir().join(format!( + "powdb_oversized_{name}_{}_{}", + std::process::id(), + std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .unwrap() + .as_nanos() + )); + std::fs::create_dir_all(&dir).unwrap(); + let engine = Engine::new(&dir).unwrap(); + (engine, dir) +} + +fn big_string(n: usize) -> String { + "x".repeat(n) +} + +#[test] +fn oversized_insert_is_clean_error_and_engine_survives() { + let (mut engine, dir) = temp_engine("insert"); + engine + .execute_powql("type T { required id: int, b: str }") + .unwrap(); + + // 5000-byte string > 4KB page capacity → must be a clean error. + let q = format!(r#"insert T {{ id := 1, b := "{}" }}"#, big_string(5000)); + let err = engine + .execute_powql(&q) + .expect_err("oversized insert must fail"); + assert!( + err.to_string().contains("row too large"), + "unexpected error: {err}" + ); + + // Engine keeps working: normal insert + query succeed. + engine + .execute_powql(r#"insert T { id := 2, b := "small" }"#) + .unwrap(); + let res = engine.execute_powql("count(T)").unwrap(); + match res { + QueryResult::Scalar(v) => assert_eq!(format!("{v:?}"), "Int(1)"), + other => panic!("expected scalar count, got {other:?}"), + } + drop(engine); + std::fs::remove_dir_all(&dir).ok(); +} + +#[test] +fn oversized_update_is_clean_error_and_row_intact() { + let (mut engine, dir) = temp_engine("update"); + engine + .execute_powql("type T { required id: int, b: str }") + .unwrap(); + engine + .execute_powql(r#"insert T { id := 1, b := "original" }"#) + .unwrap(); + + // Oversized new value on update must fail cleanly... + let q = format!( + r#"T filter .id = 1 update {{ b := "{}" }}"#, + big_string(5000) + ); + let err = engine + .execute_powql(&q) + .expect_err("oversized update must fail"); + assert!( + err.to_string().contains("row too large"), + "unexpected error: {err}" + ); + + // ...and the original row must be untouched. + let res = engine.execute_powql("T filter .id = 1").unwrap(); + match res { + QueryResult::Rows { rows, .. } => { + assert_eq!(rows.len(), 1, "row must still exist"); + let joined = format!("{:?}", rows[0]); + assert!( + joined.contains("original"), + "row value must be unchanged: {joined}" + ); + } + other => panic!("expected rows, got {other:?}"), + } + drop(engine); + std::fs::remove_dir_all(&dir).ok(); +} + +#[test] +fn oversized_update_does_not_poison_wal_replay() { + // A failed oversized update must not leave a WAL record that breaks (or + // mutates state during) recovery on the next open. + let (mut engine, dir) = temp_engine("replay"); + engine + .execute_powql("type T { required id: int, b: str }") + .unwrap(); + engine + .execute_powql(r#"insert T { id := 1, b := "original" }"#) + .unwrap(); + let q = format!( + r#"T filter .id = 1 update {{ b := "{}" }}"#, + big_string(5000) + ); + engine + .execute_powql(&q) + .expect_err("oversized update must fail"); + // Run one more statement so any buffered WAL bytes get group-committed. + engine.execute_powql("count(T)").unwrap(); + drop(engine); + + // Reopen: replay must succeed and show the original value. + let mut engine = Engine::new(&dir).expect("reopen after failed oversized update"); + let res = engine.execute_powql("T filter .id = 1").unwrap(); + match res { + QueryResult::Rows { rows, .. } => { + assert_eq!(rows.len(), 1); + let joined = format!("{:?}", rows[0]); + assert!(joined.contains("original"), "row corrupted: {joined}"); + } + other => panic!("expected rows, got {other:?}"), + } + drop(engine); + std::fs::remove_dir_all(&dir).ok(); +} diff --git a/crates/server/src/handler.rs b/crates/server/src/handler.rs index b9234c1..46ccd8d 100644 --- a/crates/server/src/handler.rs +++ b/crates/server/src/handler.rs @@ -1,5 +1,5 @@ use crate::protocol::Message; -use powdb_auth::UserStore; +use powdb_auth::{Permission, Role, UserStore}; use powdb_query::executor::{is_read_only_statement, Engine}; use powdb_query::parser; use powdb_query::result::{QueryError, QueryResult}; @@ -81,15 +81,48 @@ fn constant_time_eq(a: &[u8], b: &[u8]) -> bool { diff == 0 } -/// An authenticated connection's identity, carried alongside the session for -/// later per-operation RBAC enforcement (Slice 3). For now it is bound at -/// connect time and logged; it is not yet consulted per query. +/// An authenticated connection's identity. Bound at connect time and consulted +/// on every query by [`dispatch_query`] to enforce the user's role: a +/// `readonly` principal may only execute read statements. #[derive(Debug, Clone, PartialEq, Eq)] pub struct Principal { pub name: String, pub role: String, } +/// Whether `role` grants the `Write` permission. Unknown role names fail +/// closed (no write). Shared-password / open / embedded modes never construct +/// a [`Principal`], so they are unaffected by this gate. +fn role_can_write(role: &str) -> bool { + Role::builtin(role).is_some_and(|r| r.allows(Permission::Write)) +} + +/// Enforce the principal's role against a parsed statement. Returns an error +/// for any non-read statement (insert/update/delete/upsert/DDL/view ops/ +/// transaction control) when the role does not grant `Write`. +/// +/// Classification uses the parsed AST via +/// [`powdb_query::executor::is_read_only_statement`] — the exact same +/// classifier the RwLock read/write split relies on — so the permission +/// boundary and the concurrency boundary can never disagree. +fn check_statement_permitted( + principal: Option<&Principal>, + stmt: &powdb_query::ast::Statement, +) -> Result<(), QueryError> { + let Some(p) = principal else { + // No per-user identity (shared-password or open mode): full access, + // byte-identical to the pre-RBAC behavior. + return Ok(()); + }; + if is_read_only_statement(stmt) || role_can_write(&p.role) { + return Ok(()); + } + Err(QueryError::Execution(format!( + "permission denied: role '{}' cannot execute write statements", + p.role + ))) +} + /// Result of the connect-time authentication decision. #[derive(Debug, Clone, PartialEq, Eq)] pub enum AuthOutcome { @@ -169,6 +202,8 @@ const SAFE_ERROR_PREFIXES: &[&str] = &[ "cannot", "no such", "already exists", + "permission denied", + "row too large", ]; /// Sanitize an error message before sending it to the client. @@ -219,9 +254,25 @@ pub struct ConnOpts<'a> { /// Execute a query against the engine under the RwLock. Read-only /// statements acquire `.read()` so concurrent SELECTs can scan in /// parallel; mutations acquire `.write()`. -fn dispatch_query(engine: &Arc>, query: &str) -> Result { +/// +/// When `principal` is `Some`, the user's role is enforced first: a role +/// without the `Write` permission (i.e. `readonly`) gets a clean +/// "permission denied" error for any non-read statement, before any lock +/// is taken or any engine state is touched. +fn dispatch_query( + engine: &Arc>, + query: &str, + principal: Option<&Principal>, +) -> Result { let stmt_result = parser::parse(query).map_err(|e| e.to_string()); + // Role enforcement happens on the parsed AST. Statements that fail to + // parse fall through — the engine returns the parse error itself and + // can never execute anything for them. + if let Ok(stmt) = &stmt_result { + check_statement_permitted(principal, stmt)?; + } + let can_try_read = matches!(&stmt_result, Ok(s) if is_read_only_statement(s)); if can_try_read { let res = { @@ -298,9 +349,9 @@ where } }; - // The authenticated identity for this connection. Bound at connect time and - // carried for later per-operation RBAC enforcement (Slice 3). - let _principal: Option; + // The authenticated identity for this connection. Bound at connect time + // and enforced on every query by `dispatch_query`. + let principal: Option; match connect_msg { Message::Connect { db_name, @@ -339,12 +390,14 @@ where write_msg(&mut writer, &err).await; return; } - AuthOutcome::Authenticated { principal } => { + AuthOutcome::Authenticated { + principal: auth_principal, + } => { // Auth succeeded — clear any prior failure count. if let (Some(limiter), Some(ip)) = (rate_limiter, peer_ip) { clear_auth_failures(limiter, ip); } - match &principal { + match &auth_principal { Some(p) => { info!(peer = %peer, db = %db_name, user = %p.name, role = %p.role, "authenticated"); } @@ -352,7 +405,7 @@ where info!(peer = %peer, db = %db_name, "client connected"); } } - _principal = principal; + principal = auth_principal; } } @@ -424,7 +477,8 @@ where let handle = tokio::task::spawn_blocking({ let engine = engine.clone(); let query = query.clone(); - move || dispatch_query(&engine, &query) + let principal = principal.clone(); + move || dispatch_query(&engine, &query, principal.as_ref()) }); let abort_handle = handle.abort_handle(); match tokio::time::timeout(query_timeout, handle).await { @@ -496,7 +550,12 @@ fn value_to_display(v: &Value) -> String { u[0], u[1], u[2], u[3], u[4], u[5], u[6], u[7], u[8], u[9], u[10], u[11], u[12], u[13], u[14], u[15]), Value::Bytes(b) => format!("<{} bytes>", b.len()), - Value::Empty => "{}".into(), + // NULL is serialized as the bareword "null" on the wire. This is the + // sentinel the TypeScript client's typed-row decoder already + // documents and matches (`coerceValue` treats the exact token + // "null" as NULL for non-str columns); the previous "{}" rendering + // was a bug that neither the TS client nor the CLI recognized. + Value::Empty => "null".into(), } } @@ -504,6 +563,85 @@ fn value_to_display(v: &Value) -> String { mod tests { use super::*; + // ---- Wire NULL rendering (Fix: remote protocol rendered NULL as `{}`) ---- + + #[test] + fn null_serializes_as_null_bareword_on_wire() { + assert_eq!(value_to_display(&Value::Empty), "null"); + } + + // ---- Role enforcement (Fix: readonly role was not enforced) ---- + + fn parsed(q: &str) -> powdb_query::ast::Statement { + parser::parse(q).unwrap() + } + + fn principal(role: &str) -> Option { + Some(Principal { + name: "u".into(), + role: role.into(), + }) + } + + #[test] + fn readonly_can_read_but_not_write() { + let p = principal("readonly"); + // Reads pass. + assert!(check_statement_permitted(p.as_ref(), &parsed("User")).is_ok()); + assert!(check_statement_permitted(p.as_ref(), &parsed("count(User)")).is_ok()); + assert!(check_statement_permitted(p.as_ref(), &parsed("explain User")).is_ok()); + // Writes, DDL, and transaction control are denied. + for q in [ + r#"insert User { name := "x" }"#, + "User filter .id = 1 update { age := 2 }", + "User filter .id = 1 delete", + "drop User", + "alter User add column c: str", + "type T { required id: int }", + "begin", + "commit", + "rollback", + ] { + let err = check_statement_permitted(p.as_ref(), &parsed(q)) + .expect_err(&format!("must deny: {q}")); + assert!( + err.to_string().contains("permission denied"), + "unexpected error for {q}: {err}" + ); + } + } + + #[test] + fn readwrite_and_admin_have_full_query_access() { + for role in ["readwrite", "admin"] { + let p = principal(role); + assert!(check_statement_permitted(p.as_ref(), &parsed("User")).is_ok()); + assert!(check_statement_permitted( + p.as_ref(), + &parsed(r#"insert User { name := "x" }"#) + ) + .is_ok()); + assert!(check_statement_permitted(p.as_ref(), &parsed("drop User")).is_ok()); + } + } + + #[test] + fn unknown_role_fails_closed_for_writes() { + let p = principal("mystery"); + assert!(check_statement_permitted(p.as_ref(), &parsed("User")).is_ok()); + assert!( + check_statement_permitted(p.as_ref(), &parsed(r#"insert User { name := "x" }"#)) + .is_err() + ); + } + + #[test] + fn no_principal_means_full_access() { + // Shared-password / open mode: no per-user identity, no restriction. + assert!(check_statement_permitted(None, &parsed("drop User")).is_ok()); + assert!(check_statement_permitted(None, &parsed(r#"insert User { name := "x" }"#)).is_ok()); + } + fn store_with_alice() -> UserStore { let mut s = UserStore::new(); s.create_user("alice", "pw", "readwrite").unwrap(); diff --git a/crates/server/tests/integration.rs b/crates/server/tests/integration.rs index c0bf301..c7257a8 100644 --- a/crates/server/tests/integration.rs +++ b/crates/server/tests/integration.rs @@ -335,6 +335,25 @@ async fn test_empty_store_shared_password_fallback() { stream.write_all(&frame).await.unwrap(); let resp = read_response(&mut stream).await; assert_eq!(resp[0], 0x02, "correct shared password should connect"); + + // Shared-password mode has no per-user role: writes must keep + // working exactly as before role enforcement existed. + stream + .write_all(&encode_query("type Item { required name: str }")) + .await + .unwrap(); + let resp = read_response(&mut stream).await; + assert!( + resp[0] == 0x09 || resp[0] == 0x0B, + "shared-password DDL must succeed, got 0x{:02X}", + resp[0] + ); + stream + .write_all(&encode_query(r#"insert Item { name := "widget" }"#)) + .await + .unwrap(); + let resp = read_response(&mut stream).await; + assert_eq!(resp[0], 0x09, "shared-password insert must succeed"); } // Wrong shared password → ERROR. @@ -354,3 +373,181 @@ async fn test_empty_store_shared_password_fallback() { handle.abort(); std::fs::remove_dir_all(&data_dir).ok(); } + +/// Fix 2 (authorization bypass): the `readonly` role must be enforced at the +/// server dispatch boundary. A readonly user can run read statements but every +/// write (insert/update/delete/DDL/transaction control) is rejected with a +/// clean "permission denied" error — and the connection stays alive. +/// `readwrite` and `admin` users keep full query access. +#[tokio::test] +async fn test_readonly_role_enforced_over_tcp() { + use powdb_auth::UserStore; + use powdb_server::protocol::Message; + use std::sync::{Arc, RwLock}; + + let test_id = std::process::id(); + let port = 18500 + (test_id % 1000) as u16; + let data_dir = std::env::temp_dir().join(format!("powdb_rbac_{test_id}")); + std::fs::create_dir_all(&data_dir).unwrap(); + + let mut store = UserStore::new(); + store.create_user("root", "pw", "admin").unwrap(); + store.create_user("rw", "pw", "readwrite").unwrap(); + store.create_user("ro", "pw", "readonly").unwrap(); + let users = Arc::new(store); + + let data_dir_str = data_dir.to_str().unwrap().to_string(); + let addr = format!("127.0.0.1:{port}"); + let bind_addr = addr.clone(); + + let handle = tokio::spawn(async move { + let engine = + powdb_query::executor::Engine::new(std::path::Path::new(&data_dir_str)).unwrap(); + let engine = Arc::new(RwLock::new(engine)); + let listener = tokio::net::TcpListener::bind(&bind_addr).await.unwrap(); + loop { + let (stream, peer) = listener.accept().await.unwrap(); + let eng = engine.clone(); + let users = users.clone(); + let (_, mut rx) = tokio::sync::watch::channel(false); + tokio::spawn(async move { + powdb_server::handler::handle_connection( + stream, + powdb_server::handler::ConnOpts { + engine: eng, + expected_password: None, + users, + shutdown_rx: &mut rx, + idle_timeout: Duration::from_secs(300), + query_timeout: Duration::from_secs(30), + rate_limiter: None, + peer_addr: Some(peer), + }, + ) + .await; + }); + } + }); + + tokio::time::sleep(Duration::from_millis(100)).await; + + // Admin seeds the schema + one row. + { + let mut stream = TcpStream::connect(&addr).await.unwrap(); + stream + .write_all(&encode_connect_user("testdb", "pw", "root")) + .await + .unwrap(); + let resp = read_response(&mut stream).await; + assert_eq!(resp[0], 0x02, "admin should connect"); + + stream + .write_all(&encode_query("type User { required name: str, age: int }")) + .await + .unwrap(); + let resp = read_response(&mut stream).await; + assert!( + resp[0] == 0x09 || resp[0] == 0x0B, + "admin DDL should succeed, got 0x{:02X}", + resp[0] + ); + + stream + .write_all(&encode_query( + r#"insert User { name := "Alice", age := 30 }"#, + )) + .await + .unwrap(); + let resp = read_response(&mut stream).await; + assert_eq!(resp[0], 0x09, "admin insert should succeed"); + } + + // Readonly user: reads OK, every write shape rejected, connection alive. + { + let mut stream = TcpStream::connect(&addr).await.unwrap(); + stream + .write_all(&encode_connect_user("testdb", "pw", "ro")) + .await + .unwrap(); + let resp = read_response(&mut stream).await; + assert_eq!(resp[0], 0x02, "readonly user should connect"); + + // Reads succeed. + stream.write_all(&encode_query("User")).await.unwrap(); + let resp = read_response(&mut stream).await; + assert_eq!(resp[0], 0x07, "readonly select should return rows"); + + stream + .write_all(&encode_query("count(User)")) + .await + .unwrap(); + let resp = read_response(&mut stream).await; + assert_eq!(resp[0], 0x08, "readonly count should return scalar"); + + // Every write statement is rejected with a clean permission error. + let writes = [ + r#"insert User { name := "Mallory", age := 1 }"#, + r#"User filter .name = "Alice" update { age := 99 }"#, + r#"User filter .name = "Alice" delete"#, + "drop User", + "alter User add column hacked: str", + "begin", + ]; + for q in writes { + stream.write_all(&encode_query(q)).await.unwrap(); + let resp = read_response(&mut stream).await; + assert_eq!( + resp[0], 0x0A, + "readonly write must be rejected: {q} (got 0x{:02X})", + resp[0] + ); + match Message::decode(&resp).unwrap() { + Message::Error { message } => assert!( + message.contains("permission denied"), + "expected permission-denied error for {q}, got: {message}" + ), + other => panic!("expected Error for {q}, got {other:?}"), + } + } + + // The connection survives the rejections. + stream.write_all(&encode_query("User")).await.unwrap(); + let resp = read_response(&mut stream).await; + assert_eq!(resp[0], 0x07, "connection must stay alive after denials"); + + // And nothing was actually written/dropped. + stream + .write_all(&encode_query("count(User)")) + .await + .unwrap(); + let resp = read_response(&mut stream).await; + assert_eq!(resp[0], 0x08, "table must still exist"); + match Message::decode(&resp).unwrap() { + Message::ResultScalar { value } => { + assert_eq!(value, "1", "row count must be unchanged") + } + other => panic!("expected scalar, got {other:?}"), + } + } + + // Readwrite user keeps full write access. + { + let mut stream = TcpStream::connect(&addr).await.unwrap(); + stream + .write_all(&encode_connect_user("testdb", "pw", "rw")) + .await + .unwrap(); + let resp = read_response(&mut stream).await; + assert_eq!(resp[0], 0x02, "readwrite user should connect"); + + stream + .write_all(&encode_query(r#"insert User { name := "Bob", age := 25 }"#)) + .await + .unwrap(); + let resp = read_response(&mut stream).await; + assert_eq!(resp[0], 0x09, "readwrite insert should succeed"); + } + + handle.abort(); + std::fs::remove_dir_all(&data_dir).ok(); +} diff --git a/crates/storage/src/catalog.rs b/crates/storage/src/catalog.rs index f355f0b..16de6ba 100644 --- a/crates/storage/src/catalog.rs +++ b/crates/storage/src/catalog.rs @@ -8,6 +8,21 @@ use std::io::{self, Read, Write}; use std::path::{Path, PathBuf}; use tracing::{info, warn}; +/// Reject an encoded row that exceeds the single-page capacity BEFORE it is +/// appended to the WAL. The heap performs the same check at its own insert/ +/// update boundary, but the update paths log to the WAL first — a logged +/// record whose row the heap then rejects would poison the next replay. +fn check_encoded_row_size(encoded: &[u8]) -> io::Result<()> { + if encoded.len() > crate::page::MAX_ROW_DATA_SIZE { + return Err(crate::error::StorageError::RowTooLarge { + size: encoded.len(), + max: crate::page::MAX_ROW_DATA_SIZE, + } + .into()); + } + Ok(()) +} + /// Validate that a name (table or column) is safe for use in file paths and /// follows the identifier convention: starts with a letter or underscore, /// followed by letters, digits, or underscores. @@ -935,6 +950,9 @@ impl Catalog { let tbl = self.by_name_mut(table)?; let mut wal_bytes: Vec = Vec::new(); encode_row_into(&tbl.schema, values, &mut wal_bytes); + // Reject oversized rows BEFORE appending the WAL record: a logged + // Update that the heap then rejects would poison the next replay. + check_encoded_row_size(&wal_bytes)?; let tx_id = self.next_tx(); self.wal_log(tx_id, WalRecordType::Update, table, rid, &wal_bytes)?; self.by_name_mut(table)?.update(rid, values) @@ -961,6 +979,8 @@ impl Catalog { let tbl = self.by_name_mut(table)?; let mut wal_bytes: Vec = Vec::new(); encode_row_into(&tbl.schema, values, &mut wal_bytes); + // Same pre-WAL size gate as [`Self::update`]. + check_encoded_row_size(&wal_bytes)?; let tx_id = self.next_tx(); self.wal_log(tx_id, WalRecordType::Update, table, rid, &wal_bytes)?; self.by_name_mut(table)? diff --git a/crates/storage/src/error.rs b/crates/storage/src/error.rs index d49da00..7c9edd9 100644 --- a/crates/storage/src/error.rs +++ b/crates/storage/src/error.rs @@ -26,6 +26,12 @@ pub enum StorageError { #[error("invalid identifier: {0}")] InvalidIdentifier(String), + + /// An encoded row exceeds the single-page capacity. Returned cleanly at + /// the heap insert/update boundary instead of panicking — with + /// `panic = "abort"` a panic here would take down the whole server. + #[error("row too large: {size} bytes exceeds max {max} bytes")] + RowTooLarge { size: usize, max: usize }, } /// Convenience alias used throughout the storage crate. diff --git a/crates/storage/src/heap.rs b/crates/storage/src/heap.rs index 3b3c293..f66e6ef 100644 --- a/crates/storage/src/heap.rs +++ b/crates/storage/src/heap.rs @@ -1,5 +1,6 @@ use crate::disk::DiskManager; -use crate::page::{iter_page_slots, Page, PageType, PAGE_SIZE}; +use crate::error::StorageError; +use crate::page::{iter_page_slots, Page, PageType, MAX_ROW_DATA_SIZE, PAGE_SIZE}; use crate::types::RowId; use rustc_hash::FxHashMap; use std::io; @@ -347,6 +348,24 @@ impl HeapFile { } } + /// Reject rows that can never fit in a single page. This is the clean + /// error boundary for user-supplied oversized values: every mutation + /// entry point (`insert`, `update`) checks BEFORE touching any page so + /// no partial state (e.g. the delete half of a delete+insert update) + /// can land first. With `panic = "abort"` in release builds, the old + /// `expect("row too large for empty page")` was a remote DoS. + #[inline] + fn check_row_size(row_data: &[u8]) -> io::Result<()> { + if row_data.len() > MAX_ROW_DATA_SIZE { + return Err(StorageError::RowTooLarge { + size: row_data.len(), + max: MAX_ROW_DATA_SIZE, + } + .into()); + } + Ok(()) + } + /// Insert encoded row data. Returns RowId. /// /// Mission C Phase 1: uses the hot-page write-back cache. The common @@ -354,6 +373,7 @@ impl HeapFile { /// disk syscalls; the page stays pinned until a different page is /// touched or an explicit flush runs. pub fn insert(&mut self, row_data: &[u8]) -> io::Result { + Self::check_row_size(row_data)?; // WS1 invariant: the persistent mmap covers a snapshot of the file // taken at `enable_mmap()` time (length = num_pages * PAGE_SIZE). // It only becomes unsafe — covering a stale/short region relative @@ -426,7 +446,14 @@ impl HeapFile { // a crash, and that path (`insert_at`) re-initialises a malformed // page before use — so the hot path pays nothing here. let mut page = Page::new(page_id, PageType::Data); - let slot = page.insert(row_data).expect("row too large for empty page"); + // `check_row_size` at the top of `insert` guarantees this fits, but + // keep a graceful error (never a panic) as defence in depth. + let slot = page.insert(row_data).ok_or_else(|| { + io::Error::from(StorageError::RowTooLarge { + size: row_data.len(), + max: MAX_ROW_DATA_SIZE, + }) + })?; if page.free_space() >= 64 { self.pages_with_space.push(page_id); self.mark_free(page_id); @@ -863,6 +890,10 @@ impl HeapFile { /// Mission C Phase 1: in-place updates land on the hot page directly. /// `update_by_filter` and `update_by_pk` both route here. pub fn update(&mut self, rid: RowId, row_data: &[u8]) -> io::Result { + // Reject oversized rows BEFORE the delete+insert fallback below can + // delete the old row — otherwise a failed oversized update would + // destroy the existing data. + Self::check_row_size(row_data)?; self.ensure_hot(rid.page_id)?; { let hot = self.hot_page.as_mut().expect("ensure_hot guarantees Some"); @@ -1807,6 +1838,78 @@ mod tests { std::fs::remove_file(&path).ok(); } + #[test] + fn test_oversized_insert_returns_error_and_heap_survives() { + let (mut heap, path) = temp_heap("oversized_insert"); + let schema = user_schema(); + + // A row larger than any empty page can hold must be a clean error, + // not a panic (panic = "abort" kills the whole server process). + let big = vec![0xABu8; PAGE_SIZE]; + let err = heap.insert(&big).unwrap_err(); + assert!( + err.to_string().contains("row too large"), + "unexpected error: {err}" + ); + + // The heap must remain fully usable afterwards. + let rid = heap + .insert(&encode_row( + &schema, + &[Value::Str("ok".into()), Value::Int(1)], + )) + .unwrap(); + assert!(heap.get(rid).is_some()); + assert_eq!(heap.scan().count(), 1); + drop(heap); + std::fs::remove_file(&path).ok(); + } + + #[test] + fn test_insert_at_max_row_size_succeeds() { + use crate::page::MAX_ROW_DATA_SIZE; + let (mut heap, path) = temp_heap("max_row"); + // Exactly the max must still fit on a fresh page. + let exact = vec![0x42u8; MAX_ROW_DATA_SIZE]; + let rid = heap.insert(&exact).unwrap(); + assert_eq!(heap.get(rid).unwrap().len(), MAX_ROW_DATA_SIZE); + // One byte more must be rejected. + let over = vec![0x42u8; MAX_ROW_DATA_SIZE + 1]; + let err = heap.insert(&over).unwrap_err(); + assert!( + err.to_string().contains("row too large"), + "unexpected error: {err}" + ); + drop(heap); + std::fs::remove_file(&path).ok(); + } + + #[test] + fn test_oversized_update_returns_error_and_row_intact() { + let (mut heap, path) = temp_heap("oversized_update"); + let schema = user_schema(); + let rid = heap + .insert(&encode_row( + &schema, + &[Value::Str("Alice".into()), Value::Int(30)], + )) + .unwrap(); + let old_bytes = heap.get(rid).unwrap(); + + // Updating to an oversized row must fail cleanly WITHOUT deleting + // the old row (the delete+insert fallback must not fire). + let big = vec![0xCDu8; PAGE_SIZE]; + let err = heap.update(rid, &big).unwrap_err(); + assert!( + err.to_string().contains("row too large"), + "unexpected error: {err}" + ); + assert_eq!(heap.get(rid).unwrap(), old_bytes, "old row must survive"); + assert_eq!(heap.scan().count(), 1); + drop(heap); + std::fs::remove_file(&path).ok(); + } + #[test] fn test_multi_page_span() { let (mut heap, path) = temp_heap("multipage"); diff --git a/crates/storage/src/page.rs b/crates/storage/src/page.rs index 114f444..2f898e5 100644 --- a/crates/storage/src/page.rs +++ b/crates/storage/src/page.rs @@ -25,6 +25,15 @@ const SLOT_COUNT_SIZE: usize = 2; // u16 at bottom of page const SLOT_ENTRY_SIZE: usize = 4; // u16 offset + u16 length per slot const DELETED_MARKER: u16 = 0xFFFF; +/// Maximum encoded row size that can ever fit in a single page: a fresh +/// empty page minus the slot-count word and the row's own slot entry. +/// Anything larger must be rejected at the heap boundary as a clean +/// `StorageError::RowTooLarge` — with `panic = "abort"` in release builds, +/// a panicking insert path would otherwise kill the whole server process +/// (remote DoS via one oversized `insert`). +pub const MAX_ROW_DATA_SIZE: usize = + PAGE_SIZE - PAGE_HEADER_SIZE - SLOT_COUNT_SIZE - SLOT_ENTRY_SIZE; + /// Byte range holding the page CRC32 (WS3). Lives just after the legacy /// 16-byte header so the slot directory at the bottom of the page is /// untouched — old and new pages share the same slot/slot_count layout, diff --git a/SMOKE-AUDIT.md b/docs/audits/2026-05-26-smoke-audit.md similarity index 100% rename from SMOKE-AUDIT.md rename to docs/audits/2026-05-26-smoke-audit.md diff --git a/clients/ts/SPRINT-PLAN.md b/docs/superpowers/plans/2026-04-16-ts-client-v02-sprint.md similarity index 100% rename from clients/ts/SPRINT-PLAN.md rename to docs/superpowers/plans/2026-04-16-ts-client-v02-sprint.md From 155e13d9a17c7067fb0d5130c971b2f9893bd77d Mon Sep 17 00:00:00 2001 From: ZVN DEV <78920650+zvndev@users.noreply.github.com> Date: Tue, 9 Jun 2026 15:02:40 -0400 Subject: [PATCH 2/3] =?UTF-8?q?docs:=20ecosystem=20accuracy=20sweep=20?= =?UTF-8?q?=E2=80=94=20sync=20all=20surfaces=20to=20v0.4.5=20reality?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - site: v0.2.0 banners -> v0.4.5, MSRV 1.80 -> 1.93, all 14 benchmark workloads shown, transactions + multi-user auth + 'when not to use' sections added, repo links -> ZVN-DEV/powdb - Cargo.toml: homepage was a 404 (zvndev -> zvn-dev.github.io) - README: bench gate honestly described as local/on-demand (not a CI merge gate), auth+backup crates in tree, full env-var table - AGENTS.md: 8 crates, multi-user auth, shipped features list fixed, small-model-tested gotchas added (aggregate-keyword aliases 9/10 -> 10/10 haiku success; line-oriented REPL) - SECURITY/RELEASES/CONTRIBUTING: two auth modes + readonly caveat, six-crate publish order + ghcr image, real required CI checks - deploy examples: fly.toml POWDB_BIND=0.0.0.0 (was unreachable), docker-compose pointer fixed, railway backup bullet - clients/ts: multi-user-server incompatibility documented, demo fixes - archives: SMOKE-AUDIT -> docs/audits/, TS sprint plan -> plans/, historical banners on pre-implementation design docs - new: docs/gtm-strategy.md, docs/benchmarks/2026-06-09 local snapshot Co-Authored-By: Claude Fable 5 --- AGENTS.md | 19 +- CLAUDE.md | 7 +- CONTRIBUTING.md | 10 +- Cargo.toml | 2 +- README.md | 16 +- RELEASES.md | 20 +- SECURITY.md | 18 +- clients/ts/CHANGELOG.md | 2 +- clients/ts/README.md | 5 + clients/ts/demo/demo.ts | 8 +- docs/audits/2026-05-26-smoke-audit.md | 15 +- .../2026-06-09-local-apple-silicon.md | 53 +++++ docs/design/powdb-implementation-brief.md | 2 + docs/design/powdb-wire-protocol.md | 2 + docs/getting-started.md | 12 +- docs/gtm-strategy.md | 191 ++++++++++++++++++ docs/powdb-vs-sqlite.md | 10 +- .../plans/2026-04-16-ts-client-v02-sprint.md | 3 + examples/deploy/README.md | 16 +- examples/deploy/fly.toml | 2 + examples/deploy/railway/README.md | 8 +- site/getting-started.html | 36 +++- site/index.html | 59 +++++- site/powql.html | 27 ++- 24 files changed, 472 insertions(+), 71 deletions(-) create mode 100644 docs/benchmarks/2026-06-09-local-apple-silicon.md create mode 100644 docs/gtm-strategy.md diff --git a/AGENTS.md b/AGENTS.md index c08a9b3..f6f6169 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -19,13 +19,13 @@ The measurable result: 3–10× faster than SQLite on aggregate and scan workloa - **Embedded / edge / serverless workloads** where query latency is a tight budget and you don't want SQLite's quirks. - **Single-node analytics** over tables that fit on disk. The scan path is zero-syscall (mmap) and filters are compiled to byte-level predicates. - **You control both sides** (the DB and the app). PowDB has no Postgres wire protocol, no ODBC, no legacy compatibility. The client is a TCP binary protocol or an in-process Engine. -- **You want to read the code.** Four crates, ~20K lines, no generated parsers, no plan-language IR. +- **You want to read the code.** Eight crates, ~45K lines, no generated parsers, no plan-language IR. ### When it's *not* the right choice - You need a drop-in Postgres/MySQL replacement. PowQL is a different language; there is **no SQL compatibility layer**, and that is a deliberate design decision (the translation tier is the thing we're removing). - You need multi-node replication, sharding, or Raft-style consensus. Single-node only today. -- You need fine-grained ACLs, row-level security, or multi-tenant isolation. Auth is a single shared password. +- You need fine-grained ACLs, row-level security, or multi-tenant isolation. Auth is named users with coarse roles (admin/readwrite/readonly) since 0.4.5 — shared-password mode still available — but nothing per-table or per-row. - You need user-defined functions, stored procedures, or triggers. --- @@ -109,6 +109,7 @@ Compare SQL: `SELECT name, age FROM User WHERE age > 25 ORDER BY age DESC LIMIT | `AND`, `OR`, `NOT` | `and`, `or`, `not` (lowercase) | | `User.posts` (link navigation) | not yet implemented | | `let x := ...` | not yet implemented | +| `count: count(.name)` (aggregate keyword as alias) | fails `expected alias name` — `sum:` fails too; use `n:`, `cnt:`, `total:` | --- @@ -116,7 +117,7 @@ Compare SQL: `SELECT name, age FROM User WHERE age > 25 ORDER BY age DESC LIMIT Canonical type names: `str`, `int`, `float`, `bool`, `datetime`, `uuid`, `bytes`. -**Footgun:** the executor's type resolver falls back to `TypeId::Str` for any unknown name (`crates/query/src/executor.rs`), so `string`, `varchar`, or a typo silently produces a Str column with no error. Always use the canonical names above. +**Footgun:** the executor's type resolver falls back to `TypeId::Str` for any unknown name (`crates/query/src/executor/`), so `string`, `varchar`, or a typo silently produces a Str column with no error. Always use the canonical names above. `required` is a prefix keyword on the field, not a `!` suffix: `required name: str`, never `name: str!`. @@ -128,7 +129,7 @@ These are the design moves that buy the speedup. Understanding them keeps you fr 1. **Planner is a pure function.** It does not touch the catalog — it emits `RangeScan` speculatively, and the executor lowers to `Filter(SeqScan)` at runtime if no index exists. This keeps the parser → plan pipeline allocation-free for cache hits. 2. **Plan cache hashes canonical PowQL.** Literals are substituted at lookup time (FNV-1a hash, `crates/query/src/plan_cache.rs`). A repeated `User filter .id = ` reuses the same plan for all N. -3. **Compiled integer predicates.** `Filter(SeqScan)` on simple numeric predicates compiles into a branch-free byte-level check that skips full row decoding. See `execute_plan` fast paths in `crates/query/src/executor.rs`. +3. **Compiled integer predicates.** `Filter(SeqScan)` on simple numeric predicates compiles into a branch-free byte-level check that skips full row decoding. See `execute_plan` fast paths in `crates/query/src/executor/` (module dir). 4. **mmap-based scans.** The storage layer exposes `try_for_each_row_raw` over memory-mapped heap files. Early termination is a `return ControlFlow::Break`. 5. **Slotted 4KB pages + persistent B+tree indexes.** Standard, but the index format (BIDX, binary) is crash-safe and survives restart with no rebuild. 6. **WAL with group commit at statement boundaries.** Writes are durable by default; throughput is maintained by batching. @@ -157,6 +158,8 @@ cargo run --release -p powdb-cli # embedded REPL cargo run --release -p powdb-cli -- --remote host:5433 --password ``` +**The REPL is line-oriented.** A statement split across lines fails to parse — write each statement on one line. + ### TCP server ```bash @@ -201,9 +204,9 @@ Return shapes: ## What's shipped vs. what's planned -Shipped: joins (inner/left/right/cross, nested-loop + hash), GROUP BY + HAVING, DISTINCT, UNION / UNION ALL, subqueries (IN, EXISTS, correlated), CASE, LIKE, BETWEEN, IN-list, window functions (ROW_NUMBER, RANK, DENSE_RANK, SUM/AVG/COUNT/MIN/MAX over partition), arithmetic, string/math/datetime scalars, CAST, COALESCE, materialized views with auto-refresh, upsert, prepared queries with literal substitution, explicit transactions (`begin` / `commit` / `rollback`), password auth, TLS (`POWDB_TLS_CERT` / `POWDB_TLS_KEY`), WAL + crash recovery, persistent indexes. +Shipped: joins (inner/left/right/cross, nested-loop + hash), GROUP BY + HAVING, DISTINCT, UNION / UNION ALL, subqueries (IN, EXISTS, correlated), CASE, LIKE, BETWEEN, IN-list, window functions (ROW_NUMBER, RANK, DENSE_RANK, SUM/AVG/COUNT/MIN/MAX over partition), arithmetic, string/math/datetime scalars, CAST, COALESCE (`??`), materialized views with auto-refresh, upsert, multi-row INSERT, prepared queries with literal substitution, explicit transactions (`begin` / `commit` / `rollback`), password auth + multi-user auth (named users, admin/readwrite/readonly roles), TLS (`POWDB_TLS_CERT` / `POWDB_TLS_KEY`), WAL + crash recovery, persistent indexes, backup/restore (full/incremental/PITR, offline). -Planned (design doc only — don't use): link navigation (`User.posts`), `let` bindings, default operator (`??`), UDFs, per-row permissions, replication. +Planned (design doc only — don't use): link navigation (`User.posts`), `let` bindings, UDFs, per-row permissions, replication. --- @@ -211,9 +214,7 @@ Planned (design doc only — don't use): link navigation (`User.posts`), `let` b Build: `cargo build --workspace`. Test: `cargo test --workspace`. Lint: `cargo clippy --workspace --all-targets -- -D warnings`. Format: `cargo fmt --all`. -CI gates on `main`: -- `clippy + fmt + test` — `.github/workflows/ci.yml` -- `criterion + regression gate` — `.github/workflows/bench.yml` (blocks merges if any of 7 load-bearing workloads regress >7% against the checked-in baseline) +CI gates on `main` (all in `.github/workflows/ci.yml`): clippy/fmt/test (2-OS matrix), miri, asan, cargo audit, MSRV consistency, examples-smoke. `.github/workflows/bench.yml` (criterion + regression gate) is **manual-only** (`workflow_dispatch`) and NOT a merge gate — run the gate locally instead (see above). Internal docs: - `CLAUDE.md` — codebase guide for Claude Code (architecture, crate graph, common patterns) diff --git a/CLAUDE.md b/CLAUDE.md index 2e73a51..f1699d4 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -19,6 +19,9 @@ PowDB is a from-scratch database engine with its own query language (PowQL). No powdb-cli ──→ powdb-server ──→ powdb-query ──→ powdb-storage ↑ ↑ powdb-bench powdb-compare + +powdb-auth ←── powdb-server, powdb-cli (user store + roles; no inter-crate deps) +powdb-backup ←── powdb-cli (depends on powdb-storage + powdb-query) ``` ### Query Pipeline @@ -30,7 +33,7 @@ PowQL text → Lexer (token stream) → Parser (AST) → Planner (PlanNode tree) - **Lexer** (`crates/query/src/lexer.rs`): Tokenizes PowQL input - **Parser** (`crates/query/src/parser.rs`): Recursive descent, produces `Statement` AST - **Planner** (`crates/query/src/planner.rs`): Pure function (no catalog access), produces `PlanNode` tree. Speculatively emits `RangeScan` for range inequalities -- **Executor** (`crates/query/src/executor.rs`): Runs plans against the storage engine. Has fast paths for common patterns (count, project+limit, sort+limit, agg, update, delete). Lowers `RangeScan` → `Filter(SeqScan)` at runtime when no index exists +- **Executor** (`crates/query/src/executor/` module dir): Runs plans against the storage engine. Has fast paths for common patterns (count, project+limit, sort+limit, agg, update, delete). Lowers `RangeScan` → `Filter(SeqScan)` at runtime when no index exists - **Plan Cache** (`crates/query/src/plan_cache.rs`): FNV-1a hash, stores canonical plans, substitutes literals at lookup time ### Storage Engine @@ -81,7 +84,7 @@ cargo run -p powdb-bench --bin compare 3. Add parser production to `crates/query/src/parser.rs` 4. Add plan node (if needed) to `crates/query/src/plan.rs` 5. Add planner case to `crates/query/src/planner.rs` -6. Add executor case to `crates/query/src/executor.rs` +6. Add executor case to `crates/query/src/executor/` (start in `mod.rs` / `plan_exec.rs`) ### Adding an executor fast path Fast paths match on specific `PlanNode` shapes in `execute_plan()`. Pattern-match the plan tree and handle it before the generic recursive executor. Always verify with benchmarks. diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index bca7ce7..bb12b3e 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -29,6 +29,8 @@ cargo run --release -p powdb-compare # wide bench vs SQLite + Postgres (add --f ``` crates/storage # slotted pages, B+ tree, WAL, buffer pool, catalog crates/query # lexer, parser, planner, executor (Engine) +crates/auth # user store, roles, argon2id password hashing +crates/backup # offline backup/restore (full, incremental, PITR) crates/server # Tokio TCP server + binary wire protocol crates/cli # rustyline REPL (embedded + remote mode) crates/bench # criterion benchmarks + regression gate @@ -49,7 +51,7 @@ clients/ts # TypeScript client + demo ### Branch protection on `main` - PRs are required (no direct pushes) -- 7 status checks must pass: clippy + fmt + test (x2 OS matrix), miri, asan, audit, MSRV consistency, and the bench regression gate +- 7 status checks must pass, all from `ci.yml`: clippy + fmt + test (x2 OS matrix), miri, asan, audit, MSRV consistency, and examples-smoke - Force-push is rejected by branch protection Admin bypass exists for break-glass scenarios (security patches, recovering from a broken state). **Do not use it for routine work** — routine work goes through PRs even when bypass is technically available. @@ -74,11 +76,13 @@ PRs must pass these gates (see `.github/workflows/`): - **asan** — AddressSanitizer run - **audit** — `cargo audit` against the advisory database - **msrv-consistency** — verifies the declared MSRV (`1.93`) builds -- **criterion + regression gate** — benchmark must not regress beyond thresholds (`.github/workflows/bench.yml`) +- **examples-smoke** — terraform validate + compose config + dev.sh cycle on the deploy examples + +The criterion benchmark suite (`.github/workflows/bench.yml`) is **manual-only** (`workflow_dispatch`) and is *not* a required PR gate — shared-runner noise makes it unreliable as a blocking check. Run the regression gate locally instead (below). ## Benchmark Regression Gate -The criterion gate compares each workload's median against baselines in `crates/bench/baseline/main.json`. Thresholds vary by workload (7-20%). +The criterion gate compares each workload's median against baselines in `crates/bench/baseline/main.json`. Thresholds vary by workload (7-20%). Run it locally with `cargo bench -p powdb-bench && cargo run -p powdb-bench --bin compare`, or on demand in CI with `gh workflow run bench.yml`. If you intentionally change performance characteristics: ```bash diff --git a/Cargo.toml b/Cargo.toml index e406eed..fb8fb84 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -8,7 +8,7 @@ edition = "2021" rust-version = "1.93" license = "MIT" repository = "https://github.com/zvndev/powdb" -homepage = "https://zvndev.github.io/powdb/" +homepage = "https://zvn-dev.github.io/powdb/" readme = "README.md" [workspace.dependencies] diff --git a/README.md b/README.md index 2a6242b..f13c038 100644 --- a/README.md +++ b/README.md @@ -1,7 +1,6 @@ # PowDB [![CI](https://github.com/zvndev/powdb/actions/workflows/ci.yml/badge.svg)](https://github.com/zvndev/powdb/actions/workflows/ci.yml) -[![bench](https://github.com/zvndev/powdb/actions/workflows/bench.yml/badge.svg)](https://github.com/zvndev/powdb/actions/workflows/bench.yml) [![crates.io](https://img.shields.io/crates/v/powdb-cli.svg)](https://crates.io/crates/powdb-cli) [![docs.rs](https://img.shields.io/docsrs/powdb-query)](https://docs.rs/powdb-query) [![MSRV](https://img.shields.io/badge/MSRV-1.93-blue)](https://github.com/zvndev/powdb/blob/main/Cargo.toml) @@ -30,7 +29,7 @@ Evaluating PowDB? Start with the honest comparison: [PowDB vs SQLite -- when to | Filter + project | `SELECT name, age FROM User WHERE age > 25` | `User filter .age > 25 { .name, .age }` | | Sort + limit | `SELECT * FROM User ORDER BY age DESC LIMIT 10` | `User order .age desc limit 10` | | Aggregate with filter | `SELECT AVG(age) FROM User WHERE city = 'NYC'` | `avg(User filter .city = "NYC" { .age })` | -| Group + having | `SELECT status, COUNT(*) FROM User GROUP BY status HAVING COUNT(*) > 5` | `User group .status having count(*) > 5 { .status, count(*) }` | +| Group + having | `SELECT status, COUNT(name) FROM User GROUP BY status HAVING COUNT(name) > 5` | `User group .status having count(.name) > 5 { .status, n: count(.name) }` | PowQL uses `.field` dot syntax for column references, `:=` for assignments, and `"double quotes"` for strings. The pipeline reads like a sentence: *"User, filter age greater than 25, order by name, limit 10, give me name and age."* @@ -43,7 +42,7 @@ Full language reference: [docs/POWQL.md](https://github.com/zvndev/powdb/blob/ma cargo install powdb-cli cargo install powdb-server -# TypeScript client (Node 18+) +# TypeScript client (Node 18+) — versions independently of the server crates (currently 0.3.x) npm install @zvndev/powdb-client # Prebuilt binaries (linux x86_64, macos aarch64) @@ -201,10 +200,14 @@ if (result.kind === "rows") console.table(result.rows); | Variable | Default | Description | |---|---|---| | `POWDB_PORT` | `5433` | TCP port for the server | +| `POWDB_BIND` | `127.0.0.1` | Interface to bind; set `0.0.0.0` behind a platform proxy (Fly, Railway) | | `POWDB_DATA` | `./powdb_data` | Data directory (heap files, WAL, catalog, indexes) | | `POWDB_PASSWORD` | *(none)* | Shared password required on connect when no named users are defined (set as env var) | | `POWDB_ADMIN_USER` / `POWDB_ADMIN_PASSWORD` | *(none)* | Bootstrap an `admin` user on startup when both are set and that user does not yet exist (password never logged) | +| `POWDB_TLS_CERT` / `POWDB_TLS_KEY` | *(none)* | Paths to PEM cert + key; when both are set the server serves TLS | | `POWDB_REQUIRE_TLS` | *(off)* | When set (`1`/`true`), refuse to start if a password is configured without TLS | +| `POWDB_IDLE_TIMEOUT` | `300` | Seconds before an idle connection is closed | +| `POWDB_QUERY_TIMEOUT` | `30` | Per-query timeout in seconds | | `POWDB_QUERY_MEMORY_LIMIT` | `268435456` | Per-query memory budget in bytes (256 MiB); over-budget queries error instead of OOM-killing the server | | `RUST_LOG` | `info` | Log level (`debug`, `trace` for per-query timings) | @@ -232,6 +235,7 @@ For a self-hostable starting point, see [`examples/deploy/fly.toml`](https://git - Memory-mapped reads (zero-syscall scan path) - Compiled integer predicates (branch-free filter at the byte level) - Thread-safe concurrent reads via pread(2)/pwrite(2) +- Backup & restore: full + incremental + coarse point-in-time recovery (offline; `powdb-cli backup` / `restore` — see [docs/backup-and-restore.md](docs/backup-and-restore.md)) **Query engine** - PowQL parser + planner + executor with plan cache (FNV-1a hashing, literal substitution) @@ -273,6 +277,8 @@ For a self-hostable starting point, see [`examples/deploy/fly.toml`](https://git crates/ storage/ Heap files, B+tree, WAL, catalog, page cache, row encoding query/ Lexer, parser, planner, executor (Engine), plan cache + auth/ User store, roles, argon2id password hashing + backup/ Offline backup/restore (full, incremental, PITR) server/ Tokio TCP server + binary wire protocol cli/ Interactive REPL (embedded + remote modes) bench/ Criterion benchmarks + regression gate @@ -283,13 +289,15 @@ The engine is `powdb_query::executor::Engine`. It owns a `Catalog` (which owns ` ## Benchmarks -PowDB has a CI-enforced regression gate that blocks PRs to `main` if any workload regresses beyond its threshold. Run locally: +PowDB has a benchmark regression gate that compares every workload against checked-in baselines. Run it locally before and after touching a hot path: ```bash cargo bench -p powdb-bench # criterion suite (~60s) cargo run --release -p powdb-bench --bin compare # regression gate ``` +The gate also runs on-demand in CI via `workflow_dispatch` (`.github/workflows/bench.yml`) — it is **not** a required PR gate, because shared-runner noise makes it unreliable as a blocking check. The required PR gates live in `ci.yml`. + Run the PowDB vs SQLite comparison bench: ```bash diff --git a/RELEASES.md b/RELEASES.md index 2219872..1ffff78 100644 --- a/RELEASES.md +++ b/RELEASES.md @@ -3,9 +3,10 @@ Every PowDB release ships to the following registries and platforms. When cutting a release, follow the checklist at the bottom. -> **Current release: v0.4.4** (all four crates live on crates.io). +> **Current release: v0.4.5** (all six crates live on crates.io, including the +> new `powdb-auth` and `powdb-backup`). > **v0.4.1, v0.4.2, and v0.4.3 are yanked** for crash-recovery data-loss bugs; -> 0.4.4 fixes them and adds a standing durability regression suite. See +> 0.4.4 fixed them and added a standing durability regression suite. See > `CHANGELOG.md`. ## Registries @@ -13,10 +14,13 @@ When cutting a release, follow the checklist at the bottom. | Target | Package | Registry URL | |--------|---------|-------------| | **crates.io** | `powdb-storage` | https://crates.io/crates/powdb-storage | +| **crates.io** | `powdb-auth` | https://crates.io/crates/powdb-auth | | **crates.io** | `powdb-query` | https://crates.io/crates/powdb-query | +| **crates.io** | `powdb-backup` | https://crates.io/crates/powdb-backup | | **crates.io** | `powdb-server` | https://crates.io/crates/powdb-server | | **crates.io** | `powdb-cli` | https://crates.io/crates/powdb-cli | | **npm** | `@zvndev/powdb-client` | https://www.npmjs.com/package/@zvndev/powdb-client | +| **ghcr.io** | `ghcr.io/zvn-dev/powdb` (Docker image, `latest` + `vX.Y.Z` tags) | https://github.com/orgs/ZVN-DEV/packages | ## GitHub Releases @@ -35,9 +39,11 @@ when a `v*` tag is pushed. Inter-crate dependencies require publishing in this order: 1. `powdb-storage` (no inter-crate deps) -2. `powdb-query` (depends on storage) -3. `powdb-server` (depends on storage + query) -4. `powdb-cli` (depends on storage + query + server) +2. `powdb-auth` (no inter-crate deps) +3. `powdb-query` (depends on storage) +4. `powdb-backup` (depends on storage + query) +5. `powdb-server` (depends on storage + query + auth) +6. `powdb-cli` (depends on storage + query + server + backup + auth) Non-publishable crates (`publish = false`): `powdb-compare`, `powdb-bench`, `powdb-query-fuzz`. @@ -45,12 +51,14 @@ Non-publishable crates (`publish = false`): `powdb-compare`, `powdb-bench`, `pow ``` [ ] Update workspace version in root Cargo.toml -[ ] Update inter-crate dep versions in query/server/cli Cargo.toml +[ ] Update inter-crate dep versions in query/backup/server/cli Cargo.toml [ ] Update clients/ts/package.json version [ ] Update CHANGELOG.md [ ] Commit: "chore: release vX.Y.Z" [ ] cargo publish -p powdb-storage +[ ] cargo publish -p powdb-auth [ ] cargo publish -p powdb-query +[ ] cargo publish -p powdb-backup [ ] cargo publish -p powdb-server [ ] cargo publish -p powdb-cli [ ] cd clients/ts && npm publish --access public diff --git a/SECURITY.md b/SECURITY.md index 33c8103..c1d757d 100644 --- a/SECURITY.md +++ b/SECURITY.md @@ -4,7 +4,8 @@ | Version | Supported | | --------------- | ------------------ | -| 0.4.4 | :white_check_mark: | +| 0.4.5 | :white_check_mark: | +| 0.4.4 | :x: (superseded) | | 0.4.1 – 0.4.3 | :x: (yanked) | | 0.4.0 | :white_check_mark: | | 0.3.x | :white_check_mark: | @@ -14,7 +15,7 @@ > **v0.4.1, v0.4.2, and v0.4.3 are yanked** for data-loss bugs in crash > recovery and have been replaced by **v0.4.4**, which adds a permanent > durability regression suite. If you are on any of those three versions, -> upgrade to 0.4.4. See `CHANGELOG.md` for details. +> upgrade to the latest release (0.4.5). See `CHANGELOG.md` for details. ## Reporting a Vulnerability @@ -52,15 +53,20 @@ When both are set, the server requires TLS for all connections. When unset, the ## Authentication -PowDB uses single-password authentication via the `POWDB_PASSWORD` environment variable. When set, clients must authenticate before executing queries. +PowDB supports two authentication modes: + +1. **Shared password** — set the `POWDB_PASSWORD` environment variable. All clients authenticate with the same shared secret. Applies only when no named users are defined. +2. **Named users with roles** (since 0.4.5) — users with `admin`, `readwrite`, or `readonly` roles, managed via `powdb-cli useradd` / `passwd` / `userdel`. Passwords are stored as argon2id hashes only (`auth.json` in the data directory, `0600` on Unix). When `POWDB_ADMIN_USER` and `POWDB_ADMIN_PASSWORD` are both set, the server bootstraps an initial admin on startup without the CLI. Once any user is defined, the shared password is no longer used. + +In both modes: - **Rate limiting**: authentication attempts are rate-limited to prevent brute-force attacks. - **Pre-auth payload limits**: the server enforces frame size limits on unauthenticated connections to prevent resource exhaustion. - **Connection limits**: the server enforces a maximum number of concurrent connections. -There is no per-user auth or RBAC. The password should be treated as a shared secret for all clients connecting to a given server instance. +> **Note on the `readonly` role:** in releases up to and including 0.4.5, role storage is in place but read-only restrictions are **not enforced** at the query layer — do not rely on the `readonly` role as a security boundary against writes on those versions. Enforcement at the server dispatch layer ships in the next release: write statements from `readonly` users are rejected with `permission denied`, and unknown roles fail closed. ## Known Limitations -- Authentication is single-password. There is no per-user auth or RBAC. -- The query parser has a nesting depth limit but no query timeout mechanism yet. +- Roles are coarse (`admin` / `readwrite` / `readonly`). There are no per-table ACLs, row-level security, or multi-tenant isolation; `readonly` enforcement is absent in ≤0.4.5 (see note above). +- The query parser has a nesting depth limit. Runaway queries are bounded by `POWDB_QUERY_TIMEOUT` (default 30s) and the per-query memory budget (`POWDB_QUERY_MEMORY_LIMIT`). diff --git a/clients/ts/CHANGELOG.md b/clients/ts/CHANGELOG.md index 352e4bc..a74bc18 100644 --- a/clients/ts/CHANGELOG.md +++ b/clients/ts/CHANGELOG.md @@ -6,7 +6,7 @@ All notable changes to `@zvndev/powdb-client`. | Client version | Compatible PowDB server | Notes | |---|---|---| -| 0.3.x | 0.3.x – 0.4.x | Wire protocol v1. The client warns only on a major-version mismatch, so any `0.x` server connects. Minor server bumps may add new opcodes; the client tolerates unknown response codes by surfacing `PowDBError`. Pin both ends. | +| 0.3.x | 0.3.x – 0.4.x | Wire protocol v1. The client warns only on a major-version mismatch, so any `0.x` server connects. Minor server bumps may add new opcodes; the client tolerates unknown response codes by surfacing `PowDBError`. Pin both ends. **Caveat:** the 0.3.x client has no `user` option, so it cannot authenticate to a 0.4.5+ server running in **multi-user mode** (the server requires a username once any named user is defined). Shared-password mode works fine. | The client warns on major-version mismatch with the server during the handshake. Within `0.x`, treat any minor-version skew between client and diff --git a/clients/ts/README.md b/clients/ts/README.md index 367de10..0a49576 100644 --- a/clients/ts/README.md +++ b/clients/ts/README.md @@ -243,6 +243,11 @@ Returns a `Promise`. Options: | `connectTimeoutMs` | `number` | `5000` | Connection timeout in milliseconds | | `tls` | `boolean \| tls.ConnectionOptions` | `false` | Enable TLS; `true` uses system defaults, or pass a `tls.connect` options object | +> **Multi-user servers:** PowDB server 0.4.5 added named users with roles. The +> 0.3.x client has no `user` option, so it **cannot authenticate to a server +> running in multi-user mode** (the server requires a username once any user +> is defined). Shared-password mode (`POWDB_PASSWORD`) works as before. + ### `client.query(query, opts?)` Sends a PowQL query and returns a `Promise`: diff --git a/clients/ts/demo/demo.ts b/clients/ts/demo/demo.ts index 52a3aad..eeaaaa6 100644 --- a/clients/ts/demo/demo.ts +++ b/clients/ts/demo/demo.ts @@ -28,13 +28,14 @@ async function main() { try { // Use a unique table name per run so reruns don't collide on a - // persistent server. PowQL doesn't have DROP TABLE (yet). + // persistent server. (PowQL has `drop `, but a unique name keeps + // the demo side-effect free even if a run is interrupted.) const tableName = `Demo${Date.now().toString(36)}`; const table = ident(tableName); console.log(`→ creating type ${tableName}`); await run( client, - powql`type ${table} { required name: string, required age: int, city: string }`, + powql`type ${table} { required name: str, required age: int, city: str }`, ); console.log(`→ inserting rows`); @@ -101,6 +102,9 @@ function printResult(result: QueryResult): void { case "ok": console.log(` → ok (${result.affected} affected)`); break; + case "message": + console.log(` → ${result.message}`); + break; } } diff --git a/docs/audits/2026-05-26-smoke-audit.md b/docs/audits/2026-05-26-smoke-audit.md index bf170af..8f8c11a 100644 --- a/docs/audits/2026-05-26-smoke-audit.md +++ b/docs/audits/2026-05-26-smoke-audit.md @@ -1,13 +1,14 @@ # PowDB Smoke Audit — 2026-05-26 -> **HISTORICAL — superseded as of v0.4.4 (2026-06-05).** This is a point-in-time +> **HISTORICAL — superseded (current release: v0.4.5).** This is a point-in-time > smoke audit of the pre-release 0.4.0 build. Nearly every finding has since been > resolved: ROLLBACK now undoes heap writes (verified by transaction tests in -> `crates/query/src/executor/tests.rs`), the CHANGELOG has entries through 0.4.4, -> all four crates are published on crates.io at 0.4.4, version pins and the MSRV -> are current, and issue/PR templates exist. The three later durability P0s that -> 0.4.4 fixed are not in this audit — see `CHANGELOG.md`. Kept unaltered for -> provenance; do not treat its grades or status claims as current. +> `crates/query/src/executor/tests.rs`), the CHANGELOG has entries through 0.4.5, +> all six crates (including the new powdb-auth and powdb-backup) are published on +> crates.io at 0.4.5, version pins and the MSRV are current, and issue/PR +> templates exist. The three later durability P0s that 0.4.4 fixed are not in +> this audit — see `CHANGELOG.md`. Kept unaltered for provenance; do not treat +> its grades or status claims as current. ## Executive Summary @@ -296,7 +297,7 @@ ## Screenshots -All 12 screenshots saved to `screenshots/`: +12 screenshots were captured during the audit to a local `screenshots/` directory (gitignored; not retained in the repo): - `github-pages-homepage.png` — Landing page (polished, professional) - `github-repo.png` — Repository page (good metadata) - `github-org.png` — User profile page diff --git a/docs/benchmarks/2026-06-09-local-apple-silicon.md b/docs/benchmarks/2026-06-09-local-apple-silicon.md new file mode 100644 index 0000000..2b53aeb --- /dev/null +++ b/docs/benchmarks/2026-06-09-local-apple-silicon.md @@ -0,0 +1,53 @@ +# Local bench snapshot — 2026-06-09 (Apple Silicon) + +Captured from `crates/compare/results.csv` (2026-06-09, macOS arm64, release +build). This file preserves the run because `results.csv` is in `.gitignore` +(intentionally: it's a bench artifact, rewritten on every run). + +> **These are local laptop numbers, NOT the CI regression baseline.** +> `crates/bench/baseline/main.json` is captured on CI hardware (GitHub +> ubuntu x86) and must never be compared against — or rebaselined from — +> arm64 laptop runs. + +## Methodology + +- Harness: `cargo run --release -p powdb-compare` +- Fixture: 100,000 rows on each engine, identical schema +- Engines: PowDB (in-process, `WalSyncMode::Off`) + SQLite (in-process, + `:memory:`) — both entirely in RAM +- Each workload: ns/op reported as median + +## Results + +| workload | PowDB | SQLite | ratio (SQLite÷PowDB) | +| --- | ---: | ---: | ---: | +| point_lookup_indexed | 54 ns | 190 ns | 3.5x | +| point_lookup_nonindexed | 239.5 us | 301.9 us | 1.3x | +| scan_filter_count | 233.4 us | 1.31 ms | 5.6x | +| scan_filter_project_top100 | 6.4 us | 8.2 us | 1.3x | +| scan_filter_sort_limit10 | 2.10 ms | 6.02 ms | 2.9x | +| agg_sum | 187.7 us | 1.40 ms | 7.5x | +| agg_avg | 280.2 us | 1.65 ms | 5.9x | +| agg_min | 137.0 us | 1.60 ms | 11.7x | +| agg_max | 142.6 us | 1.38 ms | 9.7x | +| multi_col_and_filter | 1.36 ms | 3.05 ms | 2.2x | +| insert_single | 297 ns | 627 ns | 2.1x | +| insert_batch_1k | 141 ns | 203 ns | 1.4x | +| update_by_pk | 42 ns | 261 ns | 6.3x | +| update_by_filter | 1.74 ms | 4.42 ms | 2.5x | +| delete_by_filter | 1.18 ms | 1.59 ms | 1.3x | + +**Score: PowDB faster on all 15 workloads on this run.** Contrast with the +[2026-04-07 snapshot](2026-04-07-wide-bench-snapshot.md) (5 wins, 10 losses) +— the write-path and point-lookup losses identified there have since been +closed. + +## Reproducibility + +```bash +cargo run --release -p powdb-compare +# rewrites crates/compare/results.csv +``` + +Re-running will produce slightly different absolute numbers (±10-15% on an +idle laptop), but the verdicts are stable. diff --git a/docs/design/powdb-implementation-brief.md b/docs/design/powdb-implementation-brief.md index dd9b09c..6b97146 100644 --- a/docs/design/powdb-implementation-brief.md +++ b/docs/design/powdb-implementation-brief.md @@ -1,5 +1,7 @@ # PowDB: Complete implementation brief +> **Historical design doc (pre-implementation).** For current architecture see `CLAUDE.md` / `AGENTS.md`. + This document contains everything needed to implement PowDB from scratch. It is the single source of truth — all architectural decisions are backed by production benchmarks and explained with rationale. diff --git a/docs/design/powdb-wire-protocol.md b/docs/design/powdb-wire-protocol.md index 159d132..1e38992 100644 --- a/docs/design/powdb-wire-protocol.md +++ b/docs/design/powdb-wire-protocol.md @@ -1,5 +1,7 @@ # PowDB: Wire protocol and engine architecture +> **Historical design doc (pre-implementation).** For current architecture see `CLAUDE.md` / `AGENTS.md`. + ## Engine architecture PowDB is a library first, server second. The core engine is a single library diff --git a/docs/getting-started.md b/docs/getting-started.md index 7bd9e05..0f46726 100644 --- a/docs/getting-started.md +++ b/docs/getting-started.md @@ -44,7 +44,7 @@ cargo run --release -p powdb-cli You should see: ``` -PowDB v0.4.4 — embedded mode +PowDB v0.4.5 — embedded mode Data directory: ./powdb_data Type PowQL queries. Use Ctrl-D to exit. @@ -119,6 +119,8 @@ powql> insert User { name := "Grace", email := "grace@example.com" } 1 row affected ``` +> **Multi-row insert:** you can also insert many rows in one statement by separating row blocks with commas -- `insert User { ... }, { ... }, { ... }`. One statement means one WAL fsync and one network round trip, and validation is all-or-nothing. See [INSERT in the PowQL reference](POWQL.md#insert). + > **Note:** Each autocommit `insert` fsyncs to the write-ahead log for durability, which caps single-row inserts at roughly a few hundred per second on real disks. For bulk loads, wrap many inserts in a `begin` / `commit` transaction -- they share a single fsync at commit and run dozens of times faster, still fully durable. See [Transactions](POWQL.md#transactions). --- @@ -443,15 +445,15 @@ cargo run --release -p powdb-cli -- --remote localhost:5433 Output: ``` -PowDB v0.4.4 — remote mode +PowDB v0.4.5 — remote mode Connecting to localhost:5433 ... -Connected to db `main` (server v0.4.4) +Connected to db `main` (server v0.4.5) Type PowQL queries. Use Ctrl-D to exit. powql> ``` -From here, the same PowQL statements work as embedded mode. Some status messages are normalized by the wire protocol -- for example, DDL statements return an affected-row status such as `0 rows affected` in remote mode instead of the embedded REPL's `type User created` message. The server handles concurrent readers and uses a write-ahead log for durability. +From here, the same PowQL statements work as embedded mode -- DDL statements return the same friendly status messages too (e.g. `type User created`). The server handles concurrent readers and uses a write-ahead log for durability. ### Password authentication @@ -532,3 +534,5 @@ relying on the bootstrap env vars. This tutorial covered the basics: tables, inserts, queries, aggregates, updates, indexes, and deletes. PowDB supports much more, including joins, group by, subqueries, materialized views, and set operations. See the full language reference: [PowQL Reference](POWQL.md) + +Running in production? Set up backups: [Backup & restore](backup-and-restore.md) covers full and incremental backups plus coarse point-in-time recovery (offline -- stop the server first). diff --git a/docs/gtm-strategy.md b/docs/gtm-strategy.md new file mode 100644 index 0000000..84766c8 --- /dev/null +++ b/docs/gtm-strategy.md @@ -0,0 +1,191 @@ +# PowDB Go-To-Market Strategy + +**Author:** Product Lead (strategist mode) +**Date:** 2026-06-09 +**Version reviewed:** v0.4.5 (crates.io, npm, ghcr, GitHub Pages) +**Status of the product:** Real engine, honest benchmarks, pre-1.0, single-node, tiny team (ZVN/Kirby). + +--- + +## The Take (read this first) + +PowDB is a genuinely fast, genuinely pure-Rust embedded engine with an honest 3–10x SQLite win on the workloads it's built for. That is real and defensible at the *engine* level. But the headline pitch — "faster than SQLite" — is a **losing wedge**, because (a) SQLite is "fast enough" for ~95% of the people who pick it, and (b) the moment performance is the axis, **Turso Database (the from-scratch Rust SQLite rewrite, beta as of May 2026) eats your exact positioning while keeping SQL compatibility.** Same "pure Rust, no C, from scratch" story, but a user keeps every ORM, driver, and tool. On a pure perf-vs-SQLite axis, PowQL is a tax with no offsetting reward for most teams. + +The only defensible wedge is the one the brief already half-identifies: **agent-native, in-process state for Rust AI-agent runtimes.** Not "database for AI agents" in the cloud-memory-layer sense (Oracle, TiDB, mem0, Zep own that and it's a vector/graph game). The wedge is narrower and more honest: *the embedded query engine that a Rust agent harness reaches for to store and query its own structured state, where PowQL's AST=plan-tree design and the AGENTS.md cheat-sheet make small models reliably generate correct queries in-context.* That is a real gap and nobody is sitting in it. Everything else — edge analytics, generic Rust embedding — is a fast-follower fight you will lose on ecosystem. + +**My recommendation:** Reposition from "faster SQLite" to **"the embeddable query engine built for agents and the Rust services that run them."** Lead with DX and agent-correctness, keep the perf numbers as proof-of-craft, not as the headline. Pick ONE launch wedge (Rust agent-state) and go deep. Be brutally honest in the README about where SQLite/Turso win — you already are, and that honesty is itself a marketing asset in this audience. + +--- + +## 1. Positioning Statement + One-Line Pitch + +**Positioning statement:** +> For Rust developers building AI agents and latency-sensitive services who need to store and query structured state in-process, PowDB is a pure-Rust embedded database whose pipeline query language (PowQL) is designed so a small language model can generate correct queries from a one-page cheat sheet — and whose compiled execution engine runs scan and aggregate workloads 3–10x faster than SQLite. Unlike SQLite or Turso, PowDB removes the SQL translation tier entirely; unlike cloud agent-memory platforms, it runs embedded with zero network hop. + +**One-line pitch (lead with this):** +> **PowDB — the pure-Rust embedded query engine your AI agent can actually drive.** + +**Backup one-liners by audience:** +- Rust perf crowd: *"A from-scratch Rust database where the parser's AST is the query plan — 3–10x SQLite on scans and aggregates, zero C in the build."* +- Agent builders: *"Give your agent a database it can query correctly from a one-page prompt, in-process, no SQL-injection guesswork."* + +**What to STOP saying:** "3–10x faster than SQLite" as the *first* line. It invites the one comparison you lose strategically (Turso) and the one you can't win culturally (SQLite is good enough). Demote it to proof, not thesis. + +--- + +## 2. Honest Competitive Matrix + +Legend: ✅ genuine strength · ⚠️ partial / caveated · ❌ genuine weakness · — n/a + +| Dimension | **PowDB** | SQLite | DuckDB | Turso DB (libSQL + Limbo rewrite) | redb | +|---|---|---|---|---|---| +| Query language | PowQL (custom pipeline) ⚠️ | SQL ✅ | SQL ✅ | SQL ✅ | none (KV) ❌ | +| Pure Rust, no C | ✅ | ❌ (C) | ❌ (C++) | ✅ (Limbo) / ❌ (libSQL is C) | ✅ | +| Scan/aggregate perf | ✅ 3–10x SQLite | baseline | ✅✅ (columnar, beats both) | ≈ SQLite | n/a (KV) | +| OLTP point writes/lookups | ⚠️ competitive | ✅ mature | ⚠️ weak at OLTP | ✅ | ✅ | +| Maturity / battle-testing | ❌ pre-1.0 | ✅✅ billions | ✅ mature | ⚠️ libSQL prod, rewrite beta | ✅ 1.0, stable format | +| Tooling/ecosystem (ORMs, BI, drivers) | ❌ TS + Rust client only | ✅✅ everything | ✅ growing | ✅ (SQLite-compatible) | ❌ minimal | +| Edge/replication/sync | ❌ single-node | ⚠️ via libSQL | ❌ | ✅✅ embedded replicas, edge sync | ❌ | +| Server mode + auth + TLS | ✅ (shared pw / users) | ❌ not in core | ⚠️ Quack ext (new) | ✅ managed cloud | ❌ | +| Durability (WAL, crash recovery) | ✅ | ✅ | ✅ | ✅ | ✅ | +| Transactions | ✅ begin/commit/rollback | ✅ | ✅ | ✅ | ✅ | +| Vector / RAG search | ❌ | ⚠️ ext | ⚠️ ext | ✅ DiskANN native | ❌ | +| MVCC / concurrent writers | ❌ single-writer | ⚠️ WAL readers | ⚠️ | ✅ | ✅ MVCC readers | +| Agent-driveable query lang | ✅ (design intent + AGENTS.md) | ⚠️ (LLMs know SQL ~95%) | ⚠️ (SQL) | ⚠️ (SQL) | ❌ | +| LLM training-data coverage of the lang | ❌ ~zero | ✅✅ massive | ✅ | ✅ | n/a | + +### Where PowDB genuinely WINS today +1. **Pure-Rust + no C toolchain + fast aggregates, all at once.** No other option gives you all three. DuckDB is faster on analytics but is C++ and OLAP-shaped. Turso's Rust rewrite is SQL but still beta and not aggregate-optimized the way PowDB is. +2. **Smallest readable codebase.** ~20K lines, no generated parser, no bytecode VM. For a Rust shop that wants to *own and audit* its storage layer, that's a real value prop redb shares but SQLite/DuckDB/Turso don't. +3. **Embedded server mode with auth + TLS out of the box.** SQLite needs extensions; redb is KV-only. PowDB ships a TCP server today. +4. **Design coherence for in-context query generation.** AST=plan, one-page AGENTS.md, lowercase keywords, dot-field syntax — *if* the agent thesis holds, this is the one place PowDB is purpose-built and nobody else is. + +### Where PowDB genuinely LOSES today (say this out loud) +1. **The query language has no ecosystem and no LLM prior.** This is the single biggest liability. Every team that picks PowDB rewrites every query, every model has to be taught PowQL in-context every time, and no ORM/BI tool will ever speak it. SQL's 94–95% LLM execution accuracy comes from billions of training examples PowQL will never have. +2. **Turso owns "Rust + from-scratch + faster" with SQL compat.** Your differentiator-of-record (pure Rust, no SQL parse tax) is shared by a VC-funded team that kept SQL. That makes "pure Rust" table stakes, not a moat. +3. **DuckDB wins analytics outright.** If the use case is genuinely scan/aggregate-heavy, a serious evaluator benchmarks DuckDB and PowDB loses on columnar workloads. Your bench is vs SQLite, not vs the actual analytics leader. +4. **Maturity gap is existential for a DB.** Pre-1.0, shifting on-disk format, 3 fuzz targets vs SQLite's OSS-Fuzz corpus. Nobody puts their source-of-truth data on a v0.4 single-team engine. This caps you to *derived/ephemeral* state, which is actually fine — see the wedge. +5. **No MVCC, single writer.** Fine for embedded single-agent, disqualifying for any multi-writer service. + +**Honest moat assessment:** The engine moat (compiled predicates + pure Rust) is *thin and shrinking* — Turso closes it. The only **expanding** moat is **PowQL-as-agent-interface coherence + the AGENTS.md DX pattern**, and that moat only exists if you invest in it deliberately and prove it. Right now it's a hypothesis, not a moat. + +--- + +## 3. Unique Use Cases, Ranked by Wedge Potential + +### Wedge #1 (GO HERE): In-process state store for Rust AI-agent runtimes +**The job:** A Rust agent harness (think a Carl-Code/opencode-style loop, a tool-using agent, a workflow engine) needs to persist and query structured state — task queues, tool-call history, episodic facts, scratchpad tables — and let the *model itself* read/write that state via generated queries. + +**Why incumbents serve it poorly:** +- Cloud agent-memory (Oracle, TiDB, mem0, Zep) is network-hopped, vector/graph-shaped, and overkill for in-loop structured scratch state. Wrong latency class, wrong shape. +- SQLite works but the agent generates SQL against an unknown schema and you're back to error-retry loops and injection-escaping (PowDB's own TS client has no param binding yet — note this gap). +- redb is KV; the agent can't express `filter .status = "pending" order .created desc limit 5`. + +**PowDB's unfair advantage:** In-process (zero hop), pure-Rust (drops straight into the harness binary), and PowQL + AGENTS.md is a *deliberately small grammar a 7B–20B model can be taught in one prompt.* The AST=plan property means a generated query is either parseable-and-runnable or a clean parse error you feed back — exactly the "self-healing retry" loop the 2026 text-to-SQL literature describes, but over a grammar small enough to fully specify in context. + +**Wedge size & expansion:** Small today (Rust agent harnesses are a niche of a niche), but it's the fastest-growing developer category in 2026 and ZVN is *literally building one*. Land here, expand to "embedded state for any latency-sensitive Rust service." + +**Falsifiable:** If a 14B model with AGENTS.md in context can't hit >90% valid-query generation on a 10-table schema, the wedge is dead. **Test this in week one.** + +### Wedge #2: Embedded analytics inside a Rust service (the "hot rollup" cache) +**The job:** A Rust web service doing per-request aggregates (dashboards, counters, leaderboards, top-N) over tables that fit on disk, where the aggregate is on the request hot path. + +**Why incumbents serve it poorly:** SQLite is 3–10x slower here (your bench proves it); DuckDB is C++ and OLAP-process-shaped, not a great per-request embed; Turso isn't aggregate-tuned. + +**Why it's #2 not #1:** Real but small win, and DuckDB-as-library is a credible counter for anyone analytics-serious. The PowQL tax is hardest to justify here because these are developer-authored queries, not agent-generated — so the "small grammar" advantage doesn't apply. + +### Wedge #3: Pure-Rust / no-C constrained build targets +**The job:** Wasm-adjacent, minimal containers, supply-chain-paranoid shops that refuse `bindgen`/C in the build. + +**Why incumbents serve it poorly:** SQLite/DuckDB are C/C++. redb is the real competitor here but is KV-only — PowDB offers a query language. + +**Why it's #3:** Genuine but tiny audience, and Turso's Rust rewrite erodes it. Good *secondary* talking point, not a launch wedge. + +### Wedge #4 (DO NOT LEAD WITH): "Database for AI agents" (the cloud memory-layer meaning) +This is a crowded, well-funded category (Oracle AI Agent Memory, TiDB, mem0, Zep, Databricks Lakebase) playing a vector + temporal-graph + horizontal-scale game. PowDB has no vectors, no scale-out, no memory abstractions. **Do not position here** — you'll be measured against features you don't have and lose instantly. Use Wedge #1's *narrower, in-process, structured-state* framing instead and explicitly distinguish it from "agent memory layer." + +--- + +## 4. Target Personas + Channels + +### Primary persona — "Harness Hannah," the Rust agent-runtime builder +- Builds an agent loop / tool-runner / workflow engine in Rust. Cares about latency, binary size, no-C builds, and *letting the model drive tools reliably.* +- Reads: this-week-in-rust, lobste.rs, r/rust, HN, the agent-builder Discords/X. +- Painkiller: "my agent generates broken queries against SQLite and I babysit the retry loop." PowQL's small grammar + AGENTS.md is the pitch. +- **This is the persona ZVN already is.** Dogfood PowDB inside the ZVN agent stack and write that up — it's the single most credible artifact you can ship. + +### Secondary persona — "Perf-Rust Pete," the latency-sensitive Rust service dev +- Has a hot-path aggregate, already on tokio, won't add a C dep. Wants `cargo install` and 3x. +- Reads: same Rust channels + benchmark threads. Motivated by the honest comparison doc. + +### Tertiary — "Skeptical Sam," the senior eng / DB nerd on HN/lobste.rs +- Won't adopt, but will *amplify or destroy* your launch. Wins respect through honesty (your vs-SQLite doc is already calibrated for him), loses instantly to overclaiming. Treat the HN thread as the real product surface. + +### Channels (ranked) +1. **lobste.rs + r/rust + this-week-in-rust** — your actual buyers live here. Highest signal, lowest cost. +2. **A killer "show, don't tell" dogfood writeup** — "We let a 14B model run our agent's state store with a one-page cheat sheet. Here's the eval." This is your wedge proof AND your best content all at once. +3. **HN Show HN** — high-variance, high-reward. Only after the dogfood eval exists, because the top comment *will* be "why not Turso/SQLite?" and you need a crisp, honest, data-backed answer ready. +4. **Agent-builder communities (X/Discord)** — narrower but exactly Wedge #1. Seed via the eval writeup. +5. **docs.rs + crates.io quality** — passive but compounding; Rust devs judge by docs.rs polish. + +### Content strategy (3 anchor pieces, not a blog mill) +1. **"Can a small model drive a database? An eval of LLM-generated PowQL vs SQL."** — the thesis-defining/falsifying artifact. Publish the numbers even if they're mixed; honesty *is* the brand. +2. **"PowDB vs SQLite vs DuckDB vs Turso: when each one is right" (extend the existing honest doc to 4-way).** Ranks for the comparison searches and earns trust by recommending competitors when they're right. +3. **"How we built a query language whose AST is its plan tree"** — engine-craft piece for the perf/DB nerds; this is the "read the code" crowd magnet. + +--- + +## 5. Launch Plan — 30 / 60 / 90 Days (sized for 1–2 people + agents) + +### Next 30 days — **Prove or kill the wedge. No public launch yet.** +1. **Run the falsification eval (Wedge #1).** Build a harness: 10-table schema, 50–100 natural-language tasks, AGENTS.md in context, a 14B-class model (use the ZVN inference gateway), measure valid-query + correct-result rate vs the same model writing SQLite SQL. **This single number decides the whole strategy.** Track parse-success, execution-success, semantic-correctness separately. +2. **Close the one disqualifying DX gap: parameter binding over the wire.** AGENTS.md admits "no parameter binding yet" and the agent wedge *requires* safe value insertion. Without it, you're telling agent builders to escape strings by hand. Ship prepared-statement placeholders in the TS + Rust clients. (This is the highest-leverage feature on the roadmap for the chosen wedge.) +3. **Write the 4-way honest comparison doc** (add DuckDB + Turso to the existing SQLite doc). Needed before any HN exposure. +4. **Dogfood PowDB in the ZVN agent stack** for one real state-store use case. Even one shipped internal use is worth more than any benchmark. + +**Gate:** if the eval shows PowQL generation is materially *worse* than SQL and AGENTS.md can't close the gap, **stop and pivot the positioning to pure perf/Rust-embed (Wedge #2/#3) and lower ambitions.** Do not launch the agent narrative on a falsified thesis. + +### 30–60 days — **Soft launch to the friendly Rust audience.** +5. **Publish the eval writeup** + the 4-way comparison. Post to lobste.rs and r/rust (NOT HN yet). Gather the objections — they're your roadmap. +6. **Ship `powdb-agent` examples crate:** a runnable example of an LLM driving PowDB via AGENTS.md (works with the ZVN gateway and OpenAI/Anthropic). This is the wedge made tangible. +7. **Harden the honesty surface:** make sure README's top line matches the new positioning (lead with DX/agent, demote the bench headline). Tighten getting-started so `cargo install` → first query is under 2 minutes. +8. **Cut a 0.5.0** that bundles param-binding + any eval-driven fixes. Signal momentum. + +### 60–90 days — **The HN swing + decide on 1.0 path.** +9. **Show HN**, only with: the eval, the 4-way doc, the agent example, and a maintainer (Kirby) ready to answer "why not Turso/SQLite/DuckDB?" live for 6 hours. The honest comparison docs are your armor. +10. **Publish the on-disk format stability commitment + 1.0 roadmap.** The maturity objection is the #2 adoption blocker; a credible "format frozen at 1.0, here's the date" calms it. +11. **Instrument adoption:** crates.io downloads by crate, GitHub stars velocity, *and the one metric that matters* — number of external repos that depend on `powdb-*` and use it for agent state (search GitHub). One real external agent-builder adopter > 1,000 stars. +12. **Decide:** double down on agent wedge (if eval + adoption signal yes) or settle into "the fast pure-Rust embedded engine" niche (if the agent thesis underdelivers). + +--- + +## 6. Risks, Thesis-Falsifiers, and Metrics + +### What would FALSIFY the core thesis (and what to do) +| Thesis | Falsifier | If falsified | +|---|---|---| +| PowQL is easier for small models to generate correctly than SQL | 14B model with AGENTS.md scores ≤ SQL on valid+correct query rate | Kill the agent positioning; fall back to perf/Rust-embed niche | +| Removing the SQL tier is a durable perf moat | Turso's Rust rewrite hits GA and matches/beats PowDB while keeping SQL | Stop competing on perf-vs-SQLite; compete only on DX/agent + readability | +| There's demand for a non-SQL embedded DB | <5 external repos adopt in 90 days despite a clean launch | Reframe as a library/teaching engine; lower roadmap ambition, keep it as ZVN-internal infra | +| Aggregate perf is the buying reason | Evaluators benchmark DuckDB and leave | Concede analytics to DuckDB; own "fast *transactional* embedded for Rust services + agents" | + +### Top risks +1. **The PowQL ecosystem tax is unwinnable at scale.** *Mitigation:* never fight on ecosystem; win only where the query is agent-generated-in-context (so no human ecosystem is needed) or developer-owned-and-tiny. +2. **Turso commoditizes "pure Rust from scratch."** *Mitigation:* shift the moat to DX/agent-coherence + auditable small codebase; treat pure-Rust as table stakes. +3. **Maturity blocks all serious adoption.** *Mitigation:* position for *derived/ephemeral* state (agent scratch, hot caches, rollups) where data loss is recoverable; publish a format-freeze 1.0 commitment. +4. **Tiny team can't sustain a DB.** *Mitigation:* keep scope brutally small (single-node, embedded, the wedge); resist every "add replication/vectors/UDFs" request that isn't the wedge. Use the multi-agent dev workflow ZVN already runs. +5. **Launch overclaim torches credibility.** *Mitigation:* the existing honesty discipline is your single best asset — protect it. Every benchmark claim must be reproducible (`cargo run -p powdb-compare`) before it's public. + +### Metrics to track (rollups, not vanity) +**North-star:** # of external repositories using `powdb-*` for agent/service state (not stars). This is the only metric that proves the wedge. +- **Wedge-proof:** LLM-generated-PowQL valid+correct rate (the eval number), tracked per model size. +- **Activation:** time-to-first-query from `cargo install` (target < 2 min); % of new repos that run a second session (retention proxy). +- **Funnel:** crates.io downloads per crate, docs.rs visits, getting-started → second-query drop-off. +- **Credibility:** HN/lobste.rs sentiment (does the top comment defend or dismiss?), reproducibility complaints (target: zero). +- **Anti-metric to ignore:** raw GitHub stars. Stars from a benchmark HN post don't equal adoption and will mislead the roadmap. + +--- + +## Bottom line for the team + +You built a real engine with real, honestly-measured wins. The trap is letting "faster than SQLite" be the story — it's the one fight where SQLite is good enough and Turso out-positions you. The prize is being **the embedded query engine that an AI agent can correctly drive in-process**, proven first inside ZVN's own agent stack. Run the eval in week one; it tells you whether you have a category or a faster SQLite. Either way, your radical honesty in the docs is the most valuable thing you've shipped — don't trade it for a louder headline. diff --git a/docs/powdb-vs-sqlite.md b/docs/powdb-vs-sqlite.md index 94ec5a2..f3766b9 100644 --- a/docs/powdb-vs-sqlite.md +++ b/docs/powdb-vs-sqlite.md @@ -47,8 +47,11 @@ between the two. - **You're already shipping the C toolchain.** If your build already compiles `aws-lc`, `openssl`, or any other C dep, the `libsqlite3-sys` cost is zero. -- **You want full MVCC, RBAC, online backups, or any of the - decade-of-features SQLite has.** PowDB has none of these yet. +- **You want full MVCC, online backups, or any of the decade-of-features + SQLite has.** PowDB 0.4.5 shipped role-based users (admin / readwrite / + readonly) and offline full/incremental backup with coarse point-in-time + recovery, but there is still no MVCC and no *online* backup -- backups + require stopping the server. ## Side-by-side feature table @@ -67,6 +70,7 @@ between the two. | Server mode | Yes (binary wire protocol, TLS, auth) | Not in core (extensions exist) | | Fuzz testing | 3 cargo-fuzz targets (lexer, parser, roundtrip) | OSS-Fuzz, decades of corpora | | Crash recovery | WAL replay + page-zero recovery + index rebuild | WAL/rollback journal | +| Backup | Offline full/incremental + coarse PITR (0.4.5) | Online backup API, `.backup`, VACUUM INTO | | On-disk format stability | Pre-1.0, may shift | Stable for decades | | Production deployments | Pre-1.0 | Billions | @@ -134,7 +138,7 @@ Results land in `crates/compare/results.csv`. ## Caveats and roadmap - **PowDB is pre-1.0.** The on-disk format may shift across minor versions. - Pin a version (`cargo install powdb-cli --version 0.4.4 --locked`) and + Pin a version (`cargo install powdb-cli --version 0.4.5 --locked`) and expect to re-bench / re-import on upgrades until 1.0. - **SQLite is the safe default.** Decades of production exposure, an enormous test suite, and tools everywhere. If you're not sure, you diff --git a/docs/superpowers/plans/2026-04-16-ts-client-v02-sprint.md b/docs/superpowers/plans/2026-04-16-ts-client-v02-sprint.md index f2d6e90..f1911bf 100644 --- a/docs/superpowers/plans/2026-04-16-ts-client-v02-sprint.md +++ b/docs/superpowers/plans/2026-04-16-ts-client-v02-sprint.md @@ -1,4 +1,7 @@ # Sprint Plan — PowDB TS Driver v0.2 + +> **Completed / historical.** This sprint shipped in client 0.3.x; kept for provenance. + Generated: 2026-04-16 Based on: conversation review of `clients/ts/` (see session thread) diff --git a/examples/deploy/README.md b/examples/deploy/README.md index acb07ae..c8125e2 100644 --- a/examples/deploy/README.md +++ b/examples/deploy/README.md @@ -28,10 +28,20 @@ Notes: - `min_machines_running = 1` keeps the database always-on; `auto_stop_machines` is `false` so Fly never suspends a stateful service. -## Docker / Compose +## Docker -See [`docker-compose.yml`](https://github.com/zvndev/powdb/blob/main/docker-compose.yml) -in the repo root for a local-only quick-start. +Quick-start with the published image (note: the repo-root `docker-compose.yml` +is the benchmark harness — it does not define a PowDB service): + +```bash +docker run -d --name powdb \ + -p 5433:5433 \ + -v powdb_data:/data \ + -e POWDB_DATA=/data \ + -e POWDB_BIND=0.0.0.0 \ + -e POWDB_PASSWORD=change-me \ + ghcr.io/zvn-dev/powdb:v0.4.5 +``` ## AWS ECS Fargate + EFS diff --git a/examples/deploy/fly.toml b/examples/deploy/fly.toml index 944cf48..43eb313 100644 --- a/examples/deploy/fly.toml +++ b/examples/deploy/fly.toml @@ -22,6 +22,8 @@ primary_region = "iad" RUST_LOG = "info" POWDB_DATA = "/data" POWDB_PORT = "5433" + # Bind all interfaces — the server defaults to 127.0.0.1, which Fly's proxy can't reach. + POWDB_BIND = "0.0.0.0" # Phase 1 hardening: refuse to start when a password is set without TLS. # Leave OFF until you've mounted certs via secrets (POWDB_TLS_CERT / # POWDB_TLS_KEY) or fronted the service with a TLS-terminating proxy. diff --git a/examples/deploy/railway/README.md b/examples/deploy/railway/README.md index 945a072..c6c376a 100644 --- a/examples/deploy/railway/README.md +++ b/examples/deploy/railway/README.md @@ -45,9 +45,11 @@ The host/port come from the TCP proxy you configured in step 4. are pinned to a region; if Railway moves your service across regions, you'll need to snapshot and restore manually. Pin the region in the dashboard for production. -- **No managed backups.** Schedule a sidecar `cron` that runs - `powdb-cli` exports against the volume, or snapshot the volume via - Railway's API on a schedule. +- **No managed backups.** Use `powdb-cli backup` (full / incremental, + plus coarse PITR on restore) against the volume — but note backups are + **offline**: stop the server first, so schedule them in a maintenance + window or snapshot the volume via Railway's API instead. See + [docs/backup-and-restore.md](../../../docs/backup-and-restore.md). - **Bandwidth pricing.** Heavy benchmark traffic over the TCP proxy bills egress. For benchmark runs, use the Fly.io or AWS examples — Railway is best for developer-friendly persistent deploys, not for hammering. diff --git a/site/getting-started.html b/site/getting-started.html index 263e83b..cccc9b2 100644 --- a/site/getting-started.html +++ b/site/getting-started.html @@ -19,7 +19,7 @@
@@ -117,6 +117,36 @@

Benchmarks

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
282ns 3.0x
multi_col_and_filter2.22ms4.70ms2.1x
insert_batch_1k238ns320ns1.3x
delete_by_filter1.76ms2.35ms1.3x
scan_filter_project_top1009.6µs12.7µs1.3x
point_lookup_nonindexed350µs432µs1.2x
@@ -221,7 +251,7 @@

Compiled Predicates

B+ Tree Indexes

-

Disk-persisted B+ tree indexes in a custom BIDX binary format. Point lookups resolve in under 150ns. Indexes survive restarts and are used automatically.

+

Disk-persisted B+ tree indexes in a custom BIDX binary format. Indexed point lookups resolve in under 100ns. Indexes survive restarts and are used automatically.

WAL + Crash Recovery

@@ -237,7 +267,7 @@

TypeScript Client

TLS + Authentication

-

Production-ready security with TLS encryption and password authentication. Set POWDB_PASSWORD to lock down your server.

+

TLS encryption plus two auth modes: a shared password (POWDB_PASSWORD) or named users with roles (admin / readwrite / readonly, argon2id-hashed) shipped in v0.4.5.

Zero C Dependencies

@@ -251,6 +281,21 @@

Pipeline Query Language

+ +
+
+

When PowDB is not the right fit

+

PowDB is pre-1.0 and deliberately scoped. Be honest with yourself before adopting it:

+
    +
  • Pre-1.0 format stability -- the on-disk format may shift across minor versions. Pin versions and expect to re-import on upgrades until 1.0.
  • +
  • No SQL compatibility -- PowQL is a different language by design. No ORMs, BI dashboards, JDBC drivers, or DB browsers will speak to it.
  • +
  • Single-node only -- no replication, sharding, or consensus.
  • +
  • No MVCC, no online backup -- single writer with parallel readers; backups (full, incremental, PITR) are offline-only today.
  • +
+

If any of those are dealbreakers, use SQLite -- and we say so plainly in PowDB vs SQLite: when to use which.

+
+
+