From bcc43b424284acbd30b54389fa271f064ac2f506 Mon Sep 17 00:00:00 2001 From: Ragnor Comerford Date: Wed, 30 Sep 2026 15:05:20 +0200 Subject: [PATCH 01/29] planner: refusals by design carry planner diagnostics MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The query shapes engine v2's planner refuses by design now carry the same diagnostic contract as parse and type refusals. The ranking of a binding a traversal reaches gets code P001, stage `plan`, the refused call as the expression, and a fix that answers the query: declare that binding first, so the ranking starts the traversal. The edge-selection refusals added with edge alternation and wildcard traversal get P002 (no finite traversal work limit), P003 (the limit out of range or recorded twice) and P004 (CSR traversal mode), with fixes where one exists. `PlanError::Unsupported` holds the diagnostic and `PlanError::refused` builds it; the gate's `Unrouted::UnsupportedQuery` carries it, and `plan_source` returns it as `OmniError::Compiler` on the ordinary and the inspected door, so the server answers 400 with the `diagnostic` object and the CLI prints the code, stage and fix. The diagnostic kind `plan` renders its one-line text as `plan error: P001: …`. Fixes #786. --- .../planner-refusal-diagnostic.fixed.md | 6 ++ crates/omnigraph-compiler/src/query/codes.rs | 14 ++++- .../src/query/diagnostic.rs | 34 +++++++++++- ..._an_unranked_binding_and_a_destination.gqt | 4 +- crates/omnigraph-planner/src/error.rs | 24 +++++++- crates/omnigraph-planner/src/gate.rs | 18 ++++-- crates/omnigraph-planner/src/optimizer.rs | 55 +++++++++++++------ crates/omnigraph-planner/src/physical.rs | 27 +++++---- .../tests/query_plan/edge_selections.rs | 12 +++- crates/omnigraph/src/engine/plan_source.rs | 12 +++- .../omnigraph/tests/engine_v2_plan_replay.rs | 53 +++++++++++++----- crates/omnigraph/tests/rrf_prefilter_gate.rs | 8 ++- docs/user/queries/diagnostics.md | 20 ++++++- 13 files changed, 218 insertions(+), 69 deletions(-) create mode 100644 changelog.d/planner-refusal-diagnostic.fixed.md diff --git a/changelog.d/planner-refusal-diagnostic.fixed.md b/changelog.d/planner-refusal-diagnostic.fixed.md new file mode 100644 index 000000000..a52b8ca09 --- /dev/null +++ b/changelog.d/planner-refusal-diagnostic.fixed.md @@ -0,0 +1,6 @@ +- A query shape the planner refuses by design, a `nearest()` or `bm25()` + ranking on a binding a traversal reaches, is a bad request with the planner + code `P001`, the refused expression and a fix that answers the query: + declare that binding first in `match`. See the [diagnostics guide][p001-diagnostics]. + +[p001-diagnostics]: ../docs/user/queries/diagnostics.md diff --git a/crates/omnigraph-compiler/src/query/codes.rs b/crates/omnigraph-compiler/src/query/codes.rs index 957c72716..ac85789b1 100644 --- a/crates/omnigraph-compiler/src/query/codes.rs +++ b/crates/omnigraph-compiler/src/query/codes.rs @@ -1,6 +1,7 @@ //! The catalog of query compile diagnostic codes (RFC 0047). A code is //! stable once published: its meaning is frozen, its message text may -//! improve. `Q…` codes are the parser's, `T…` codes the typechecker's; the +//! improve. `Q…` codes are the parser's, `T…` codes the typechecker's, `P…` +//! codes the planner's; the //! numbering has gaps where earlier stages were withdrawn, and a withdrawn //! number is never reused. @@ -69,6 +70,12 @@ codes! { T51 = "unknown type"; T52 = "nested list literals are not supported"; T53 = "list literal elements must share one scalar type"; + + // Planner. + P001 = "a ranking orders a binding a traversal reaches"; + P002 = "an edge alternation or wildcard traversal needs a finite traversal work limit"; + P003 = "the traversal work limit is out of range or recorded twice"; + P004 = "an edge alternation or wildcard traversal cannot run in CSR traversal mode"; } #[cfg(test)] @@ -82,7 +89,10 @@ mod tests { for code in ALL { assert!(seen.insert(code.as_str()), "duplicate code {code}"); let (prefix, number) = code.as_str().split_at(1); - assert!(matches!(prefix, "Q" | "T"), "{code}: unknown namespace"); + assert!( + matches!(prefix, "Q" | "T" | "P"), + "{code}: unknown namespace" + ); assert!( number.parse::().is_ok(), "{code}: the number must parse" diff --git a/crates/omnigraph-compiler/src/query/diagnostic.rs b/crates/omnigraph-compiler/src/query/diagnostic.rs index b5165d29c..b7dcd3024 100644 --- a/crates/omnigraph-compiler/src/query/diagnostic.rs +++ b/crates/omnigraph-compiler/src/query/diagnostic.rs @@ -22,7 +22,8 @@ pub struct QueryCodeSpec { /// A stable diagnostic code, a pointer-sized handle to its catalog entry so /// a diagnostic stays small enough to travel in a `Result`. Parse codes are -/// `Q…`, typecheck codes are `T…`; the catalog is `super::codes`. +/// `Q…`, typecheck codes are `T…`, planner codes are `P…`; the catalog is +/// `super::codes`. #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub struct QueryCode(pub &'static QueryCodeSpec); @@ -50,12 +51,13 @@ impl std::fmt::Display for QueryCode { } } -/// Which compiler phase refused the source; decides the rendered prefix. +/// Which phase refused the source; decides the rendered prefix. #[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize)] #[serde(rename_all = "snake_case")] pub enum QueryDiagnosticKind { Parse, Type, + Plan, } /// A source position: 1-based line and column (in characters) plus the byte @@ -172,6 +174,23 @@ impl QueryDiagnostic { } } + /// A planner diagnostic: a well-formed, type-checked query shape the + /// planner refuses by design, so it carries the `plan` stage. + pub fn plan(code: QueryCode, message: impl Into) -> Self { + Self { + kind: QueryDiagnosticKind::Plan, + code, + message: message.into(), + position: None, + stage: Some(Box::new(Stage { + name: "plan", + expression: None, + })), + fix: None, + suggestion: None, + } + } + pub fn with_fix(mut self, fix: impl Into) -> Self { self.fix = Some(fix.into()); self @@ -204,6 +223,9 @@ impl std::fmt::Display for QueryDiagnostic { QueryDiagnosticKind::Type => { write!(f, "type error: {}: {}", self.code, self.message) } + QueryDiagnosticKind::Plan => { + write!(f, "plan error: {}: {}", self.code, self.message) + } } } } @@ -213,7 +235,7 @@ impl std::error::Error for QueryDiagnostic {} #[cfg(test)] mod tests { use super::*; - use crate::query::codes::{Q001, T1}; + use crate::query::codes::{P001, Q001, T1}; #[test] fn position_counts_lines_and_character_columns() { @@ -230,6 +252,12 @@ mod tests { assert_eq!(parse.to_string(), "parse error: expected `(`"); let typed = QueryDiagnostic::typecheck(T1, "unknown node type `X`").with_fix("declare X"); assert_eq!(typed.to_string(), "type error: T1: unknown node type `X`"); + let plan = QueryDiagnostic::plan(P001, "`nearest()` orders `$t`").with_expression("x"); + assert_eq!( + plan.to_string(), + "plan error: P001: `nearest()` orders `$t`" + ); + assert_eq!(plan.stage.as_ref().map(|stage| stage.name), Some("plan")); } #[test] diff --git a/crates/omnigraph-gqt/cases/v2/planner/search_order_on_an_unranked_binding_and_a_destination.gqt b/crates/omnigraph-gqt/cases/v2/planner/search_order_on_an_unranked_binding_and_a_destination.gqt index 51154161a..ddbec1fb0 100644 --- a/crates/omnigraph-gqt/cases/v2/planner/search_order_on_an_unranked_binding_and_a_destination.gqt +++ b/crates/omnigraph-gqt/cases/v2/planner/search_order_on_an_unranked_binding_and_a_destination.gqt @@ -1,5 +1,5 @@ # issue: none -# notes: A search-only order over a pipeline with a second, unranked binding sorts by the score and tie-breaks on every scope binding's id, which the planner reads for the scan; a search order on a traversal destination is refused at planning with the shape named. +# notes: A search-only order over a pipeline with a second, unranked binding sorts by the score and tie-breaks on every scope binding's id, which the planner reads for the scan; a search order on a traversal destination is refused at planning with the shape named and planner code P001. # red_on: 2026-09-23, task 0226 tree before the one-pass tie-break: step 1 failed with `the planned tie-break column 'z.__id' is not in the sort input` (the tie-break was derived at lowering, after projection pushdown); step 2 failed with `column 'd._distance' not found for ordering` instead of a planning refusal. --- runner @@ -62,4 +62,4 @@ query nearest_on_a_traversal_destination($q: Vector(4)) { } --- params {"q": [0.0, 0.0, 0.0, 0.0]} ---- expect error: a traversal destination, which engine v2 does not support +--- expect error: plan error: P001: `nearest()` orders `$d`, a traversal destination diff --git a/crates/omnigraph-planner/src/error.rs b/crates/omnigraph-planner/src/error.rs index fa52c7422..4b57d448b 100644 --- a/crates/omnigraph-planner/src/error.rs +++ b/crates/omnigraph-planner/src/error.rs @@ -1,3 +1,5 @@ +use omnigraph_compiler::QueryDiagnostic; +use omnigraph_compiler::query::diagnostic::QueryCode; use thiserror::Error; /// A planning failure. For a change-feed or merge operation the gate turns @@ -9,11 +11,27 @@ pub enum PlanError { #[error("the plan source could not resolve: {detail}")] Unresolved { detail: String }, /// A well-formed query shape the planner refuses by design; the caller's - /// error, answered as a bad request, never as a planner defect. - #[error("{detail}")] - Unsupported { detail: String }, + /// error, answered as a bad request carrying its diagnostic, never as a + /// planner defect. + #[error("{0}")] + Unsupported(Box), /// A pass met a plan it has no rule for. A registered shape never reaches /// this arm; the registry test pins that. #[error("planner internal error: {0}")] Internal(String), } + +impl PlanError { + /// A refusal by design: the planner code, what was refused, and the one + /// fix that answers it, or `None` when the message names the decision. + pub fn refused(code: QueryCode, message: impl Into, fix: Option<&str>) -> Self { + let diagnostic = QueryDiagnostic::plan(code, message); + Self::Unsupported(Box::new(match fix { + Some(fix) => diagnostic.with_fix(fix), + None => diagnostic, + })) + } +} + +/// The fix of a traversal that needs a work limit (`P002`). +pub(crate) const SET_TRAVERSAL_WORK_LIMIT: &str = "set `traversal_work_limit` before the query, for example `set traversal_work_limit = 1000000;`"; diff --git a/crates/omnigraph-planner/src/gate.rs b/crates/omnigraph-planner/src/gate.rs index aad31d711..df4510a0d 100644 --- a/crates/omnigraph-planner/src/gate.rs +++ b/crates/omnigraph-planner/src/gate.rs @@ -229,8 +229,11 @@ pub enum Unrouted { /// planner defect. A read query fails with this message. PlannerError { message: String }, /// A well-formed query shape the planner refuses by design - /// (`PlanError::Unsupported`); the caller's error, not a planner defect. - UnsupportedQuery { message: String }, + /// (`PlanError::Unsupported`); the caller's error, not a planner defect, + /// with the diagnostic the caller answers with. + UnsupportedQuery { + diagnostic: Box, + }, } impl Unrouted { @@ -238,7 +241,7 @@ impl Unrouted { /// everything else is a planner error. fn of(error: PlanError) -> Self { match error { - PlanError::Unsupported { detail } => Self::UnsupportedQuery { message: detail }, + PlanError::Unsupported(diagnostic) => Self::UnsupportedQuery { diagnostic }, other => Self::PlannerError { message: other.to_string(), }, @@ -266,9 +269,12 @@ impl Unrouted { Self::RegistryRouteExecutor { entry } => json!({ "kind": self.kind(), "entry": entry }), Self::DeclaredBytesOverBound { node } => json!({ "kind": self.kind(), "node": node }), Self::Override => json!({ "kind": self.kind() }), - Self::PlannerError { message } | Self::UnsupportedQuery { message } => { - json!({ "kind": self.kind(), "message": message }) - } + Self::PlannerError { message } => json!({ "kind": self.kind(), "message": message }), + Self::UnsupportedQuery { diagnostic } => json!({ + "kind": self.kind(), + "code": diagnostic.code.as_str(), + "message": diagnostic.message, + }), } } } diff --git a/crates/omnigraph-planner/src/optimizer.rs b/crates/omnigraph-planner/src/optimizer.rs index 034c9016c..8d3bcbf74 100644 --- a/crates/omnigraph-planner/src/optimizer.rs +++ b/crates/omnigraph-planner/src/optimizer.rs @@ -6,8 +6,10 @@ use std::collections::{BTreeMap, BTreeSet, HashMap, HashSet}; use std::sync::Arc; use arrow_schema::{DataType, Field, Schema, SchemaRef}; +use omnigraph_compiler::QueryDiagnostic; use omnigraph_compiler::ir::{IRExpr, IROp, IROrdering, IRProjection, QueryIR}; use omnigraph_compiler::query::ast::AggFunc; +use omnigraph_compiler::query::codes::{P001, P002, P004}; use omnigraph_compiler::settings::Traversal; use omnigraph_compiler::traversal::{EDGE_TYPE_COLUMN, EdgeSelection}; @@ -16,7 +18,7 @@ use crate::cost::{ choose_access_path, choose_expand_mode, direction_probe_factor, estimate_rows, executed_hops, scan_row_estimate, }; -use crate::error::PlanError; +use crate::error::{PlanError, SET_TRAVERSAL_WORK_LIMIT}; use crate::logical::{ ColumnRef, EDGE_TYPE_MEMBER, GqFilter, IDENTITY_MEMBER, KeyJoinKind, LOGICAL_ID, LogicalId, LogicalNode, LogicalPlan, Predicate, RuntimeFilterKind, RuntimeFilterSpec, ScanSpec, SearchArm, @@ -161,11 +163,13 @@ fn resolve_query( source: &dyn PlanSource, ) -> Result<(), PlanError> { if ir.has_edge_selections() { - let limit = source - .traversal_work_limit() - .ok_or_else(|| PlanError::Unsupported { - detail: "edge selections require a finite traversal_work_limit".to_string(), - })?; + let limit = source.traversal_work_limit().ok_or_else(|| { + PlanError::refused( + P002, + "edge selections require a finite traversal_work_limit", + Some(SET_TRAVERSAL_WORK_LIMIT), + ) + })?; let assumptions = Assumptions { traversal_work_limit: Some(limit), ..Default::default() @@ -173,10 +177,11 @@ fn resolve_query( assumptions.validated_traversal_work_limit()?; plan.set_traversal_work_limit(Some(limit)); if source.traversal() == Traversal::Csr { - return Err(PlanError::Unsupported { - detail: "edge selections do not support traversal = csr; use auto or indexed" - .to_string(), - }); + return Err(PlanError::refused( + P004, + "edge selections do not support traversal = csr; use auto or indexed", + Some("run the traversal with traversal mode `auto` or `indexed`"), + )); } } let QueryIR { @@ -2005,11 +2010,23 @@ impl Lowering<'_> { RankKind::Nearest => "nearest", RankKind::Bm25 => "bm25", }; - return Err(PlanError::Unsupported { - detail: format!( - "`{function}()` orders `${binding}`, a traversal destination, which engine v2 does not support; order on the traversal's source binding, or match the destination with search()" - ), - }); + // Declaring the binding first makes it the component's scan root, + // which is ranked; the compiler picks the first-declared binding. + return Err(PlanError::Unsupported(Box::new( + QueryDiagnostic::plan( + P001, + format!( + "`{function}()` orders `${binding}`, a traversal destination; engine v2 ranks only the binding a traversal starts from" + ), + ) + .with_expression(format!( + "{function}(${binding}.{}, {})", + access.property, access.query + )) + .with_fix(format!( + "declare `${binding}` first in `match`, so the ranking starts the traversal" + )), + ))); } self.unmark(scan); match self.physical.node_mut(scan) { @@ -2747,8 +2764,12 @@ impl Lowering<'_> { ExpandPolicy::Budgeted, )); } - let member = edges.named().ok_or_else(|| PlanError::Unsupported { - detail: "edge selections require a finite traversal_work_limit".to_string(), + let member = edges.named().ok_or_else(|| { + PlanError::refused( + P002, + "edge selections require a finite traversal_work_limit", + Some(SET_TRAVERSAL_WORK_LIMIT), + ) })?; let edge_type = &member.edge_type; let direction = member.direction; diff --git a/crates/omnigraph-planner/src/physical.rs b/crates/omnigraph-planner/src/physical.rs index 6237301f5..a1a818611 100644 --- a/crates/omnigraph-planner/src/physical.rs +++ b/crates/omnigraph-planner/src/physical.rs @@ -10,13 +10,14 @@ use serde::{Deserialize, Serialize}; use serde_json::{Value, json}; use crate::cost::{AccessPath, ExpandMode, ExpandPolicy}; -use crate::error::PlanError; +use crate::error::{PlanError, SET_TRAVERSAL_WORK_LIMIT}; use crate::logical::{ ColumnRef, KeyJoinKind, ScanSpec, filters_json, metadata_count_json, ordering_text, scan_json, tiebreak_text, }; use crate::mirror::EdgeSelectionMirror; use crate::source::SideId; +use omnigraph_compiler::query::codes::{P002, P003}; /// The index of a node in a [`PhysicalPlan`]. pub type NodeId = usize; @@ -134,18 +135,24 @@ impl Assumptions { /// `1..=i64::MAX`, or wildcard provenance without a captured limit. pub fn validated_traversal_work_limit(&self) -> Result, PlanError> { if self.settings.contains_key("traversal_work_limit") { - return Err(PlanError::Unsupported { - detail: "traversal_work_limit must use the captured typed allowance, not a duplicate settings entry".to_string(), - }); + return Err(PlanError::refused( + P003, + "traversal_work_limit must use the captured typed allowance, not a duplicate settings entry", + None, + )); } match self.traversal_work_limit { - Some(limit) if limit == 0 || limit > i64::MAX as u64 => Err(PlanError::Unsupported { - detail: "traversal_work_limit must be in 1..=i64::MAX".to_string(), - }), + Some(limit) if limit == 0 || limit > i64::MAX as u64 => Err(PlanError::refused( + P003, + "traversal_work_limit must be in 1..=i64::MAX", + Some("set `traversal_work_limit` to a value in 1..=9223372036854775807"), + )), Some(limit) => Ok(NonZeroU64::new(limit)), - None if self.has_wildcard_traversal => Err(PlanError::Unsupported { - detail: "wildcard traversal requires a finite traversal_work_limit".to_string(), - }), + None if self.has_wildcard_traversal => Err(PlanError::refused( + P002, + "wildcard traversal requires a finite traversal_work_limit", + Some(SET_TRAVERSAL_WORK_LIMIT), + )), None => Ok(None), } } diff --git a/crates/omnigraph-planner/tests/query_plan/edge_selections.rs b/crates/omnigraph-planner/tests/query_plan/edge_selections.rs index 65d5ea801..2dacc451c 100644 --- a/crates/omnigraph-planner/tests/query_plan/edge_selections.rs +++ b/crates/omnigraph-planner/tests/query_plan/edge_selections.rs @@ -628,25 +628,31 @@ fn issue_659_nested_selection_budgets_every_expand_and_pins_every_member() { #[test] fn issue_659_selection_admission_precedes_lowering_of_outer_named_edges() { let op = correlated(alternatives()); - for (source, expected) in [ - (source(), "require a finite traversal_work_limit"), + for (source, expected, code) in [ + (source(), "require a finite traversal_work_limit", "P002"), ( source().with_traversal_work_limit(0), "must be in 1..=i64::MAX", + "P003", ), ( source().with_traversal_work_limit(i64::MAX as u64 + 1), "must be in 1..=i64::MAX", + "P003", ), ( source() .with_traversal_work_limit(100) .with_traversal(Traversal::Csr), "do not support traversal = csr", + "P004", ), ] { let error = resolve(&op, &source).unwrap_err(); - assert!(matches!(error, PlanError::Unsupported { .. })); + let PlanError::Unsupported(diagnostic) = &error else { + panic!("a refusal by design: {error:?}"); + }; + assert_eq!(diagnostic.code.as_str(), code, "{error}"); assert!(error.to_string().contains(expected), "{error}"); } } diff --git a/crates/omnigraph/src/engine/plan_source.rs b/crates/omnigraph/src/engine/plan_source.rs index c1437a8d6..47aeaf6bc 100644 --- a/crates/omnigraph/src/engine/plan_source.rs +++ b/crates/omnigraph/src/engine/plan_source.rs @@ -338,6 +338,12 @@ impl PlanSource for QuerySource<'_> { } } +/// A query shape the planner refuses by design: the caller's error, a bad +/// request carrying the planner's diagnostic on every door. +fn unsupported_query(diagnostic: Box) -> OmniError { + OmniError::Compiler(omnigraph_compiler::error::CompilerError::Query(diagnostic)) +} + fn no_plan(reason: impl std::fmt::Display) -> OmniError { OmniError::manifest_internal(format!( "the planner built no plan for this query: {reason}" @@ -379,7 +385,7 @@ fn resolve_params(ir: &QueryIR, params: &ParamMap) -> Result { pub(crate) fn plan_query(source: &QuerySource<'_>) -> Result { omnigraph_planner::plan_query(&source.ir, source, &source.bounds()).map_err(|reason| { match reason { - Unrouted::UnsupportedQuery { message } => OmniError::manifest(message), + Unrouted::UnsupportedQuery { diagnostic } => unsupported_query(diagnostic), reason => no_plan(reason.to_json()), } }) @@ -407,9 +413,9 @@ pub(crate) fn explain_query(source: &QuerySource<'_>) -> Result physical: plan, }), Decision::Executor { - reason: Unrouted::UnsupportedQuery { message }, + reason: Unrouted::UnsupportedQuery { diagnostic }, .. - } => Err(OmniError::manifest(message)), + } => Err(unsupported_query(diagnostic)), Decision::Executor { reason, .. } => Err(no_plan(reason.to_json())), Decision::Routed { entry, .. } => Err(OmniError::manifest_internal(format!( "the registry routed a GQ query through entry `{}`; a read query runs only \ diff --git a/crates/omnigraph/tests/engine_v2_plan_replay.rs b/crates/omnigraph/tests/engine_v2_plan_replay.rs index c99a73a41..3086accf4 100644 --- a/crates/omnigraph/tests/engine_v2_plan_replay.rs +++ b/crates/omnigraph/tests/engine_v2_plan_replay.rs @@ -7,7 +7,7 @@ mod helpers; use omnigraph::db::{Omnigraph, ReadTarget}; -use omnigraph::error::{ManifestErrorKind, OmniError}; +use omnigraph::error::OmniError; use omnigraph::loader::LoadMode; use omnigraph_compiler::ir::ParamMap; use omnigraph_compiler::query::ast::Literal; @@ -994,10 +994,11 @@ async fn a_write_to_an_unread_table_leaves_the_replay_accepted() { } /// The planner's refusal of a search order on a traversal destination is the -/// caller's error (a bad request), not a planner defect: the HTTP door maps -/// the kind, which no `.gqt` case observes. +/// caller's error, a bad request carrying a plan diagnostic, on the ordinary +/// door and the inspected one alike; the HTTP door maps the kind, which no +/// `.gqt` case observes. #[tokio::test] -async fn a_search_order_on_a_traversal_destination_is_a_bad_request() { +async fn a_search_order_on_a_traversal_destination_is_a_typed_bad_request_issue_786() { let dir = tempfile::tempdir().unwrap(); let db = docs(&dir).await; let source = r#" @@ -1009,7 +1010,7 @@ query nearest_destination($q: Vector(4)) { } "#; let params = ParamMap::from([("q".to_string(), Literal::List(vec![Literal::Float(0.0); 4]))]); - let refused = db + let inspected = db .query_inspected( ReadTarget::branch("main"), source, @@ -1018,15 +1019,39 @@ query nearest_destination($q: Vector(4)) { ) .await .err() - .expect("the shape is refused"); - assert!( - matches!(&refused, OmniError::Manifest(error) if error.kind == ManifestErrorKind::BadRequest), - "{refused:?}" - ); - assert!( - refused.to_string().contains("a traversal destination"), - "{refused}" - ); + .expect("the shape is refused on the inspected door"); + let ordinary = db + .query( + ReadTarget::branch("main"), + source, + "nearest_destination", + ¶ms, + ) + .await + .expect_err("the shape is refused on the ordinary door"); + for refused in [&inspected, &ordinary] { + let diagnostic = refused + .diagnostic() + .unwrap_or_else(|| panic!("a plan refusal carries its diagnostic: {refused:?}")); + assert_eq!(diagnostic.code.as_str(), "P001"); + let stage = diagnostic + .stage + .as_deref() + .expect("a plan refusal names its stage"); + assert_eq!(stage.name, "plan"); + assert_eq!( + stage.expression.as_deref(), + Some("nearest($t.embedding, $q)") + ); + assert_eq!( + diagnostic.fix.as_deref(), + Some("declare `$t` first in `match`, so the ranking starts the traversal") + ); + assert!( + refused.to_string().contains("a traversal destination"), + "{refused}" + ); + } } /// `now()` is bound at gather and rides in the value table, so the replay diff --git a/crates/omnigraph/tests/rrf_prefilter_gate.rs b/crates/omnigraph/tests/rrf_prefilter_gate.rs index 38928993c..3fa16886d 100644 --- a/crates/omnigraph/tests/rrf_prefilter_gate.rs +++ b/crates/omnigraph/tests/rrf_prefilter_gate.rs @@ -36,7 +36,6 @@ use serial_test::serial; use omnigraph::Session; use omnigraph::db::Omnigraph; -use omnigraph::error::{ManifestErrorKind, OmniError}; use omnigraph::instrumentation::{ QueryIoProbes, RrfGateFallback, RrfGatePlan, RrfGateVerdict, with_query_io_probes, }; @@ -578,8 +577,11 @@ async fn shape_fence_covers_all_fallback_rows() { ) .await .expect_err("engine v2 refuses a ranked order on a traversal destination"); - assert!( - matches!(&refused, OmniError::Manifest(error) if error.kind == ManifestErrorKind::BadRequest), + assert_eq!( + refused + .diagnostic() + .map(|diagnostic| diagnostic.code.as_str()), + Some("P001"), "ranked_var_is_expand_dst: the refusal is a user error: {refused:?}" ); assert!( diff --git a/docs/user/queries/diagnostics.md b/docs/user/queries/diagnostics.md index 85e4a0ee3..f14101682 100644 --- a/docs/user/queries/diagnostics.md +++ b/docs/user/queries/diagnostics.md @@ -8,15 +8,16 @@ fix names the decision instead. | Field | Meaning | |---|---| -| `code` | Stable identifier: `Q…` from the parser, `T…` from the type checker. A code's meaning is frozen; its message text may improve. | +| `code` | Stable identifier: `Q…` from the parser, `T…` from the type checker, `P…` from the planner. A code's meaning is frozen; its message text may improve. | | `position` | For a parse refusal: `line` and `column` (1-based, in characters) and the `byte` offset. | -| `stage` and `expression` | For a refusal after parsing: the compiler stage (`typecheck`) and, when the site can render it, the expression it refused. | +| `stage` and `expression` | For a refusal after parsing: the stage (`typecheck` or `plan`) and, when the site can render it, the expression it refused. | | `expected` | What was expected or violated, one line, without a position. | | `fix` | One concrete fix, absent when `expected` names the decision. | | `suggestion` | Optional source edits with `applicability`, plus `start`, `end` and `replacement` for each edit. | The one-line `error` text is the code and the expectation in the legacy form -(`parse error: …`, `type error: T33: …`); the other fields travel beside it. +(`parse error: …`, `type error: T33: …`, `plan error: P001: …`); the other +fields travel beside it. ## The measured case @@ -114,3 +115,16 @@ do not mean there is no possible fix. Type codes are listed where the construct they guard is described, in [Query language](index.md). + +## Planner codes + +A planner code refuses a well-formed, type-checked query shape by design. It +is the caller's error, answered as a bad request with the diagnostic, never as +an internal error. + +| Code | Meaning | +|---|---| +| `P001` | A ranking (`nearest`, `bm25`) orders a binding that a traversal reaches from another binding. The fix declares that binding first in `match`, so the ranking starts the traversal. | +| `P002` | An edge alternation or wildcard traversal has no finite traversal work limit. The fix sets `traversal_work_limit` before the query. | +| `P003` | The traversal work limit is outside `1..=9223372036854775807`, or a plan records it twice. | +| `P004` | An edge alternation or wildcard traversal was asked to run in CSR traversal mode; it runs in `auto` or `indexed` mode. | From ccfe79864ee6062d369a59be2e58f42eb6c9cc67 Mon Sep 17 00:00:00 2001 From: Ragnor Comerford Date: Thu, 1 Oct 2026 16:31:41 +0200 Subject: [PATCH 02/29] engine: a refusal raised in the statistics pass keeps its diagnostic Before the planning that runs, `load_column_statistics` resolves and rewrites the query once to find the tables whose column statistics the cost model needs. It mapped every planning error through `no_plan`, so a refusal by design raised while resolving, such as an edge wildcard under CSR traversal mode (P004), came back as an internal error carrying the diagnostic only as text. It now maps a refusal through the same conversion the gate's result uses, and leaves `no_plan` for defects. --- crates/omnigraph/src/engine/plan_source.rs | 11 +++++++- .../omnigraph/tests/engine_v2_plan_replay.rs | 28 +++++++++++++++++++ 2 files changed, 38 insertions(+), 1 deletion(-) diff --git a/crates/omnigraph/src/engine/plan_source.rs b/crates/omnigraph/src/engine/plan_source.rs index 47aeaf6bc..2bfab6d0a 100644 --- a/crates/omnigraph/src/engine/plan_source.rs +++ b/crates/omnigraph/src/engine/plan_source.rs @@ -159,7 +159,7 @@ impl<'a> QuerySource<'a> { return Ok(()); } let tables = omnigraph_planner::optimizer::column_statistics_needed(operation, self) - .map_err(no_plan)?; + .map_err(plan_error)?; for type_key in tables { let dataset = Arc::new(self.snapshot.open_lance_dataset(&type_key).await?); if dataset.manifest().data_storage_format.lance_file_format() == ConcreteFileVersion::V1 @@ -344,6 +344,15 @@ fn unsupported_query(diagnostic: Box) -> Om OmniError::Compiler(omnigraph_compiler::error::CompilerError::Query(diagnostic)) } +/// A planning failure outside the gate: a refusal by design keeps its +/// diagnostic, anything else is a planner defect. +fn plan_error(error: PlanError) -> OmniError { + match error { + PlanError::Unsupported(diagnostic) => unsupported_query(diagnostic), + other => no_plan(other), + } +} + fn no_plan(reason: impl std::fmt::Display) -> OmniError { OmniError::manifest_internal(format!( "the planner built no plan for this query: {reason}" diff --git a/crates/omnigraph/tests/engine_v2_plan_replay.rs b/crates/omnigraph/tests/engine_v2_plan_replay.rs index 3086accf4..07863d61a 100644 --- a/crates/omnigraph/tests/engine_v2_plan_replay.rs +++ b/crates/omnigraph/tests/engine_v2_plan_replay.rs @@ -1052,6 +1052,34 @@ query nearest_destination($q: Vector(4)) { "{refused}" ); } + + // A refusal raised while the query is resolved, before lowering, keeps + // its diagnostic too: the statistics pass that resolves the query first + // must not turn it into a planner defect. An edge wildcard refuses CSR + // traversal mode, which only a session pin selects. + let csr = with_traversal(&db, omnigraph_compiler::settings::Traversal::Csr); + let wildcard = r#" +query wildcard() { + match { $a: Doc $b: Doc $a * $b } + return { $b.slug } +} +"#; + let refused = csr + .query( + ReadTarget::branch("main"), + wildcard, + "wildcard", + &ParamMap::new(), + ) + .await + .expect_err("an edge wildcard refuses CSR traversal mode"); + assert_eq!( + refused + .diagnostic() + .map(|diagnostic| diagnostic.code.as_str()), + Some("P004"), + "{refused:?}" + ); } /// `now()` is bound at gather and rides in the value table, so the replay From c455400430275de0a7641ef14bb781556332fd2b Mon Sep 17 00:00:00 2001 From: Ragnor Comerford Date: Thu, 1 Oct 2026 17:19:45 +0200 Subject: [PATCH 03/29] planner: accept every read plan against its checked query One validator in omnigraph-planner (validate/) constructs the only executable plan type, AcceptedPlan, whose fields are private. Ordinary execution, the explain document, the explain statement and the inspected run call the same accept_query path; explain renders the accepted plan and reports validation.scope. The checked declaration and its type context travel beside the IR (CompiledQuery), and the requirements are derived from them: binding identity, search identity and approximation, retained eligibility predicates, correlated blocks, the projection and score origin, order and cut, and the declared candidate and probe caps. A fresh plan failing a check is a planner defect; an exhausted validation limit is a resource outcome. Replay decodes an envelope (query source and name, scope, schema digest, bound plan) within a byte limit, checks its versions first, refuses a changed schema as incompatible facts, and accepts the plan again against requirements derived afresh. The implicit tie-break now orders declared bindings before the bindings the lowering makes up (anonymous endpoints, cycle temps), as RFC 0047's comparator requires; the validator rejected the old order. --- crates/omnigraph-compiler/src/ir/lower.rs | 14 +- crates/omnigraph-compiler/src/ir/mod.rs | 2 + crates/omnigraph-compiler/src/lib.rs | 1 + .../omnigraph-compiler/src/query/checked.rs | 41 + crates/omnigraph-compiler/src/query/mod.rs | 1 + ...dpoint_ties_order_by_declared_bindings.gqt | 62 ++ crates/omnigraph-gqt/src/plan.rs | 60 +- crates/omnigraph-planner/src/explain.rs | 5 + crates/omnigraph-planner/src/gate.rs | 69 ++ crates/omnigraph-planner/src/lib.rs | 8 +- crates/omnigraph-planner/src/optimizer.rs | 15 +- crates/omnigraph-planner/src/physical.rs | 6 + .../omnigraph-planner/src/validate/budget.rs | 150 +++ .../src/validate/invariants.rs | 881 ++++++++++++++++++ crates/omnigraph-planner/src/validate/mod.rs | 216 +++++ .../omnigraph-planner/src/validate/replay.rs | 216 +++++ .../src/validate/requirements.rs | 603 ++++++++++++ crates/omnigraph/src/engine/bind.rs | 20 +- crates/omnigraph/src/engine/explain.rs | 25 +- crates/omnigraph/src/engine/mod.rs | 40 +- crates/omnigraph/src/engine/plan_source.rs | 206 ++-- crates/omnigraph/src/engine/report.rs | 21 +- crates/omnigraph/src/exec/query_doors.rs | 92 +- crates/omnigraph/src/runtime_cache.rs | 39 +- crates/omnigraph/tests/engine_v2.rs | 147 +++ .../omnigraph/tests/engine_v2_plan_replay.rs | 158 +++- .../tests/engine_v2_scrubbed_replay.rs | 17 +- 27 files changed, 2916 insertions(+), 199 deletions(-) create mode 100644 crates/omnigraph-compiler/src/query/checked.rs create mode 100644 crates/omnigraph-gqt/cases/v2/anonymous_endpoint_ties_order_by_declared_bindings.gqt create mode 100644 crates/omnigraph-planner/src/validate/budget.rs create mode 100644 crates/omnigraph-planner/src/validate/invariants.rs create mode 100644 crates/omnigraph-planner/src/validate/mod.rs create mode 100644 crates/omnigraph-planner/src/validate/replay.rs create mode 100644 crates/omnigraph-planner/src/validate/requirements.rs diff --git a/crates/omnigraph-compiler/src/ir/lower.rs b/crates/omnigraph-compiler/src/ir/lower.rs index 58a14276c..6e19b6dbd 100644 --- a/crates/omnigraph-compiler/src/ir/lower.rs +++ b/crates/omnigraph-compiler/src/ir/lower.rs @@ -21,18 +21,28 @@ struct FreshNames { temp: usize, } +const ANON_PREFIX: &str = "__anon_"; +const TEMP_PREFIX: &str = "__temp_"; + impl FreshNames { fn anon(&mut self) -> String { self.anon += 1; - format!("__anon_{}", self.anon) + format!("{ANON_PREFIX}{}", self.anon) } fn temp(&mut self, dst: &str) -> String { self.temp += 1; - format!("__temp_{}_{}", dst, self.temp) + format!("{TEMP_PREFIX}{}_{}", dst, self.temp) } } +/// Whether `variable` is a name the lowering made up: an anonymous `_` +/// endpoint or a cycle-closing or renamed temp. The query declares no such +/// binding, so its identity orders nothing a reader can see. +pub fn is_fresh_variable(variable: &str) -> bool { + variable.starts_with(ANON_PREFIX) || variable.starts_with(TEMP_PREFIX) +} + /// What every expression site shares: the parameters, the physical column /// spellings and where a `contains` left operand's type lives. Every emitted /// expression lowers through one, so `contains` resolution and `fold` apply everywhere. diff --git a/crates/omnigraph-compiler/src/ir/mod.rs b/crates/omnigraph-compiler/src/ir/mod.rs index 4097fc497..cb136706d 100644 --- a/crates/omnigraph-compiler/src/ir/mod.rs +++ b/crates/omnigraph-compiler/src/ir/mod.rs @@ -2,6 +2,8 @@ pub mod fold; pub(crate) mod lower; pub(crate) mod validate; +pub use lower::is_fresh_variable; + use std::collections::HashMap; use crate::query::ast::{AggFunc, BinaryOp, CompOp, Literal, NOW_PARAM_NAME, Param, Precedence}; diff --git a/crates/omnigraph-compiler/src/lib.rs b/crates/omnigraph-compiler/src/lib.rs index 063303851..25ada4e76 100644 --- a/crates/omnigraph-compiler/src/lib.rs +++ b/crates/omnigraph-compiler/src/lib.rs @@ -42,6 +42,7 @@ pub const GQ_LANGUAGE_VERSION: (u16, u16) = (2, 1); pub use ir::lower::{lower_mutation_query, lower_query}; pub use lint::{DiagnosticCode, Family, SafetyTier, Severity}; pub use query::ast::Literal; +pub use query::checked::CheckedQuery; pub use query::descriptor::{ QueryGraphFact, QueryGraphFactKind, QueryOperationDescriptor, QueryResultFieldDescriptor, QueryValueKind, describe_query_operation, diff --git a/crates/omnigraph-compiler/src/query/checked.rs b/crates/omnigraph-compiler/src/query/checked.rs new file mode 100644 index 000000000..abacf0d74 --- /dev/null +++ b/crates/omnigraph-compiler/src/query/checked.rs @@ -0,0 +1,41 @@ +//! A read declaration the type checker accepted, kept beside its IR so a +//! later stage derives what the query requires from what it wrote, not from +//! the IR alone (RFC 0047, "Requirements come from the checked query"). The +//! IR loses meaning on the way: a projected `bm25($d.title, $q)` becomes the +//! column `$d._score`, inline matches become comparisons, and literals fold. + +use std::sync::Arc; + +use crate::catalog::Catalog; +use crate::error::Result; +use crate::query::ast::QueryDecl; +use crate::query::typecheck::{TypeContext, typecheck_query}; + +/// One checked read declaration and the type context the checker built for +/// it. Shared by `Arc`: a compiled query is cached and read by every door. +#[derive(Debug, Clone)] +pub struct CheckedQuery { + decl: Arc, + types: Arc, +} + +impl CheckedQuery { + /// Type-check `decl` against `catalog`, keeping both. + pub fn check(catalog: &Catalog, decl: &QueryDecl) -> Result { + let types = typecheck_query(catalog, decl)?; + Ok(Self { + decl: Arc::new(decl.clone()), + types: Arc::new(types), + }) + } + + /// The declaration as the query wrote it. + pub fn decl(&self) -> &QueryDecl { + &self.decl + } + + /// The type context the checker built for [`Self::decl`]. + pub fn types(&self) -> &TypeContext { + &self.types + } +} diff --git a/crates/omnigraph-compiler/src/query/mod.rs b/crates/omnigraph-compiler/src/query/mod.rs index b8c2e246b..a1f4387a5 100644 --- a/crates/omnigraph-compiler/src/query/mod.rs +++ b/crates/omnigraph-compiler/src/query/mod.rs @@ -1,4 +1,5 @@ pub mod ast; +pub mod checked; pub mod codes; pub mod descriptor; pub mod diagnostic; diff --git a/crates/omnigraph-gqt/cases/v2/anonymous_endpoint_ties_order_by_declared_bindings.gqt b/crates/omnigraph-gqt/cases/v2/anonymous_endpoint_ties_order_by_declared_bindings.gqt new file mode 100644 index 000000000..5749c83e8 --- /dev/null +++ b/crates/omnigraph-gqt/cases/v2/anonymous_endpoint_ties_order_by_declared_bindings.gqt @@ -0,0 +1,62 @@ +# issue: none +# notes: The implicit tie-break orders by every declared binding's identity in +# notes: binding-name order (RFC 0047, "One total order"). An anonymous `$_` +# notes: endpoint is no declared binding: its identity may only follow the +# notes: declared ones. `a` and `b` tie on rank; `a` reaches doc `y` and `b` +# notes: reaches doc `x`, so ordering by the anonymous endpoint first would +# notes: put `b` before `a`. + +--- runner +timeout_ms: 10000 +environments: + - target: omnigraph-engine + storage: local-filesystem + +--- schema +node Person { + name: String @key + rank: I64 +} +node Doc { + slug: String @key +} +edge Likes: Person -> Doc + +--- seed +{"type":"Person","data":{"name":"a","rank":1}} +{"type":"Person","data":{"name":"b","rank":1}} +{"type":"Doc","data":{"slug":"x"}} +{"type":"Doc","data":{"slug":"y"}} +{"edge":"Likes","from":"a","to":"y"} +{"edge":"Likes","from":"b","to":"x"} + +--- query +query first_by_rank() { + match { $p: Person $p likes $_ } + return { $p.name } + order { $p.rank } + limit 1 +} + +--- expect ordered +{"p.name": "a"} + +--- expect shape +p.name: String + +--- query +query all_by_rank() { + match { $p: Person $p likes $_ } + return { $p.name } + order { $p.rank } +} + +--- expect ordered +{"p.name": "a"} +{"p.name": "b"} + +--- expect shape +p.name: String + +--- expect plan +sort tiebreak [$p.@id, $__anon_1.@id] diff --git a/crates/omnigraph-gqt/src/plan.rs b/crates/omnigraph-gqt/src/plan.rs index 93b27c490..41725486a 100644 --- a/crates/omnigraph-gqt/src/plan.rs +++ b/crates/omnigraph-gqt/src/plan.rs @@ -23,6 +23,7 @@ //! filter reads [a.state, b.state] //! pass projection_pushdown //! not pass aggregate_pushdown +//! validation scope exact_subset //! ``` //! //! A `ran` claim names the side of the node's declared switch that ran, @@ -30,6 +31,7 @@ //! node's row), joined to the explain row by the node's `id`. use omnigraph_compiler::catalog::Catalog; +use omnigraph_planner::ValidationScope; use omnigraph_planner::optimizer::{ PASS_ACCESS_PATH, PASS_ADDRESS_SHORT_CIRCUIT, PASS_AGGREGATE_PUSHDOWN, PASS_EXPAND_MODE, PASS_FRAGMENT_SCOPE, PASS_JOIN_ALGORITHM, PASS_LATE_MATERIALIZATION, PASS_PREDICATE_PUSHDOWN, @@ -40,7 +42,7 @@ use serde_json::Value; use crate::report::Row; -const FORMS: &str = "forms: `scan [ as $var]: columns [a, b]`, `scan [ as $var]: not columns [a, b]`, `scan [ as $var]: filter reads [v.a]`, `scan [ as $var]: no filter`, `scan [ as $var]: access id_lookup`, `hash join $var[ ran ]`, `scan [ as $var]: ranked [ fetch ][ nprobes ]`, `scan [ as $var]: runtime filter `, `scan [ as $var]: no runtime filter`, `contains join $h.x contains $n.y`, `cross join $h.x contains $n.y`, `expand $src $dst: mode [ ran ]`, `filter reads [a.x, b.y]`, `sort tiebreak [$a.@id, $e.@type]`, `rank fuse row tiebreak [$e.@type, $e.@id]`, `expand $a $b: selection alternation [Knows out, Likes in]`, `sort no tiebreak`, `pass `, `not pass `"; +const FORMS: &str = "forms: `scan [ as $var]: columns [a, b]`, `scan [ as $var]: not columns [a, b]`, `scan [ as $var]: filter reads [v.a]`, `scan [ as $var]: no filter`, `scan [ as $var]: access id_lookup`, `hash join $var[ ran ]`, `scan [ as $var]: ranked [ fetch ][ nprobes ]`, `scan [ as $var]: runtime filter `, `scan [ as $var]: no runtime filter`, `contains join $h.x contains $n.y`, `cross join $h.x contains $n.y`, `expand $src $dst: mode [ ran ]`, `filter reads [a.x, b.y]`, `sort tiebreak [$a.@id, $e.@type]`, `rank fuse row tiebreak [$e.@type, $e.@id]`, `expand $a $b: selection alternation [Knows out, Likes in]`, `sort no tiebreak`, `pass `, `not pass `, `validation scope `"; const ID_LOOKUP: &str = "id_lookup"; const JOIN_SIDES: [&str; 2] = ["hash_join", "id_lookup"]; @@ -130,6 +132,9 @@ pub(crate) enum PlanLine { Sort { tiebreak: Vec }, /// The optimizer pass `name` fired, or did not when `negated`. Pass { name: String, negated: bool }, + /// Acceptance checked the plan to `scope` (`exact_subset` or + /// `invariants_only`), as the document's `validation` reports it. + ValidationScope { scope: String }, } #[derive(Debug, Default)] @@ -179,6 +184,21 @@ pub(crate) fn parse_plan_body(body: &[(usize, &str)]) -> Result, S }); continue; } + if let Some(scope) = line.strip_prefix("validation scope ") { + let scope = scope.trim(); + if ![ + ValidationScope::ExactSubset.as_str(), + ValidationScope::InvariantsOnly.as_str(), + ] + .contains(&scope) + { + return Err(refused("names `exact_subset` or `invariants_only`")); + } + lines.push(PlanLine::ValidationScope { + scope: scope.to_string(), + }); + continue; + } if let Some(list) = line.strip_prefix("filter reads") { let reads = column_list(list).ok_or_else(|| refused("lists reads in `[...]`"))?; lines.push(PlanLine::Filter { reads }); @@ -859,6 +879,20 @@ pub(crate) fn plan_mismatch( .unwrap_or_default(); for line in lines { match line { + PlanLine::ValidationScope { scope } => { + let reported = explain + .get("validation") + .and_then(|validation| validation.get("scope")) + .and_then(Value::as_str); + if reported != Some(scope.as_str()) { + return Some(format!( + "expect plan: validation scope {scope}; the document reports {}", + reported.map_or("no validation".to_string(), |reported| format!( + "scope {reported}" + )) + )); + } + } PlanLine::Pass { name, negated } => { let fired = passes.iter().any(|pass| pass == name); if fired && *negated { @@ -1411,6 +1445,30 @@ mod tests { assert_eq!(check(&lines, &explain()), None); } + /// The scope claim reads the document's `validation`; a different scope + /// or a document without one fails, and an unknown scope is refused. + #[test] + fn validation_scope_claims_read_the_documents_validation() { + let lines = parse_plan_body(&[(0, "validation scope invariants_only")]).unwrap(); + assert_eq!( + lines[0], + PlanLine::ValidationScope { + scope: "invariants_only".to_string() + } + ); + let accepted = json!({"validation": {"scope": "invariants_only"}}); + assert_eq!(check(&lines, &accepted), None); + let exact = json!({"validation": {"scope": "exact_subset"}}); + let mismatch = check(&lines, &exact).unwrap(); + assert!( + mismatch.contains("reports scope exact_subset"), + "{mismatch}" + ); + let mismatch = check(&lines, &explain()).unwrap(); + assert!(mismatch.contains("reports no validation"), "{mismatch}"); + assert!(parse_plan_body(&[(0, "validation scope everything")]).is_err()); + } + #[test] fn a_read_column_fails_the_negative_form() { let lines = parse_plan_body(&[(0, "scan Doc: not columns [slug]")]).unwrap(); diff --git a/crates/omnigraph-planner/src/explain.rs b/crates/omnigraph-planner/src/explain.rs index dd293b419..9c4477117 100644 --- a/crates/omnigraph-planner/src/explain.rs +++ b/crates/omnigraph-planner/src/explain.rs @@ -9,6 +9,7 @@ use crate::operation::{Operation, PageBudgetSpec, ScopeSpec}; use crate::physical::StatisticSource; use crate::registry::{Entry, Route}; use crate::route::RouteOverride; +use crate::validate::ValidationSummary; /// Incremented when a field's meaning changes, or when a node kind is added /// or removed; additive keys do not bump it. Version 2: the logical `Filter` @@ -119,6 +120,9 @@ pub struct Explain { #[serde(skip_serializing_if = "Option::is_none")] pub statistics: Option>, pub passes: Vec<&'static str>, + /// What acceptance checked, on a read query's accepted plan. + #[serde(skip_serializing_if = "Option::is_none")] + pub validation: Option, } impl Explain { @@ -136,6 +140,7 @@ impl Explain { pipelines: None, statistics: None, passes: Vec::new(), + validation: None, } } diff --git a/crates/omnigraph-planner/src/gate.rs b/crates/omnigraph-planner/src/gate.rs index df4510a0d..6d37da49b 100644 --- a/crates/omnigraph-planner/src/gate.rs +++ b/crates/omnigraph-planner/src/gate.rs @@ -23,6 +23,7 @@ use crate::source::{ AdjacencyProof, EXPAND_INDEXED_MAX_FRONTIER_ENV, EXPAND_INDEXED_MAX_HOPS_ENV, ExpandStatistics, FragmentStat, NodeTypeSpec, PlanSource, SideId, }; +use crate::validate::{self, AcceptInput, AcceptedPlan, ValidationError}; /// A `PlanSource` that records what the planner read through it: the /// parameter names of every filter it asked about and every setting, so the @@ -234,6 +235,9 @@ pub enum Unrouted { UnsupportedQuery { diagnostic: Box, }, + /// Plan acceptance ran out of a configured validation limit before it + /// finished: a resource outcome, not evidence the plan is invalid. + ValidationExhausted { limit: &'static str, value: u64 }, } impl Unrouted { @@ -257,6 +261,7 @@ impl Unrouted { Self::Override => "override", Self::PlannerError { .. } => "planner_error", Self::UnsupportedQuery { .. } => "unsupported_query", + Self::ValidationExhausted { .. } => "validation_exhausted", } } @@ -275,6 +280,9 @@ impl Unrouted { "code": diagnostic.code.as_str(), "message": diagnostic.message, }), + Self::ValidationExhausted { limit, value } => { + json!({ "kind": self.kind(), "limit": limit, "value": value }) + } } } } @@ -333,6 +341,67 @@ pub fn plan_query( .map_err(Unrouted::of) } +/// Plan a read query and accept the plan: the one path from a query to an +/// executable plan, shared by ordinary execution and every explain. +pub fn accept_query( + input: &AcceptInput<'_>, + source: &dyn PlanSource, + bounds: &Bounds, +) -> Result { + plan_and_accept(input, source, bounds, false).map(|(accepted, _)| accepted) +} + +/// [`accept_query`] with the explain document of the accepted plan, rendered +/// from the same planning run. +pub fn accept_query_explained( + input: &AcceptInput<'_>, + source: &dyn PlanSource, + bounds: &Bounds, +) -> Result<(AcceptedPlan, Explain), Unrouted> { + let (accepted, explain) = plan_and_accept(input, source, bounds, true)?; + let explain = explain.ok_or_else(|| Unrouted::PlannerError { + message: "an explained acceptance rendered no document".to_string(), + })?; + Ok((accepted, explain)) +} + +fn plan_and_accept( + input: &AcceptInput<'_>, + source: &dyn PlanSource, + bounds: &Bounds, + explain: bool, +) -> Result<(AcceptedPlan, Option), Unrouted> { + let query = input.ir; + let operation = Operation::Query(Box::new(query.clone())); + let recorded = Recorded::new(source, query.has_wildcard_traversal()); + let mut logical = resolve(&operation, &recorded).map_err(Unrouted::of)?; + let fired = rewrite(&mut logical, &recorded).map_err(Unrouted::of)?; + let mut optimized = + physical_plan(&mut logical, &recorded, bounds, fired.clone()).map_err(Unrouted::of)?; + optimized + .physical + .set_assumptions(recorded.assumptions(bounds)); + let rendered = explain.then(|| { + LogicalView::of(&logical) + .explain( + OperationSummary::of(&operation), + RouteOverride::Registry, + None, + Some(&optimized), + &fired, + ) + .engine() + }); + let accepted = + validate::accept(optimized.physical, input).map_err(ValidationError::into_unrouted)?; + let rendered = rendered.map(|mut explain| { + explain.pipelines = None; + explain.validation = Some(accepted.summary()); + explain + }); + Ok((accepted, rendered)) +} + /// Decide the route of one operation. The census is computed from the /// operation and the plan source only; the override is applied after the /// registry lookup. The logical plan is built for every operation and the diff --git a/crates/omnigraph-planner/src/lib.rs b/crates/omnigraph-planner/src/lib.rs index e8a188e7c..37071f3c3 100644 --- a/crates/omnigraph-planner/src/lib.rs +++ b/crates/omnigraph-planner/src/lib.rs @@ -36,6 +36,7 @@ pub mod physical; pub mod registry; pub mod route; pub mod source; +pub mod validate; pub use bound::{BOUND_PLAN_VERSION, BoundPlan, ValueTable}; pub use cost::{ @@ -46,7 +47,7 @@ pub use cost::{ }; pub use error::PlanError; pub use explain::Explain; -pub use gate::{Decision, Unrouted, plan_query, route}; +pub use gate::{Decision, Unrouted, accept_query, accept_query_explained, plan_query, route}; pub use logical::{ Census, ColumnRef, JoinKind, KeyJoinKind, LogicalId, LogicalKind, LogicalNode, LogicalPlan, Predicate, RuntimeFilterKind, RuntimeFilterSpec, ScanSpec, SearchArm, @@ -67,6 +68,11 @@ pub use source::{ AdjacencyProof, EXPAND_INDEXED_MAX_FRONTIER_ENV, EXPAND_INDEXED_MAX_HOPS_ENV, ExpandStatistics, FragmentStat, MemorySource, NodeTypeSpec, PlanSource, SideId, }; +pub use validate::{ + AcceptInput, AcceptedBoundPlan, AcceptedPlan, ConstantEvaluator, Evidence, ReplayEnvelope, + ReplayQuery, ReplayRefusal, ValidationError, ValidationLimits, ValidationScope, + ValidationSummary, accept_replay, catalog_digest, decode_replay, +}; #[cfg(test)] #[path = "../tests/support/bounds.rs"] diff --git a/crates/omnigraph-planner/src/optimizer.rs b/crates/omnigraph-planner/src/optimizer.rs index 8d3bcbf74..186fba1f3 100644 --- a/crates/omnigraph-planner/src/optimizer.rs +++ b/crates/omnigraph-planner/src/optimizer.rs @@ -7,7 +7,7 @@ use std::sync::Arc; use arrow_schema::{DataType, Field, Schema, SchemaRef}; use omnigraph_compiler::QueryDiagnostic; -use omnigraph_compiler::ir::{IRExpr, IROp, IROrdering, IRProjection, QueryIR}; +use omnigraph_compiler::ir::{IRExpr, IROp, IROrdering, IRProjection, QueryIR, is_fresh_variable}; use omnigraph_compiler::query::ast::AggFunc; use omnigraph_compiler::query::codes::{P001, P002, P004}; use omnigraph_compiler::settings::Traversal; @@ -647,8 +647,11 @@ fn visible_scope(plan: &LogicalPlan, id: LogicalId) -> Vec<&LogicalNode> { nodes } -/// Candidate identity keys in binding order. Physical spellings are used -/// only to recognize user order keys; the plan retains logical metadata. +/// Candidate identity keys: every binding the query declares in +/// binding-name order, then the bindings the lowering made up (anonymous +/// endpoints, cycle temps), whose identities only order rows a reader cannot +/// tell apart. Physical spellings are used only to recognize user order +/// keys; the plan retains logical metadata. fn scope_tiebreaks( plan: &LogicalPlan, id: LogicalId, @@ -678,6 +681,8 @@ fn scope_tiebreaks( _ => {} } } + let mut bindings: Vec<_> = bindings.into_iter().collect(); + bindings.sort_by_key(|(binding, _)| is_fresh_variable(binding)); let mut out = Vec::new(); for (binding, (id_column, selected_edge)) in bindings { if selected_edge { @@ -1204,6 +1209,8 @@ fn expected_rank_fuse_row_tiebreaks( } } bindings.remove(fused_binding); + let mut bindings: Vec<_> = bindings.into_iter().collect(); + bindings.sort_by_key(|(binding, _)| is_fresh_variable(binding)); let mut keys = Vec::new(); for (binding, selected) in bindings { if selected { @@ -3062,7 +3069,7 @@ fn variable_offset_width(data_type: &DataType) -> u64 { /// The candidates a nearest arm of an `rrf()` asks for when the query has /// no limit. -const RRF_NEAREST_ARM_K: usize = 100; +pub(crate) const RRF_NEAREST_ARM_K: usize = 100; /// The ordering a ranked scan or a fusion carries: `nearest` ranks by /// ascending `_distance`, `bm25` by descending `_score`, `rrf` by the fused rank. diff --git a/crates/omnigraph-planner/src/physical.rs b/crates/omnigraph-planner/src/physical.rs index a1a818611..11e9f0317 100644 --- a/crates/omnigraph-planner/src/physical.rs +++ b/crates/omnigraph-planner/src/physical.rs @@ -33,6 +33,12 @@ pub enum RankKind { } impl RankKind { + /// Whether the retrieval's contract is approximate: `nearest` membership + /// is, even when a run scores flat; `bm25` matching is exact. + pub fn approximate(self) -> bool { + self == Self::Nearest + } + /// The column Lance appends and the direction the query sorts it by. pub fn score(self) -> (&'static str, bool) { match self { diff --git a/crates/omnigraph-planner/src/validate/budget.rs b/crates/omnigraph-planner/src/validate/budget.rs new file mode 100644 index 000000000..16d710bbc --- /dev/null +++ b/crates/omnigraph-planner/src/validate/budget.rs @@ -0,0 +1,150 @@ +//! The finite limits acceptance runs under (RFC 0047, "Evidence has bounded +//! size and checking work"). Every check charges its work before doing it, +//! so the same query, values, facts, evidence and limits always produce the +//! same accept, violation or exhaustion; wall time decides nothing here. + +use serde::{Deserialize, Serialize}; + +use super::ValidationError; + +/// The validation inputs that bound its cost, captured with the evidence. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +pub struct ValidationLimits { + /// Bytes of serialized evidence a replay may supply, checked before any + /// decoding. + pub evidence_bytes: u64, + /// Typed nodes a derivation may hold. + pub nodes: u64, + /// Rule applications a derivation may record. + pub steps: u64, + /// Expression and plan nodes the checks may visit, decoding, + /// reconstruction and property derivation included. + pub work: u64, +} + +impl ValidationLimits { + /// Limits far above what any accepted plan needs: a derivation of the + /// exact fragment holds a handful of nodes and steps. + pub const DEFAULT: Self = Self { + evidence_bytes: 1 << 20, + nodes: 4096, + steps: 1024, + work: 1 << 20, + }; +} + +impl Default for ValidationLimits { + fn default() -> Self { + Self::DEFAULT + } +} + +/// The running charge against [`ValidationLimits`]. +#[derive(Debug)] +pub struct Budget { + limits: ValidationLimits, + nodes: u64, + steps: u64, + work: u64, +} + +impl Budget { + pub fn new(limits: ValidationLimits) -> Self { + Self { + limits, + nodes: 0, + steps: 0, + work: 0, + } + } + + pub fn limits(&self) -> ValidationLimits { + self.limits + } + + /// Charge `count` visits before making them. + pub fn visit(&mut self, count: u64) -> Result<(), ValidationError> { + charge(&mut self.work, count, self.limits.work, "work") + } + + /// Charge one retained derivation node before allocating it. + pub fn node(&mut self) -> Result<(), ValidationError> { + charge(&mut self.nodes, 1, self.limits.nodes, "nodes")?; + self.visit(1) + } + + /// Charge one rule application before reconstructing its successor. + pub fn step(&mut self) -> Result<(), ValidationError> { + charge(&mut self.steps, 1, self.limits.steps, "steps")?; + self.visit(1) + } + + /// Refuse `bytes` of serialized evidence above the byte limit, before it + /// is decoded. + pub fn evidence_bytes(&self, bytes: u64) -> Result<(), ValidationError> { + if bytes > self.limits.evidence_bytes { + return Err(ValidationError::Exhausted { + limit: "evidence_bytes", + value: self.limits.evidence_bytes, + }); + } + Ok(()) + } +} + +fn charge( + used: &mut u64, + count: u64, + limit: u64, + name: &'static str, +) -> Result<(), ValidationError> { + let next = used.saturating_add(count); + if next > limit { + return Err(ValidationError::Exhausted { + limit: name, + value: limit, + }); + } + *used = next; + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn a_charge_past_its_limit_exhausts_before_counting() { + let mut budget = Budget::new(ValidationLimits { + evidence_bytes: 8, + nodes: 1, + steps: 1, + work: 3, + }); + budget.node().unwrap(); + assert_eq!( + budget.node(), + Err(ValidationError::Exhausted { + limit: "nodes", + value: 1 + }) + ); + budget.step().unwrap(); + assert_eq!( + budget.visit(2), + Err(ValidationError::Exhausted { + limit: "work", + value: 3 + }) + ); + budget.visit(1).unwrap(); + assert!(budget.evidence_bytes(8).is_ok()); + assert_eq!( + budget.evidence_bytes(9), + Err(ValidationError::Exhausted { + limit: "evidence_bytes", + value: 8 + }) + ); + } +} diff --git a/crates/omnigraph-planner/src/validate/invariants.rs b/crates/omnigraph-planner/src/validate/invariants.rs new file mode 100644 index 000000000..0b21cd333 --- /dev/null +++ b/crates/omnigraph-planner/src/validate/invariants.rs @@ -0,0 +1,881 @@ +//! The checks every accepted plan passes (RFC 0047, "Checks have explicit +//! boundaries"): binding identity, search identity and its declared +//! approximation, retained eligibility predicates, correlated blocks, the +//! projection and the origin of every projected score, the required order +//! and the row cut, and the policy data the plan declares for its search. +//! They establish that the plan keeps what the query requires; they do not +//! prove row-selection equivalence, which only the exact subset's checked +//! derivation does. +//! +//! The order of a fused (`rrf`) search and of a search-ordered aggregate is +//! not checked here: the plan does not yet implement the total order those +//! shapes require (RFC 0047, "One total order"), and the step that does +//! extends these checks. + +use omnigraph_compiler::ir::{IRExpr, IROrdering, IRProjection}; +use omnigraph_compiler::query::ast::{Expr, Literal}; +use omnigraph_compiler::query::typecheck::BoundVariable; + +use super::budget::Budget; +use super::requirements::{ + Matcher, Position, Requirements, Retrieval, Search, result_column, score_column, +}; +use super::{AcceptInput, ValidationError}; +use crate::logical::{ColumnRef, EDGE_TYPE_MEMBER, IDENTITY_MEMBER}; +use crate::lower::ContainsJoinFields; +use crate::optimizer::RRF_NEAREST_ARM_K; +use crate::physical::{ + NodeId, OverfetchRung, PhysicalNode, PhysicalPlan, RankKind, RankScope, RankedAccess, +}; + +/// The nodes of one pipeline: every node reachable from `root` without +/// entering a correlated block's inner tree. +fn pipeline( + plan: &PhysicalPlan, + root: NodeId, + budget: &mut Budget, +) -> Result, ValidationError> { + let mut nodes = Vec::new(); + let mut pending = vec![root]; + while let Some(id) = pending.pop() { + budget.visit(1)?; + let node = plan.node(id).ok_or_else(|| { + ValidationError::violated("plan shape", format!("node {id} is a tombstone")) + })?; + nodes.push(id); + match node { + PhysicalNode::AntiJoin { input, .. } => pending.push(*input), + other => pending.extend(other.inputs()), + } + } + Ok(nodes) +} + +/// The pipelines whose rows a query's eligibility constrains: one per `rrf` +/// arm, each its own copy of the match, or the whole top level. +fn constrained_pipelines( + plan: &PhysicalPlan, + budget: &mut Budget, +) -> Result>, ValidationError> { + let top = pipeline(plan, plan.root(), budget)?; + let fuse = top.iter().find_map(|id| match plan.node(*id) { + Some(PhysicalNode::RankFuse { arms, .. }) => Some(arms.clone()), + _ => None, + }); + match fuse { + Some(arms) => arms + .iter() + .map(|arm| pipeline(plan, arm.input, budget)) + .collect(), + None => Ok(vec![top]), + } +} + +/// Every conjunct a pipeline's nodes test, wherever placement put it. +fn conjuncts(plan: &PhysicalPlan, nodes: &[NodeId]) -> Vec { + let mut out = Vec::new(); + for id in nodes { + match plan.node(*id) { + Some(PhysicalNode::Scan { spec, .. }) => { + if let Some(filter) = &spec.filter { + out.extend(filter.gq_filters()); + } + } + Some( + PhysicalNode::Filter { filters, .. } | PhysicalNode::CrossJoin { filters, .. }, + ) => { + out.extend(filters.iter().cloned()); + } + Some(PhysicalNode::ContainsJoin { + haystack, + needle, + residual, + .. + }) => { + let fields = ContainsJoinFields { + haystack: (&haystack.0, &haystack.1), + needle: (&needle.0, &needle.1), + residual, + }; + out.push(fields.conjunct()); + out.extend(residual.iter().cloned()); + } + _ => {} + } + } + out +} + +/// The ranked scans of the plan with their ids. +fn ranked_scans(plan: &PhysicalPlan) -> Vec<(NodeId, &str, &RankedAccess)> { + plan.live() + .filter_map(|(id, node)| match node { + PhysicalNode::Scan { + spec, + ranked: Some(ranked), + .. + } => Some((id, spec.binding.as_deref().unwrap_or_default(), ranked)), + _ => None, + }) + .collect() +} + +impl Requirements { + pub(crate) fn check( + &self, + plan: &PhysicalPlan, + input: &AcceptInput<'_>, + budget: &mut Budget, + ) -> Result<(), ValidationError> { + let matcher = Matcher::new(input, self); + let pipelines = constrained_pipelines(plan, budget)?; + for nodes in &pipelines { + self.check_bindings(plan, nodes)?; + self.check_eligibility(plan, nodes, &matcher, budget)?; + } + let top = pipeline(plan, plan.root(), budget)?; + self.check_blocks(plan, &top, &matcher, budget)?; + self.check_search(plan, &matcher, budget)?; + self.check_returns(plan, &top, &matcher, budget)?; + self.check_order_and_cut(plan, &top, &matcher, budget)?; + Ok(()) + } + + /// Every written binding is read with its type: a node binding by a scan + /// (or a pushed-down count) of its node type, an edge binding by the + /// traversal that names it over its edge types; every traversal clause + /// is one traversal. + fn check_bindings(&self, plan: &PhysicalPlan, nodes: &[NodeId]) -> Result<(), ValidationError> { + for (name, bound) in &self.bindings { + let found = nodes.iter().any(|id| match (plan.node(*id), bound) { + ( + Some( + PhysicalNode::Scan { spec, .. } | PhysicalNode::MetadataCount { spec, .. }, + ), + BoundVariable::Node { type_name }, + ) => { + spec.binding.as_deref() == Some(name.as_str()) + && spec.table.node_type_name() == Some(type_name.as_str()) + } + ( + Some(PhysicalNode::Expand { + edge_binding: Some(edge), + edges, + .. + }), + BoundVariable::Edge { type_names }, + ) => { + let mut members: Vec<&str> = edges + .members() + .iter() + .map(|member| member.edge_type.as_str()) + .collect(); + let mut expected: Vec<&str> = type_names.iter().map(String::as_str).collect(); + members.sort_unstable(); + members.dedup(); + expected.sort_unstable(); + expected.dedup(); + edge == name && members == expected + } + _ => false, + }); + if !found { + return Err(ValidationError::violated( + "binding identity", + format!("no read of the plan binds `${name}` as the query declares it"), + )); + } + } + let expands = nodes + .iter() + .filter(|id| matches!(plan.node(**id), Some(PhysicalNode::Expand { .. }))) + .count(); + if expands != self.traversals { + return Err(ValidationError::violated( + "binding identity", + format!( + "the query writes {} traversals; the plan runs {expands}", + self.traversals + ), + )); + } + Ok(()) + } + + /// Every written eligibility conjunct is tested somewhere in each + /// pipeline the query's rows come from; a constant `true` constrains + /// nothing and needs no place. + fn check_eligibility( + &self, + plan: &PhysicalPlan, + nodes: &[NodeId], + matcher: &Matcher<'_>, + budget: &mut Budget, + ) -> Result<(), ValidationError> { + let placed = conjuncts(plan, nodes); + let mut written_conjuncts = Vec::new(); + for clause in &self.eligibility { + matcher.conjuncts(clause, &mut written_conjuncts); + } + for written in written_conjuncts { + if matcher.constant(written) && matcher.value(written) == Some(Literal::Bool(true)) { + continue; + } + let mut found = false; + for conjunct in &placed { + if matcher.matches(written, conjunct, Position::Filter, budget)? { + found = true; + break; + } + } + if !found { + return Err(ValidationError::violated( + "predicate retention", + format!("the plan tests no conjunct for `{written}`"), + )); + } + } + Ok(()) + } + + /// One correlated block of the top level per written block, each with + /// the aggregate and comparison the query wrote. + fn check_blocks( + &self, + plan: &PhysicalPlan, + top: &[NodeId], + matcher: &Matcher<'_>, + budget: &mut Budget, + ) -> Result<(), ValidationError> { + let mut planned: Vec<_> = top + .iter() + .filter_map(|id| match plan.node(*id) { + Some(PhysicalNode::AntiJoin { predicate, .. }) => Some(predicate), + _ => None, + }) + .collect(); + if planned.len() != self.blocks.len() { + return Err(ValidationError::violated( + "predicate retention", + format!( + "the query writes {} correlated blocks; the plan runs {}", + self.blocks.len(), + planned.len() + ), + )); + } + for block in &self.blocks { + let mut found = None; + for (index, predicate) in planned.iter().enumerate() { + if predicate.func == block.func + && predicate.op == block.op + && matcher.matches(&block.right, &predicate.right, Position::Plain, budget)? + { + found = Some(index); + break; + } + } + match found { + Some(index) => { + planned.remove(index); + } + None => { + return Err(ValidationError::violated( + "predicate retention", + format!( + "no correlated block of the plan compares `{} {} {}`", + block.func, block.op, block.right + ), + )); + } + } + } + Ok(()) + } + + /// The plan ranks exactly the retrievals the leading `order` key names, + /// each on its binding and property, with its argument and the + /// approximation its contract declares, in the scope it orders. + fn check_search( + &self, + plan: &PhysicalPlan, + matcher: &Matcher<'_>, + budget: &mut Budget, + ) -> Result<(), ValidationError> { + let ranked = ranked_scans(plan); + let fuse = plan.live().find_map(|(_, node)| match node { + PhysicalNode::RankFuse { arms, k, .. } => Some((arms, k)), + _ => None, + }); + let expected: Vec<(RankScope, &Retrieval)> = match &self.search { + None => Vec::new(), + Some(Search::Rank(retrieval)) => vec![(RankScope::Order, retrieval)], + Some(Search::Fuse { arms, .. }) => vec![ + (RankScope::Primary, &arms[0]), + (RankScope::Secondary, &arms[1]), + ], + }; + if ranked.len() != expected.len() { + return Err(ValidationError::violated( + "search identity", + format!( + "the query names {} retrievals; the plan ranks {} scans", + expected.len(), + ranked.len() + ), + )); + } + for (scope, retrieval) in &expected { + let Some((_, binding, access)) = + ranked.iter().find(|(_, _, access)| access.scope == *scope) + else { + return Err(ValidationError::violated( + "search identity", + format!("no ranked scan of the plan orders the `{scope:?}` retrieval"), + )); + }; + let same = *binding == retrieval.binding + && access.kind == retrieval.kind + && access.property == retrieval.property + && matcher.argument(&retrieval.query, &access.query, budget)?; + if !same { + return Err(ValidationError::violated( + "search identity", + format!( + "the plan ranks `${binding}.{}` by {:?} with `{}`; the query names {:?} of `${}.{}` with `{}`", + access.property, + access.kind, + access.query, + retrieval.kind, + retrieval.binding, + retrieval.property, + retrieval.query + ), + )); + } + if access.kind.approximate() != retrieval.approximate() { + return Err(ValidationError::violated( + "approximation", + format!( + "the plan declares the {:?} retrieval of `${binding}` {}; its contract is {}", + access.kind, + exactness(access.kind.approximate()), + exactness(retrieval.approximate()) + ), + )); + } + self.check_access(plan, access)?; + } + match (&self.search, fuse) { + (Some(Search::Fuse { arms, k }), Some((planned, planned_k))) => { + for (arm, retrieval) in planned.iter().zip(arms) { + if arm.binding != retrieval.binding || arm.kind != retrieval.kind { + return Err(ValidationError::violated( + "search identity", + format!( + "the fusion's arm ranks `${}` by {:?}; the query names {:?} of `${}`", + arm.binding, arm.kind, retrieval.kind, retrieval.binding + ), + )); + } + } + let same_k = match (k, planned_k) { + (None, None) => true, + (Some(written), Some(planned)) => matcher.argument(written, planned, budget)?, + _ => false, + }; + if !same_k { + return Err(ValidationError::violated( + "search identity", + "the fusion's `k` is not the one the query writes", + )); + } + } + (Some(Search::Fuse { .. }), None) => { + return Err(ValidationError::violated( + "search identity", + "the query fuses two retrievals; the plan has no fusion", + )); + } + (_, Some(_)) => { + return Err(ValidationError::violated( + "search identity", + "the plan fuses retrievals the query does not fuse", + )); + } + (_, None) => {} + } + Ok(()) + } + + /// The candidate cap, probe cap and overfetch ladder a ranked scan + /// declares, from the cut its scope requires: an ordering `nearest` + /// fetches the `limit` and may widen it by the declared ladder, a + /// fusion's `nearest` arm fetches the `limit` (or the default arm size) + /// and never widens, and `bm25` fetches every match. + fn check_access( + &self, + plan: &PhysicalPlan, + access: &RankedAccess, + ) -> Result<(), ValidationError> { + let limit = self.limit.and_then(|limit| usize::try_from(limit).ok()); + let (fetch, overfetch) = match (access.kind, access.scope) { + (RankKind::Bm25, _) => (None, Vec::new()), + (RankKind::Nearest, RankScope::Order) => { + (limit, limit.map(OverfetchRung::ladder).unwrap_or_default()) + } + (RankKind::Nearest, RankScope::Primary | RankScope::Secondary) => { + (Some(limit.unwrap_or(RRF_NEAREST_ARM_K)), Vec::new()) + } + }; + if access.fetch != fetch || access.overfetch != overfetch { + return Err(ValidationError::violated( + "row cut", + format!( + "the {:?} scan fetches {:?} with overfetch {:?}; the query's cut requires {fetch:?} with {overfetch:?}", + access.kind, access.fetch, access.overfetch + ), + )); + } + let setting = plan + .assumptions() + .settings + .get(omnigraph_compiler::settings::SettingId::AnnNprobes.name()); + let nprobes = match access.kind { + RankKind::Nearest => setting + .map(|value| match value.as_str() { + "0" => None, + value => value.parse::().ok(), + }) + .ok_or_else(|| { + ValidationError::violated( + "declared policy", + "a nearest scan's probe cap was read from no recorded `ann_nprobes`", + ) + })?, + RankKind::Bm25 => None, + }; + if access.nprobes != nprobes { + return Err(ValidationError::violated( + "declared policy", + format!( + "the {:?} scan probes {:?}; the recorded setting is {nprobes:?}", + access.kind, access.nprobes + ), + )); + } + Ok(()) + } + + /// The plan's projection is the query's `return`, item for item, and + /// every projected score is the score of the retrieval it names. + fn check_returns( + &self, + plan: &PhysicalPlan, + top: &[NodeId], + matcher: &Matcher<'_>, + budget: &mut Budget, + ) -> Result<(), ValidationError> { + let projections = projections_of(plan, top); + let [returns] = projections.as_slice() else { + return Err(ValidationError::violated( + "projection", + format!( + "the plan has {} projections of the query's `return`; it needs one", + projections.len() + ), + )); + }; + if returns.len() != self.returns.len() { + return Err(ValidationError::violated( + "projection", + format!( + "the plan returns {} items; the query returns {}", + returns.len(), + self.returns.len() + ), + )); + } + for (written, projected) in self.returns.iter().zip(returns.iter()) { + let alias = written + .alias + .clone() + .or_else(|| meta_field_result_key(&written.expr)); + if projected.alias != alias + || !matcher.matches(&written.expr, &projected.expr, Position::Return, budget)? + { + return Err(ValidationError::violated( + "projection", + format!( + "the plan returns `{}`; the query returns `{}`", + projected.expr, written.expr + ), + )); + } + } + for (_, call) in super::requirements::projected_rank_calls(&self.returns) { + let Some(projected) = projected_retrieval(call) else { + continue; + }; + let mut origin = false; + for retrieval in self.retrievals() { + if retrieval.kind == projected.kind + && retrieval.binding == projected.binding + && retrieval.property == projected.property + && retrieval.query == projected.query + { + origin = true; + break; + } + } + let planned = ranked_scans(plan).into_iter().any(|(_, binding, access)| { + binding == projected.binding + && access.kind == projected.kind + && access.property == projected.property + }); + if !origin || !planned { + return Err(ValidationError::violated( + "score origin", + format!( + "the projected `{call}` reads `${}.{}`, which no retrieval of the query with these arguments produces", + projected.binding, + score_column(projected.kind) + ), + )); + } + } + Ok(()) + } + + /// The plan's rows leave in the order the query requires and are cut + /// where it requires: the final sort's comparator is the search's score, + /// the remaining written keys and the binding identities, the `limit` + /// cuts its output, and nothing between them reorders or drops rows. + fn check_order_and_cut( + &self, + plan: &PhysicalPlan, + top: &[NodeId], + matcher: &Matcher<'_>, + budget: &mut Budget, + ) -> Result<(), ValidationError> { + let mut id = plan.root(); + if let Some(limit) = self.limit { + let rows = usize::try_from(limit).unwrap_or(usize::MAX); + match plan.node(id) { + Some(PhysicalNode::Limit { input, rows: cut }) if *cut == rows => id = *input, + _ => { + return Err(ValidationError::violated( + "row cut", + format!("the plan's root does not cut its rows to the `limit` {limit}"), + )); + } + } + } + let fused = matches!(self.search, Some(Search::Fuse { .. })); + if fused { + let planned = top.iter().find_map(|id| match plan.node(*id) { + Some(PhysicalNode::RankFuse { limit, .. }) => Some(*limit), + _ => None, + }); + let limit = self.limit.and_then(|limit| usize::try_from(limit).ok()); + if planned != Some(limit) { + return Err(ValidationError::violated( + "row cut", + format!("the fusion keeps {planned:?} rows; the query's `limit` is {limit:?}"), + )); + } + return Ok(()); + } + let score = match &self.search { + Some(Search::Rank(retrieval)) => Some(retrieval), + _ => None, + }; + if score.is_some() && self.aggregate { + return Ok(()); + } + let written: Vec<&omnigraph_compiler::query::ast::Ordering> = self + .order + .iter() + .filter(|key| !matcher.constant(&key.expr)) + .collect(); + if score.is_none() && self.order.is_empty() { + let sorted = top + .iter() + .any(|id| matches!(plan.node(*id), Some(PhysicalNode::Sort { .. }))); + if sorted { + return Err(ValidationError::violated( + "order", + "the plan sorts rows the query does not order", + )); + } + return Ok(()); + } + loop { + match plan.node(id) { + Some(PhysicalNode::Sort { .. }) => break, + Some(PhysicalNode::Projection { input, .. }) => id = *input, + _ => { + return Err(ValidationError::violated( + "order", + "no sort orders the rows the query's root returns", + )); + } + } + } + let Some(PhysicalNode::Sort { + order_by, + fetch, + tiebreak, + .. + }) = plan.node(id) + else { + unreachable!("the loop stops at a sort"); + }; + let expected_fetch = self.limit.and_then(|limit| usize::try_from(limit).ok()); + if *fetch != expected_fetch { + return Err(ValidationError::violated( + "row cut", + format!( + "the final sort keeps {fetch:?} rows; the query's `limit` is {expected_fetch:?}" + ), + )); + } + let mut keys = order_by.iter(); + if let Some(retrieval) = score { + let (column, descending) = retrieval.kind.score(); + let leads = keys.next().is_some_and(|key| { + key.descending == descending + && matches!(&key.expr, IRExpr::PropAccess { variable, property } + if *variable == retrieval.binding && property == column) + }); + if !leads { + return Err(ValidationError::violated( + "order", + format!( + "the final sort does not lead with the {:?} score of `${}`", + retrieval.kind, retrieval.binding + ), + )); + } + } + let rest: Vec<&IROrdering> = keys.filter(|key| !constant_ir(&key.expr)).collect(); + if rest.len() != written.len() { + return Err(ValidationError::violated( + "order", + format!( + "the final sort has {} written keys; the query writes {} that are not constant", + rest.len(), + written.len() + ), + )); + } + let returns = projection_of(plan, top) + .map(Vec::as_slice) + .unwrap_or_default(); + for (key, planned) in written.iter().zip(&rest) { + if key.descending != planned.descending + || !self.same_key(&key.expr, planned, returns, matcher, budget)? + { + return Err(ValidationError::violated( + "order", + format!( + "the final sort orders by `{}`; the query writes `{}`", + planned.expr, key.expr + ), + )); + } + } + self.check_tiebreak(order_by, tiebreak, returns, matcher)?; + Ok(()) + } + + /// Whether a planned sort key is the written key: as written, or the + /// `return` item the planner bound it to by its result column. In a + /// grouped query a written meta-field key binds by name to the unaliased + /// return item whose column it names, as the compiler lowers it. + fn same_key( + &self, + written: &Expr, + planned: &IROrdering, + returns: &[IRProjection], + matcher: &Matcher<'_>, + budget: &mut Budget, + ) -> Result { + if let IRExpr::AliasRef(column) = &planned.expr + && !matches!(written, Expr::AliasRef(alias) if alias == column) + { + let Some(index) = returns + .iter() + .position(|projection| result_column(projection).as_deref() == Some(column)) + else { + return Ok(false); + }; + let Some(item) = self.returns.get(index) else { + return Ok(false); + }; + let by_name = self.aggregate + && matches!(written, Expr::PropAccess { .. }) + && item.alias.is_none() + && meta_field_result_key(written).as_deref() == Some(column) + && meta_field_result_key(&item.expr).as_deref() == Some(column); + return Ok(item.expr == *written || by_name); + } + matcher.matches(written, &planned.expr, Position::Plain, budget) + } + + /// A written order key with an alias resolved to the `return` item it + /// names. + fn resolved_key<'e>(&'e self, key: &'e Expr) -> &'e Expr { + match key { + Expr::AliasRef(alias) => self + .returns + .iter() + .find(|projection| projection.alias.as_deref() == Some(alias)) + .map_or(key, |projection| &projection.expr), + other => other, + } + } + + /// The sort's metadata keys complete its comparator as RFC 0047 defines: + /// every written binding's identity (`@type` then `@id` for a selected + /// edge) in binding-name order, ascending, nulls first. Three planner + /// omissions are comparator equivalences and accepted: a key the + /// comparator already holds, every key when the query returns only order + /// keys (rows that tie are indistinguishable), and identities of + /// bindings the query does not name (anonymous traversal endpoints) + /// after every named one. + fn check_tiebreak( + &self, + order_by: &[IROrdering], + tiebreak: &[ColumnRef], + returns: &[IRProjection], + matcher: &Matcher<'_>, + ) -> Result<(), ValidationError> { + if self.aggregate { + if !tiebreak.is_empty() { + return Err(ValidationError::violated( + "order", + "a sort of group rows appends binding identities", + )); + } + return Ok(()); + } + let covered = self.returns.iter().all(|projection| { + self.order + .iter() + .any(|key| *self.resolved_key(&key.expr) == projection.expr) + || match &self.search { + Some(Search::Rank(retrieval)) => projected_retrieval(&projection.expr) + .is_some_and(|projected| { + projected.kind == retrieval.kind + && projected.binding == retrieval.binding + && projected.property == retrieval.property + && projected.query == retrieval.query + }), + _ => false, + } + }); + if covered { + return Ok(()); + } + let keyed = |column: &ColumnRef| { + let physical = match column.property.as_deref() { + Some(IDENTITY_MEMBER) => matcher.physical(SYSTEM_ID), + Some(EDGE_TYPE_MEMBER) => matcher.physical(EDGE_TYPE_MEMBER), + _ => return false, + }; + order_by.iter().any(|key| { + let expr = match &key.expr { + IRExpr::AliasRef(alias) => returns + .iter() + .find(|projection| projection.alias.as_deref() == Some(alias)) + .map_or(&key.expr, |projection| &projection.expr), + expr => expr, + }; + matches!(expr, IRExpr::PropAccess { variable, property } + if *variable == column.binding && *property == physical) + }) + }; + let mut expected = Vec::new(); + for name in self.bindings.keys() { + if self.selected_edges.contains(name) { + expected.push(ColumnRef::property(name, EDGE_TYPE_MEMBER)); + } + expected.push(ColumnRef::property(name, IDENTITY_MEMBER)); + } + expected.retain(|column| !keyed(column)); + let planned: Vec<&ColumnRef> = tiebreak.iter().filter(|column| !keyed(column)).collect(); + let named = planned.len().min(expected.len()); + let (head, tail) = planned.split_at(named); + let head_ok = head.iter().copied().eq(expected.iter()); + let tail_ok = tail + .iter() + .all(|column| !self.bindings.contains_key(&column.binding)); + if !head_ok || named != expected.len() || !tail_ok { + return Err(ValidationError::violated( + "order", + format!( + "the final sort breaks ties by {:?}; the query's comparator requires {:?} first", + tiebreak.iter().map(ToString::to_string).collect::>(), + expected.iter().map(ToString::to_string).collect::>() + ), + )); + } + Ok(()) + } +} + +const SYSTEM_ID: &str = "@id"; + +/// Every node of `top` that projects the query's `return`. +fn projections_of<'p>(plan: &'p PhysicalPlan, top: &[NodeId]) -> Vec<&'p Vec> { + top.iter() + .filter_map(|id| match plan.node(*id) { + Some( + PhysicalNode::Projection { return_exprs, .. } + | PhysicalNode::Aggregate { return_exprs, .. } + | PhysicalNode::MetadataCount { return_exprs, .. }, + ) => Some(return_exprs), + _ => None, + }) + .collect() +} + +/// The one projection of `top`, when there is one. +fn projection_of<'p>(plan: &'p PhysicalPlan, top: &[NodeId]) -> Option<&'p Vec> { + match projections_of(plan, top).as_slice() { + [returns] => Some(*returns), + _ => None, + } +} + +/// Whether a planned key reads no row: ordering by it orders nothing. +fn constant_ir(expr: &IRExpr) -> bool { + match expr { + IRExpr::Literal(_) | IRExpr::Param(_) => true, + IRExpr::Binary { left, right, .. } => constant_ir(left) && constant_ir(right), + IRExpr::Not(inner) | IRExpr::IsNull { expr: inner, .. } => constant_ir(inner), + _ => false, + } +} + +fn exactness(approximate: bool) -> &'static str { + if approximate { "approximate" } else { "exact" } +} + +/// The retrieval a projected rank call names. +fn projected_retrieval(expr: &Expr) -> Option { + super::requirements::retrieval_of(expr) +} + +/// The alias the compiler gives an unaliased meta-field projection: the +/// logical `var.@prop`. +fn meta_field_result_key(expr: &Expr) -> Option { + match expr { + Expr::PropAccess { variable, property } if property.starts_with('@') => { + Some(format!("{variable}.{property}")) + } + Expr::Aggregate { arg, .. } => meta_field_result_key(arg), + _ => None, + } +} diff --git a/crates/omnigraph-planner/src/validate/mod.rs b/crates/omnigraph-planner/src/validate/mod.rs new file mode 100644 index 000000000..7eeb0589c --- /dev/null +++ b/crates/omnigraph-planner/src/validate/mod.rs @@ -0,0 +1,216 @@ +//! Plan acceptance (RFC 0047, "Validation across planning stages"). The one +//! door from a planned query to an executable one: [`accept_query`] plans a +//! read and validates the plan against what the checked declaration +//! requires, and only a successful check constructs an [`AcceptedPlan`], +//! whose fields are private. Ordinary execution, `explain` and the inspected +//! run all call the same planning and validation path; explain renders the +//! accepted plan and never validates more or less than a run does. +//! +//! A fresh plan that fails a check is a planner defect, reported as +//! [`Unrouted::PlannerError`]; it never becomes a language refusal. A check +//! that exhausts a [`ValidationLimits`] budget is a resource outcome +//! ([`Unrouted::ValidationExhausted`]), not evidence that the query or the +//! plan is invalid. + +mod budget; +mod invariants; +mod replay; +mod requirements; + +pub use budget::{Budget, ValidationLimits}; +pub use replay::{ + REPLAY_VERSION, RULES_VERSION, ReplayEnvelope, ReplayQuery, ReplayRefusal, SEMANTICS_VERSION, + accept_replay, catalog_digest, decode_replay, +}; +pub use requirements::{ConstantEvaluator, Requirements}; + +use omnigraph_compiler::CheckedQuery; +use omnigraph_compiler::catalog::Catalog; +use omnigraph_compiler::ir::{ParamMap, QueryIR}; +use serde::{Deserialize, Serialize}; + +use crate::bound::{BoundPlan, ValueTable}; +use crate::gate::Unrouted; +use crate::physical::PhysicalPlan; + +/// How much of the plan's derivation acceptance checked. `exact_subset`: +/// the query is a member of the closed exact fragment and every rewrite from +/// its canonical form to the plan was reconstructed and compared; +/// `invariants_only`: the plan was checked against the query's requirements +/// (search identity, retained predicates, score origin, approximation, +/// order and cut, declared policies), which does not prove row-selection +/// equivalence. Only the validator chooses it; nothing upgrades it. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum ValidationScope { + ExactSubset, + InvariantsOnly, +} + +impl ValidationScope { + pub fn as_str(self) -> &'static str { + match self { + Self::ExactSubset => "exact_subset", + Self::InvariantsOnly => "invariants_only", + } + } +} + +/// What a successful explain document reports about acceptance. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize)] +pub struct ValidationSummary { + pub scope: ValidationScope, +} + +/// What a plan check found wrong, or why it stopped. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum ValidationError { + /// The plan breaks a requirement of its query: on a fresh plan a planner + /// defect, on a supplied plan invalid evidence. + Violated { check: &'static str, detail: String }, + /// A configured validation limit ran out before the check finished. + Exhausted { limit: &'static str, value: u64 }, +} + +impl ValidationError { + pub(crate) fn violated(check: &'static str, detail: impl Into) -> Self { + Self::Violated { + check, + detail: detail.into(), + } + } + + /// The route of a fresh plan's failed check. + pub(crate) fn into_unrouted(self) -> Unrouted { + match self { + Self::Violated { check, detail } => Unrouted::PlannerError { + message: format!("plan validation failed ({check}): {detail}"), + }, + Self::Exhausted { limit, value } => Unrouted::ValidationExhausted { limit, value }, + } + } +} + +impl std::fmt::Display for ValidationError { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + Self::Violated { check, detail } => write!(f, "{check}: {detail}"), + Self::Exhausted { limit, value } => { + write!(f, "validation limit `{limit}` ({value}) exhausted") + } + } + } +} + +/// Everything acceptance reads beside the plan source: the checked +/// declaration the requirements come from, the catalog it was checked +/// against, the IR the planner plans (constants of filter positions already +/// folded to their bound values), the bound parameter values (`now()` +/// among them), the evaluator of a folded constant, and the limits. +pub struct AcceptInput<'a> { + pub checked: &'a CheckedQuery, + pub catalog: &'a Catalog, + pub ir: &'a QueryIR, + pub params: &'a ParamMap, + pub constants: &'a dyn ConstantEvaluator, + pub limits: ValidationLimits, +} + +/// The internal record of one acceptance: its scope and the limits it ran +/// under. Inspection and replay read it; ordinary responses carry only the +/// scope. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct Evidence { + scope: ValidationScope, + limits: ValidationLimits, +} + +impl Evidence { + pub fn scope(&self) -> ValidationScope { + self.scope + } + + pub fn limits(&self) -> ValidationLimits { + self.limits + } +} + +/// A physical plan the validator accepted, with its evidence. The fields are +/// private and no deserializer builds one: a plan reaches execution only +/// through [`accept_query`] or the replay acceptance. +#[derive(Debug, Clone)] +pub struct AcceptedPlan { + plan: PhysicalPlan, + evidence: Evidence, +} + +impl AcceptedPlan { + pub fn plan(&self) -> &PhysicalPlan { + &self.plan + } + + pub fn scope(&self) -> ValidationScope { + self.evidence.scope + } + + pub fn evidence(&self) -> &Evidence { + &self.evidence + } + + pub fn summary(&self) -> ValidationSummary { + ValidationSummary { + scope: self.scope(), + } + } + + /// The accepted plan with the values of one run. Binding changes no node + /// of the plan, so it needs no second validation. + pub fn bind(self, values: ValueTable) -> AcceptedBoundPlan { + AcceptedBoundPlan { + bound: BoundPlan { + plan: self.plan, + values, + }, + evidence: self.evidence, + } + } +} + +/// An [`AcceptedPlan`] bound to its values: what execution takes. +#[derive(Debug, Clone)] +pub struct AcceptedBoundPlan { + bound: BoundPlan, + evidence: Evidence, +} + +impl AcceptedBoundPlan { + pub fn bound(&self) -> &BoundPlan { + &self.bound + } + + pub fn evidence(&self) -> &Evidence { + &self.evidence + } + + pub fn into_parts(self) -> (BoundPlan, Evidence) { + (self.bound, self.evidence) + } +} + +/// Validate `plan`, built for `input`, and wrap it. Every check reads the +/// requirements the checked declaration states and the plan alone. +pub(crate) fn accept( + plan: PhysicalPlan, + input: &AcceptInput<'_>, +) -> Result { + let mut budget = Budget::new(input.limits); + let requirements = Requirements::derive(input, &mut budget)?; + requirements.check(&plan, input, &mut budget)?; + Ok(AcceptedPlan { + plan, + evidence: Evidence { + scope: ValidationScope::InvariantsOnly, + limits: input.limits, + }, + }) +} diff --git a/crates/omnigraph-planner/src/validate/replay.rs b/crates/omnigraph-planner/src/validate/replay.rs new file mode 100644 index 000000000..2c9bc259a --- /dev/null +++ b/crates/omnigraph-planner/src/validate/replay.rs @@ -0,0 +1,216 @@ +//! Replay through acceptance (RFC 0047, "Execution and replay use the same +//! acceptance path"). A saved plan travels in a [`ReplayEnvelope`] with the +//! inputs of its query and the scope it was accepted under; replay decodes +//! it within the evidence byte limit, refuses unsupported versions before +//! decoding the plan, and accepts it again against requirements derived +//! afresh from the recompiled query. Deserialization never builds an +//! accepted plan, and a serialized type context or binding cannot validate +//! itself. + +use omnigraph_compiler::CatalogIdentity; +use omnigraph_compiler::catalog::Catalog; +use serde::{Deserialize, Serialize}; + +use super::budget::Budget; +use super::requirements::Requirements; +use super::{ + AcceptInput, AcceptedBoundPlan, Evidence, ValidationError, ValidationLimits, ValidationScope, +}; +use crate::bound::BoundPlan; + +/// The replay envelope's format. A change to its fields or their meaning +/// bumps it; an old envelope is then refused as replan-required, never +/// reinterpreted. +pub const REPLAY_VERSION: u32 = 1; + +/// The catalogue of checks and rewrite rules acceptance applies. A rule id +/// keeps its meaning within one version. +pub const RULES_VERSION: u32 = 1; + +/// The compiler and operator semantics a plan was built under. +pub const SEMANTICS_VERSION: u32 = 1; + +/// The query a saved plan came from: its source and declaration name. Its +/// parameter values are the plan's own bound values. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct ReplayQuery { + pub source: String, + pub name: String, +} + +/// A saved plan with everything replay needs to accept it again. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +pub struct ReplayEnvelope { + pub replay_version: u32, + pub rules_version: u32, + pub semantics_version: u32, + pub query: ReplayQuery, + /// The scope the plan was accepted under; replay must reach it again. + pub scope: ValidationScope, + /// The digest of the accepted schema the plan was accepted under + /// (`None` for a catalog built from source alone). Replay against a + /// different schema is refused as incompatible facts. + pub catalog: Option, + pub plan: BoundPlan, +} + +/// The versions of an envelope, read before anything else is decoded. +#[derive(Deserialize)] +struct Header { + replay_version: u32, + rules_version: u32, + semantics_version: u32, +} + +impl ReplayEnvelope { + /// The envelope of an accepted run: its query, its bound plan, the scope + /// its acceptance reached and the schema it was accepted under. + pub fn new( + source: &str, + name: &str, + plan: BoundPlan, + evidence: &Evidence, + catalog: Option, + ) -> Self { + Self { + replay_version: REPLAY_VERSION, + rules_version: RULES_VERSION, + semantics_version: SEMANTICS_VERSION, + query: ReplayQuery { + source: source.to_string(), + name: name.to_string(), + }, + scope: evidence.scope(), + catalog, + plan, + } + } + + pub fn to_bytes(&self) -> Vec { + serde_json::to_vec(self).expect("a replay envelope serializes") + } +} + +/// Why replay refused a saved plan. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum ReplayRefusal { + /// The envelope is absent, obsolete or of an unsupported version: the + /// caller resupplies the query for fresh planning. + ReplanRequired { reason: String }, + /// A fact the plan was accepted under differs on the replay target: + /// replan against the current view. + IncompatibleFacts { prerequisite: String }, + /// The envelope is malformed, or its plan fails a check against the + /// query's requirements. + InvalidEvidence { reason: String }, + /// A configured validation limit ran out. + Exhausted { limit: &'static str, value: u64 }, +} + +impl std::fmt::Display for ReplayRefusal { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + Self::ReplanRequired { reason } => write!(f, "replan required: {reason}"), + Self::IncompatibleFacts { prerequisite } => { + write!(f, "incompatible facts: {prerequisite}") + } + Self::InvalidEvidence { reason } => write!(f, "invalid replay evidence: {reason}"), + Self::Exhausted { limit, value } => { + write!(f, "validation limit `{limit}` ({value}) exhausted") + } + } + } +} + +impl From for ReplayRefusal { + fn from(error: ValidationError) -> Self { + match error { + ValidationError::Exhausted { limit, value } => Self::Exhausted { limit, value }, + violated @ ValidationError::Violated { .. } => Self::InvalidEvidence { + reason: violated.to_string(), + }, + } + } +} + +/// Decode `bytes` as a replay envelope: its length within the evidence byte +/// limit first, then its versions, then the rest. +pub fn decode_replay( + bytes: &[u8], + limits: ValidationLimits, +) -> Result { + Budget::new(limits).evidence_bytes(u64::try_from(bytes.len()).unwrap_or(u64::MAX))?; + let header: Header = + serde_json::from_slice(bytes).map_err(|error| ReplayRefusal::ReplanRequired { + reason: format!("the bytes carry no replay envelope header: {error}"), + })?; + for (field, found, supported) in [ + ("replay_version", header.replay_version, REPLAY_VERSION), + ("rules_version", header.rules_version, RULES_VERSION), + ( + "semantics_version", + header.semantics_version, + SEMANTICS_VERSION, + ), + ] { + if found != supported { + return Err(ReplayRefusal::ReplanRequired { + reason: format!( + "{field} {found} is not the supported {supported}; resubmit the query" + ), + }); + } + } + serde_json::from_slice(bytes).map_err(|error| ReplayRefusal::InvalidEvidence { + reason: format!("the replay envelope does not decode: {error}"), + }) +} + +/// Accept a saved plan again: the requirements come from `input`, the query +/// recompiled against the replay target's catalog and bound to the plan's +/// own parameter values, and the plan must reach the scope it was saved +/// under. +pub fn accept_replay( + envelope: ReplayEnvelope, + input: &AcceptInput<'_>, +) -> Result { + let catalog = catalog_digest(input.catalog); + if catalog != envelope.catalog { + return Err(ReplayRefusal::IncompatibleFacts { + prerequisite: format!( + "the plan was accepted under schema {}; the replay target's schema is {}", + envelope.catalog.as_deref().unwrap_or(""), + catalog.as_deref().unwrap_or("") + ), + }); + } + let mut budget = Budget::new(input.limits); + let requirements = Requirements::derive(input, &mut budget)?; + requirements.check(&envelope.plan.plan, input, &mut budget)?; + let scope = ValidationScope::InvariantsOnly; + if scope != envelope.scope { + return Err(ReplayRefusal::InvalidEvidence { + reason: format!( + "the envelope claims scope {}; replay established {}", + envelope.scope.as_str(), + scope.as_str() + ), + }); + } + Ok(AcceptedBoundPlan { + bound: envelope.plan, + evidence: Evidence { + scope, + limits: input.limits, + }, + }) +} + +/// The digest of the accepted schema a catalog projects, `None` for a +/// catalog built from source alone. +pub fn catalog_digest(catalog: &Catalog) -> Option { + match &catalog.identity { + CatalogIdentity::Bound(schema) => omnigraph_compiler::schema_ir_hash(schema).ok(), + CatalogIdentity::SourceUnbound => None, + } +} diff --git a/crates/omnigraph-planner/src/validate/requirements.rs b/crates/omnigraph-planner/src/validate/requirements.rs new file mode 100644 index 000000000..76f5c4acb --- /dev/null +++ b/crates/omnigraph-planner/src/validate/requirements.rs @@ -0,0 +1,603 @@ +//! What a checked read declaration requires of every plan for it (RFC 0047, +//! "Requirements come from the checked query"), derived from the declaration +//! as written, its type context and the catalog, never from the IR: the IR +//! has already lost meaning (a projected `bm25()` is a score column, an +//! inline match is a comparison, constants are folded). The requirements are +//! a derived validation input, never an editable description of the query. +//! +//! The [`Matcher`] relates a written expression to the IR expression a plan +//! carries through the compiler's defined lowering patterns: property leaves +//! on their physical columns, parameters as parameters, `in` as `contains` +//! with its operands swapped, `contains` over a String as `StringContains`, +//! a bare search call as `call = true`, a projected rank call as its score +//! column, and a constant subtree as the literal it evaluates to under the +//! bound parameters. It checks a lowering; it does not lower. + +use std::collections::{BTreeMap, HashSet}; + +use omnigraph_compiler::catalog::Catalog; +use omnigraph_compiler::ir::{IRExpr, IRProjection}; +use omnigraph_compiler::query::ast::{ + AggFunc, BinaryOp, Clause, CompOp, DISTANCE_COLUMN, Expr, Literal, NOW_PARAM_NAME, Ordering, + Projection, SCORE_COLUMN, +}; +use omnigraph_compiler::query::typecheck::BoundVariable; +use omnigraph_compiler::traversal::{EDGE_TYPE_COLUMN, EDGE_TYPE_META}; +use omnigraph_compiler::{SYSTEM_COLUMNS_META, SystemColumns}; + +use super::budget::Budget; +use super::{AcceptInput, ValidationError}; +use crate::physical::RankKind; + +/// The value of a constant expression (literals, bound parameters, `now()`, +/// and comparisons, `and`, `or`, `not` and null tests over them) under the +/// bound parameters, or `None` when it has none. The engine implements it +/// with the evaluator its constant folding uses, so a folded literal is +/// checked against the same rules that produced it. +pub trait ConstantEvaluator { + fn evaluate(&self, expr: &IRExpr) -> Option; +} + +/// One retrieval a search order names: `nearest($v.p, q)` or `bm25($v.p, q)`. +#[derive(Debug, Clone)] +pub(crate) struct Retrieval { + pub kind: RankKind, + pub binding: String, + pub property: String, + pub query: Expr, +} + +/// The retrieval a rank call names, `None` for any other expression. +pub(crate) fn retrieval_of(expr: &Expr) -> Option { + Retrieval::of(expr) +} + +impl Retrieval { + fn of(expr: &Expr) -> Option { + match expr { + Expr::Nearest { + variable, + property, + query, + } => Some(Self { + kind: RankKind::Nearest, + binding: variable.clone(), + property: property.clone(), + query: query.as_ref().clone(), + }), + Expr::Bm25 { field, query } => match field.as_ref() { + Expr::PropAccess { variable, property } => Some(Self { + kind: RankKind::Bm25, + binding: variable.clone(), + property: property.clone(), + query: query.as_ref().clone(), + }), + _ => None, + }, + _ => None, + } + } + + /// The approximation the retrieval's contract declares: `nearest` + /// membership is approximate even when a run scores flat; `bm25` is exact. + pub fn approximate(&self) -> bool { + self.kind == RankKind::Nearest + } +} + +/// The search a leading `order` key names. +#[derive(Debug, Clone)] +pub(crate) enum Search { + Rank(Retrieval), + Fuse { + arms: [Retrieval; 2], + k: Option, + }, +} + +/// A correlated block of the top-level scope: the aggregate it compares. +#[derive(Debug, Clone)] +pub(crate) struct Block { + pub func: AggFunc, + pub op: CompOp, + pub right: Expr, +} + +/// What every plan for one checked declaration must preserve. +#[derive(Debug, Clone)] +pub struct Requirements { + /// The named bindings of the top-level scope and their types. + pub(crate) bindings: BTreeMap, + /// Edge bindings of a traversal over more than one edge type, whose + /// identity is the pair (`@type`, `@id`). + pub(crate) selected_edges: HashSet, + /// Traversal clauses of the top-level scope. + pub(crate) traversals: usize, + /// Correlated blocks of the top-level scope, in written order. + pub(crate) blocks: Vec, + pub(crate) search: Option, + /// Every eligibility clause of the top-level scope as written, an + /// inline binding match spelled as its comparison; [`Matcher::conjuncts`] + /// splits a clause the way its lowering does. + pub(crate) eligibility: Vec, + pub(crate) returns: Vec, + /// The `order` keys after a leading search key. + pub(crate) order: Vec, + pub(crate) limit: Option, + pub(crate) aggregate: bool, +} + +impl Requirements { + pub(crate) fn derive( + input: &AcceptInput<'_>, + budget: &mut Budget, + ) -> Result { + let decl = input.checked.decl(); + let types = input.checked.types(); + let mut eligibility = Vec::new(); + let mut traversals = 0; + let mut blocks = Vec::new(); + let mut selected_edges = HashSet::new(); + for clause in &decl.match_clause { + budget.visit(1)?; + match clause { + Clause::Binding(binding) => { + let node_type = input + .catalog + .node_types + .get(&binding.type_name) + .ok_or_else(|| { + ValidationError::violated( + "binding identity", + format!( + "`${}` binds `{}`, which the catalog does not hold", + binding.variable, binding.type_name + ), + ) + })?; + for matched in &binding.prop_matches { + let list = node_type + .properties + .get(&matched.prop_name) + .is_some_and(|property| property.list); + eligibility.push(Expr::comparison( + Expr::PropAccess { + variable: binding.variable.clone(), + property: matched.prop_name.clone(), + }, + if list { CompOp::Contains } else { CompOp::Eq }, + matched.value.clone(), + )); + } + } + Clause::Traversal(_) => {} + Clause::Filter(filter) => { + eligibility.push(filter.clone().with_search_predicates_spelled()); + } + Clause::Subquery(block) => blocks.push(Block { + func: block.func, + op: block.op, + right: block.right.clone(), + }), + } + } + for traversal in &types.traversals { + budget.visit(1)?; + traversals += 1; + if let Some(edge) = traversal + .edge_binding + .as_deref() + .filter(|edge| *edge != "_") + && traversal.edges.named().is_none() + { + selected_edges.insert(edge.to_string()); + } + } + let mut order = decl.order_clause.clone(); + let search = match order.first().map(|key| &key.expr) { + Some(Expr::Rrf { + primary, + secondary, + k, + }) => match (Retrieval::of(primary), Retrieval::of(secondary)) { + (Some(primary), Some(secondary)) => Some(Search::Fuse { + arms: [primary, secondary], + k: k.as_deref().cloned(), + }), + _ => None, + }, + Some(expr) => Retrieval::of(expr).map(Search::Rank), + None => None, + }; + if search.is_some() { + order.remove(0); + } + Ok(Self { + bindings: types + .bindings + .iter() + .map(|(name, bound)| (name.clone(), bound.clone())) + .collect(), + selected_edges, + traversals, + blocks, + search, + eligibility, + returns: decl.return_clause.clone(), + order, + limit: decl.limit, + aggregate: decl + .return_clause + .iter() + .any(|projection| matches!(projection.expr, Expr::Aggregate { .. })), + }) + } + + /// Every retrieval the query names, in written order. + pub(crate) fn retrievals(&self) -> Vec<&Retrieval> { + match &self.search { + None => Vec::new(), + Some(Search::Rank(retrieval)) => vec![retrieval], + Some(Search::Fuse { arms, .. }) => arms.iter().collect(), + } + } +} + +/// Where an expression sits, which decides the lowering pattern it follows. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) enum Position { + /// A `match` conjunct: constant subtrees are folded to their value. + Filter, + /// A `return` item: a rank call is its score column. + Return, + /// An `order` key or a call argument: lowered as written. + Plain, +} + +/// Relates written expressions to the IR expressions of a plan. +pub(crate) struct Matcher<'a> { + pub catalog: &'a Catalog, + pub params: HashSet<&'a str>, + pub bindings: &'a BTreeMap, + pub constants: &'a dyn ConstantEvaluator, +} + +impl<'a> Matcher<'a> { + pub(crate) fn new(input: &'a AcceptInput<'a>, requirements: &'a Requirements) -> Self { + Self { + catalog: input.catalog, + params: input + .checked + .decl() + .params + .iter() + .map(|param| param.name.as_str()) + .collect(), + bindings: &requirements.bindings, + constants: input.constants, + } + } + + fn system_columns(&self) -> SystemColumns { + self.catalog.system_columns + } + + /// The physical column a written property leaf reads. + pub(crate) fn physical(&self, property: &str) -> String { + let columns = self.system_columns(); + match property { + EDGE_TYPE_META => EDGE_TYPE_COLUMN.to_string(), + name if name == SYSTEM_COLUMNS_META.id => columns.id.to_string(), + name if name == SYSTEM_COLUMNS_META.src => columns.src.to_string(), + name if name == SYSTEM_COLUMNS_META.dst => columns.dst.to_string(), + other => other.to_string(), + } + } + + /// The one edge type a single-type edge binding's `@type` reads, which + /// the lowering writes as a String literal. + fn single_edge_type(&self, variable: &str, property: &str) -> Option<&str> { + if property != EDGE_TYPE_META { + return None; + } + match self.bindings.get(variable) { + Some(BoundVariable::Edge { type_names }) => match type_names.as_slice() { + [name] => Some(name), + _ => None, + }, + _ => None, + } + } + + /// Whether `expr` reads only literals, parameters, the clock and the + /// `@type` of a single-type edge binding. + pub(crate) fn constant(&self, expr: &Expr) -> bool { + match expr { + Expr::Literal(_) | Expr::Now => true, + Expr::PropAccess { variable, property } => { + self.single_edge_type(variable, property).is_some() + } + Expr::Variable(name) => self.params.contains(name.as_str()), + Expr::Binary { left, right, .. } => self.constant(left) && self.constant(right), + Expr::In { needle, list } => self.constant(needle) && self.constant(list), + Expr::Not(inner) | Expr::IsNull { expr: inner, .. } => self.constant(inner), + _ => false, + } + } + + /// A constant written subtree as the IR expression its value is + /// computed from, for the evaluator. + fn constant_ir(&self, expr: &Expr) -> Option { + Some(match expr { + Expr::Literal(literal) => IRExpr::Literal(literal.clone()), + Expr::PropAccess { variable, property } => IRExpr::Literal(Literal::String( + self.single_edge_type(variable, property)?.to_string(), + )), + Expr::Now => IRExpr::Param(NOW_PARAM_NAME.to_string()), + Expr::Variable(name) if self.params.contains(name.as_str()) => { + IRExpr::Param(name.clone()) + } + Expr::Binary { left, op, right } => IRExpr::Binary { + left: Box::new(self.constant_ir(left)?), + op: *op, + right: Box::new(self.constant_ir(right)?), + }, + Expr::In { needle, list } => IRExpr::Binary { + left: Box::new(self.constant_ir(list)?), + op: BinaryOp::Compare(CompOp::Contains), + right: Box::new(self.constant_ir(needle)?), + }, + Expr::Not(inner) => IRExpr::Not(Box::new(self.constant_ir(inner)?)), + Expr::IsNull { expr, negated } => IRExpr::IsNull { + expr: Box::new(self.constant_ir(expr)?), + negated: *negated, + }, + _ => return None, + }) + } + + /// A written clause split into the conjuncts its lowering tests: the + /// top-level `and` chain, except that a constant `and` is folded into one + /// value before the split. + pub(crate) fn conjuncts<'e>(&self, clause: &'e Expr, out: &mut Vec<&'e Expr>) { + match clause { + Expr::Binary { + left, + op: BinaryOp::And, + right, + } if !self.constant(clause) => { + self.conjuncts(left, out); + self.conjuncts(right, out); + } + other => out.push(other), + } + } + + /// The value of a constant written expression under the bound + /// parameters. + pub(crate) fn value(&self, expr: &Expr) -> Option { + self.constants.evaluate(&self.constant_ir(expr)?) + } + + /// Whether `ir` is the lowering of `written` at `position`. + pub(crate) fn matches( + &self, + written: &Expr, + ir: &IRExpr, + position: Position, + budget: &mut Budget, + ) -> Result { + budget.visit(1)?; + if let IRExpr::Literal(value) = ir + && !matches!(written, Expr::Literal(_)) + && self.constant(written) + { + return Ok(self + .constant_ir(written) + .and_then(|constant| self.constants.evaluate(&constant)) + .is_some_and(|evaluated| evaluated == *value)); + } + if position == Position::Return + && let Some((variable, column)) = written.score_column() + { + return Ok(matches!( + ir, + IRExpr::PropAccess { variable: v, property: p } if v == variable && p == column + )); + } + Ok(match (written, ir) { + (Expr::Now, IRExpr::Param(name)) => name == NOW_PARAM_NAME, + (Expr::Literal(left), IRExpr::Literal(right)) => left == right, + (Expr::Variable(name), IRExpr::Param(param)) => { + self.params.contains(name.as_str()) && name == param + } + (Expr::Variable(name), IRExpr::Variable(variable)) => { + !self.params.contains(name.as_str()) && name == variable + } + (Expr::AliasRef(left), IRExpr::AliasRef(right)) => left == right, + (Expr::PropAccess { variable, property }, IRExpr::Literal(Literal::String(name))) => { + self.single_edge_type(variable, property) == Some(name.as_str()) + } + ( + Expr::PropAccess { variable, property }, + IRExpr::PropAccess { + variable: v, + property: p, + }, + ) => variable == v && self.physical(property) == *p, + ( + Expr::Binary { left, op, right }, + IRExpr::Binary { + left: l, + op: o, + right: r, + }, + ) => { + same_op(*op, *o) + && self.matches(left, l, position, budget)? + && self.matches(right, r, position, budget)? + } + ( + Expr::In { needle, list }, + IRExpr::Binary { + left: l, + op: o, + right: r, + }, + ) => { + same_op(BinaryOp::Compare(CompOp::Contains), *o) + && self.matches(list, l, position, budget)? + && self.matches(needle, r, position, budget)? + } + (Expr::Not(inner), IRExpr::Not(i)) => self.matches(inner, i, position, budget)?, + ( + Expr::IsNull { expr, negated }, + IRExpr::IsNull { + expr: e, + negated: n, + }, + ) => negated == n && self.matches(expr, e, position, budget)?, + (Expr::Aggregate { func, arg }, IRExpr::Aggregate { func: f, arg: a }) => { + func == f && self.matches(arg, a, Position::Plain, budget)? + } + ( + Expr::Nearest { + variable, + property, + query, + }, + IRExpr::Nearest { + variable: v, + property: p, + query: q, + }, + ) => { + variable == v && property == p && self.matches(query, q, Position::Plain, budget)? + } + (Expr::Search { field, query }, IRExpr::Search { field: f, query: q }) + | (Expr::MatchText { field, query }, IRExpr::MatchText { field: f, query: q }) + | (Expr::Bm25 { field, query }, IRExpr::Bm25 { field: f, query: q }) => { + self.matches(field, f, Position::Plain, budget)? + && self.matches(query, q, Position::Plain, budget)? + } + ( + Expr::Fuzzy { + field, + query, + max_edits, + }, + IRExpr::Fuzzy { + field: f, + query: q, + max_edits: m, + }, + ) => { + self.matches(field, f, Position::Plain, budget)? + && self.matches(query, q, Position::Plain, budget)? + && match (max_edits, m) { + (None, None) => true, + (Some(written), Some(ir)) => { + self.matches(written, ir, Position::Plain, budget)? + } + _ => false, + } + } + ( + Expr::Rrf { + primary, + secondary, + k, + }, + IRExpr::Rrf { + primary: p, + secondary: s, + k: kk, + }, + ) => { + self.matches(primary, p, Position::Plain, budget)? + && self.matches(secondary, s, Position::Plain, budget)? + && match (k, kk) { + (None, None) => true, + (Some(written), Some(ir)) => { + self.matches(written, ir, Position::Plain, budget)? + } + _ => false, + } + } + _ => false, + }) + } + + /// Whether `ir` carries the written query argument of a retrieval: as + /// written, or a constant argument as its bound value. + pub(crate) fn argument( + &self, + written: &Expr, + ir: &IRExpr, + budget: &mut Budget, + ) -> Result { + self.matches(written, ir, Position::Plain, budget) + } +} + +/// `op` as the lowering spells it: `contains` over a String left operand +/// becomes `StringContains`; every other operator is kept. +fn same_op(written: BinaryOp, ir: BinaryOp) -> bool { + written == ir + || written == BinaryOp::Compare(CompOp::Contains) + && ir == BinaryOp::Compare(CompOp::StringContains) +} + +/// The column a projected rank expression lowers to, as `(binding, column)`. +pub(crate) fn score_column(kind: RankKind) -> &'static str { + match kind { + RankKind::Nearest => DISTANCE_COLUMN, + RankKind::Bm25 => SCORE_COLUMN, + } +} + +/// The `return` item an order key names by its result column, as the +/// planner binds a non-property key: the alias, else the expression's own +/// column. +pub(crate) fn result_column(projection: &IRProjection) -> Option { + fn column(expr: &IRExpr) -> Option { + match expr { + IRExpr::PropAccess { variable, property } => Some(format!("{variable}.{property}")), + IRExpr::Variable(name) | IRExpr::Param(name) => Some(name.clone()), + IRExpr::Literal(_) => Some("literal".to_string()), + IRExpr::Aggregate { arg, .. } => column(arg), + _ => None, + } + } + projection + .alias + .clone() + .or_else(|| column(&projection.expr)) +} + +/// Every written return item that projects a rank call, at its root or in +/// the Boolean structure around it, with the call. +pub(crate) fn projected_rank_calls(returns: &[Projection]) -> Vec<(usize, &Expr)> { + fn calls<'e>(expr: &'e Expr, out: &mut Vec<&'e Expr>) { + match expr { + Expr::Binary { left, right, .. } => { + calls(left, out); + calls(right, out); + } + Expr::In { needle, list } => { + calls(needle, out); + calls(list, out); + } + Expr::Not(inner) | Expr::IsNull { expr: inner, .. } => calls(inner, out), + call if call.score_column().is_some() => out.push(call), + _ => {} + } + } + let mut out = Vec::new(); + for (index, projection) in returns.iter().enumerate() { + let mut found = Vec::new(); + calls(&projection.expr, &mut found); + out.extend(found.into_iter().map(|call| (index, call))); + } + out +} diff --git a/crates/omnigraph/src/engine/bind.rs b/crates/omnigraph/src/engine/bind.rs index 3ec22d966..b05cdd1e6 100644 --- a/crates/omnigraph/src/engine/bind.rs +++ b/crates/omnigraph/src/engine/bind.rs @@ -1,20 +1,21 @@ //! Values into the plan: the one place the embedding client is used. `bind` -//! fills a `BoundPlan`'s value table from the gathered parameters and, per -//! `nearest` scan, the query vector its `RankedAccess` names, keyed by the -//! scan's node id. It reads the plan and the parameters and nothing else, -//! and refuses a binding that lacks a parameter the plan assumed. +//! fills an accepted plan's value table from the gathered parameters and, +//! per `nearest` scan, the query vector its `RankedAccess` names, keyed by +//! the scan's node id. It reads the plan and the parameters and nothing +//! else, and refuses a binding that lacks a parameter the plan assumed. use std::collections::BTreeMap; -use omnigraph_planner::{BoundPlan, PhysicalNode, PhysicalPlan, RankKind, ValueTable}; +use omnigraph_planner::{AcceptedBoundPlan, AcceptedPlan, PhysicalNode, RankKind, ValueTable}; use super::*; pub(super) async fn bind( - plan: PhysicalPlan, + accepted: AcceptedPlan, source: &QuerySource<'_>, embedding: &EmbeddingResolver<'_>, -) -> Result { +) -> Result { + let plan = accepted.plan(); let params = Arc::clone(source.params.shared()); let unbound: Vec<&str> = plan .assumptions() @@ -57,8 +58,5 @@ pub(super) async fn bind( .await?; vectors.insert(id, vector); } - Ok(BoundPlan { - plan, - values: ValueTable { params, vectors }, - }) + Ok(accepted.bind(ValueTable { params, vectors })) } diff --git a/crates/omnigraph/src/engine/explain.rs b/crates/omnigraph/src/engine/explain.rs index 525672b05..b602f3733 100644 --- a/crates/omnigraph/src/engine/explain.rs +++ b/crates/omnigraph/src/engine/explain.rs @@ -3,10 +3,11 @@ //! trees, then one per pass and per other field). use datafusion::physical_plan::{DisplayFormatType, ExecutionPlan}; -use omnigraph_planner::PhysicalPlan; +use omnigraph_planner::AcceptedPlan; use super::plan_source::explain_query; use super::*; +use crate::runtime_cache::CompiledQuery; #[cfg(doc)] use omnigraph_planner::BoundPlan; @@ -14,13 +15,13 @@ use omnigraph_planner::BoundPlan; /// The planner's document for `ir` under `settings`, the settings the /// caller's query doors run the same source under. pub(crate) async fn explain_document( - ir: &QueryIR, + query: &CompiledQuery, params: &ParamMap, catalog: &Arc, snapshot: &Snapshot, settings: &SessionSettings, ) -> Result { - let source = QuerySource::gather(ir, catalog, snapshot, params, settings).await?; + let source = QuerySource::gather(query, catalog, snapshot, params, settings).await?; Ok(explain_query(&source)?.explain.to_value()) } @@ -65,9 +66,9 @@ enum LoweredTree { /// The lowered DataFusion tree of the query's first pass in pre-order. An /// explain cannot embed query text, so binding such a plan is `Unavailable`; /// other failures propagate. Planning may already have read dataset metadata. -async fn lowered_tree(plan: PhysicalPlan, source: &QuerySource<'_>) -> Result { +async fn lowered_tree(accepted: AcceptedPlan, source: &QuerySource<'_>) -> Result { let embedding = EmbeddingResolver::explain(); - let bound = match bind(plan, source, &embedding).await { + let bound = match bind(accepted, source, &embedding).await { Err(_) if embedding.was_requested() => { return Ok(LoweredTree::Unavailable( "the nearest() query is a string; embedding it needs the embedding client" @@ -81,7 +82,7 @@ async fn lowered_tree(plan: PhysicalPlan, source: &QuerySource<'_>) -> Result String { /// read: parameter names, setting values and the memory limit, never a /// parameter value. pub(crate) async fn explain_rows( - ir: &QueryIR, + query: &CompiledQuery, params: &ParamMap, snapshot: &Snapshot, catalog: &Arc, settings: &SessionSettings, ) -> Result { - let source = QuerySource::gather(ir, catalog, snapshot, params, settings).await?; + let source = QuerySource::gather(query, catalog, snapshot, params, settings).await?; let planned = explain_query(&source)?; - let assumptions = planned.physical.assumptions().clone(); - let lowered = lowered_tree(planned.physical, &source).await?; + let assumptions = planned.accepted.plan().assumptions().clone(); + let lowered = lowered_tree(planned.accepted, &source).await?; let omnigraph_planner::explain::Explain { explain_version, route, @@ -235,6 +236,7 @@ pub(crate) async fn explain_rows( pipelines, statistics, passes, + validation, } = &planned.explain; let mut rows = ExplainRows::default(); if let Some(root) = logical_plan { @@ -272,6 +274,9 @@ pub(crate) async fn explain_rows( if let Some(statistics) = statistics { rows.push_field("statistics", statistics)?; } + if let Some(validation) = validation { + rows.push_field("validation", validation)?; + } rows.push_field("assumptions", &assumptions)?; rows.into_result() } diff --git a/crates/omnigraph/src/engine/mod.rs b/crates/omnigraph/src/engine/mod.rs index 8f23517cf..76f1aabe5 100644 --- a/crates/omnigraph/src/engine/mod.rs +++ b/crates/omnigraph/src/engine/mod.rs @@ -19,14 +19,14 @@ use arrow_schema::{DataType, Field, Schema}; use lance::Dataset; use omnigraph_compiler::SystemColumns; use omnigraph_compiler::catalog::Catalog; -use omnigraph_compiler::ir::{IRExpr, IROp, IROrdering, IRProjection, ParamMap, QueryIR}; +use omnigraph_compiler::ir::{IRExpr, IROp, IROrdering, IRProjection, ParamMap}; use omnigraph_compiler::query::ast::{AggFunc, BinaryOp, CompOp, Literal}; use omnigraph_compiler::result::QueryResult; use omnigraph_compiler::settings::SessionSettings; use omnigraph_compiler::types::Direction; use omnigraph_compiler::types::ScalarType; use omnigraph_planner::{ - BoundPlan, DatasetPin, ExpandMode, ExpandPolicy, NodeId, OverfetchRung, PhysicalNode, + AcceptedBoundPlan, DatasetPin, ExpandMode, ExpandPolicy, NodeId, OverfetchRung, PhysicalNode, PhysicalPlan, Prefilter, RankKind, RankScope, }; @@ -38,6 +38,7 @@ use crate::instrumentation::{ RrfGateFallback, RrfGatePlan, RrfGateVerdict, record_ann_prefilter_verdict, record_rrf_gate_verdict, }; +use crate::runtime_cache::CompiledQuery; mod adapters; mod bind; @@ -71,7 +72,8 @@ use context::QueryContext; pub(crate) use explain::{explain_document, explain_rows}; pub(crate) use graph::{EmbeddingResolver, GraphIndexHandle}; use lower::Lowering; -use plan_source::{ExplainedQuery, QuerySource, explain_query, plan_query}; +use plan_source::{ExplainedQuery, QuerySource, accept_query, explain_query}; +pub(crate) use plan_source::{accept_replay, replay_refused}; pub(crate) use report::{Executed, PlanRun}; use report::{ExecutionReport, ReportRow}; use run::{pass_rows, run_plan}; @@ -102,7 +104,8 @@ pub(crate) fn dataset_pin(entry: &DatasetEntry) -> DatasetPin { /// Refuse a snapshot that is not the one `plan` was built on: the planner /// recorded every dataset it read in the plan's `Assumptions`, by path, -/// branch and version, and a replay reads exactly those or nothing. +/// branch and version, and a replay reads exactly those or nothing. A +/// changed prerequisite is a conflict: replan against the current view. pub(crate) fn plan_pins_snapshot(plan: &PhysicalPlan, snapshot: &Snapshot) -> Result<()> { for (table, planned) in &plan.assumptions().datasets { let pinned = snapshot.dataset(table).map(dataset_pin); @@ -119,7 +122,7 @@ pub(crate) fn plan_pins_snapshot(plan: &PhysicalPlan, snapshot: &Snapshot) -> Re ), None => "no such table".to_string(), }; - return Err(OmniError::manifest_internal(format!( + return Err(OmniError::manifest_conflict(format!( "`{table}` was planned at dataset {}; the snapshot holds {}", spell(planned), spell(&pinned) @@ -273,7 +276,7 @@ impl ResolvedParams { /// Widen nearest candidates when traversal or filtering leaves a full scan's /// answer short of the limit. pub(crate) async fn execute_query( - ir: &QueryIR, + query: &CompiledQuery, params: &ParamMap, snapshot: &Snapshot, graph_index: GraphIndexHandle, @@ -281,9 +284,9 @@ pub(crate) async fn execute_query( embedding: &EmbeddingResolver<'_>, settings: &SessionSettings, ) -> Result { - let source = QuerySource::gather(ir, catalog, snapshot, params, settings).await?; - let physical = plan_query(&source)?; - let bound = bind(physical, &source, embedding).await?; + let source = QuerySource::gather(query, catalog, snapshot, params, settings).await?; + let accepted = accept_query(&source)?; + let bound = bind(accepted, &source, embedding).await?; let context = EngineContext { snapshot, catalog, @@ -298,7 +301,7 @@ pub(crate) async fn execute_query( /// [`execute_query`] as an [`Executed`]: the gate plans once, and that one /// plan is both the plan the run executes and the plan its explain renders. pub(crate) async fn execute_query_inspected( - ir: &QueryIR, + query: &CompiledQuery, params: &ParamMap, snapshot: &Snapshot, graph_index: GraphIndexHandle, @@ -306,9 +309,9 @@ pub(crate) async fn execute_query_inspected( embedding: &EmbeddingResolver<'_>, settings: &SessionSettings, ) -> Result { - let source = QuerySource::gather(ir, catalog, snapshot, params, settings).await?; - let ExplainedQuery { explain, physical } = explain_query(&source)?; - let bound = bind(physical, &source, embedding).await?; + let source = QuerySource::gather(query, catalog, snapshot, params, settings).await?; + let ExplainedQuery { explain, accepted } = explain_query(&source)?; + let bound = bind(accepted, &source, embedding).await?; let context = EngineContext { snapshot, catalog, @@ -317,11 +320,14 @@ pub(crate) async fn execute_query_inspected( let PlanRun { result, plan, + evidence, report, } = Box::pin(execute(bound, &context)).await?; Ok(Executed { result, plan, + evidence, + catalog: omnigraph_planner::catalog_digest(catalog), explain, report, }) @@ -407,7 +413,11 @@ fn validate_traversal_admission(plan: &PhysicalPlan) -> Result) -> Result { +pub(crate) async fn execute( + accepted: AcceptedBoundPlan, + context: &EngineContext<'_>, +) -> Result { + let (bound, evidence) = accepted.into_parts(); let traversal_limit = validate_traversal_admission(&bound.plan)?; omnigraph_planner::optimizer::validate_rank_fuse_row_tiebreaks(&bound.plan) .map_err(|error| OmniError::manifest_internal(error.to_string()))?; @@ -432,6 +442,7 @@ pub(crate) async fn execute(bound: BoundPlan, context: &EngineContext<'_>) -> Re return Ok(PlanRun { result: QueryResult::new(fused.schema(), vec![fused]), plan: bound, + evidence, report: executed, }); } @@ -544,6 +555,7 @@ pub(crate) async fn execute(bound: BoundPlan, context: &EngineContext<'_>) -> Re Ok(PlanRun { result: QueryResult::new(result_batch.schema(), vec![result_batch]), plan: bound, + evidence, report: executed, }) } diff --git a/crates/omnigraph/src/engine/plan_source.rs b/crates/omnigraph/src/engine/plan_source.rs index 2bfab6d0a..063ff49cb 100644 --- a/crates/omnigraph/src/engine/plan_source.rs +++ b/crates/omnigraph/src/engine/plan_source.rs @@ -9,16 +9,17 @@ use arrow_schema::SchemaRef; use lance::dataset::statistics::DatasetStatisticsExt; use lance::datatypes::Field; use lance_file::version::ConcreteFileVersion; +use omnigraph_compiler::CheckedQuery; use omnigraph_compiler::catalog::Catalog; use omnigraph_compiler::ir::{IRExpr, IROp, ParamMap, QueryIR}; use omnigraph_compiler::query::ast::Literal; use omnigraph_compiler::settings::{RrfPlan, SessionSettings, Traversal}; use omnigraph_compiler::types::Direction; use omnigraph_planner::{ - AdjacencyProof, Bounds, DatasetPin, Decision, EXPAND_INDEXED_MAX_FRONTIER_ENV, - EXPAND_INDEXED_MAX_HOPS_ENV, ExpandStatistics, Explain, FragmentStat, GatePolicy, NodeTypeSpec, - Operation, PhysicalPlan, PlanError, PlanSource, PrefilterMode, RouteOverride, SideId, TableRef, - Unrouted, + AcceptInput, AcceptedBoundPlan, AcceptedPlan, AdjacencyProof, Bounds, ConstantEvaluator, + DatasetPin, EXPAND_INDEXED_MAX_FRONTIER_ENV, EXPAND_INDEXED_MAX_HOPS_ENV, ExpandStatistics, + Explain, FragmentStat, GatePolicy, NodeTypeSpec, Operation, PlanError, PlanSource, + PrefilterMode, ReplayEnvelope, ReplayRefusal, SideId, TableRef, Unrouted, ValidationLimits, }; use super::ResolvedParams; @@ -26,6 +27,7 @@ use super::scan::ir_expr_to_df_expr; use super::search::check_param_date_literals; use crate::db::Snapshot; use crate::error::{OmniError, Result}; +use crate::runtime_cache::CompiledQuery; const KEY_WIDTH_BYTES: u64 = 8 + 8 + 32; @@ -92,6 +94,9 @@ pub(crate) struct QuerySource<'a> { /// The compiled query with every constant of a filter position folded to /// its bound value (`engine::constant`); the planner reads no other form. pub ir: QueryIR, + /// The declaration `ir` was lowered from, as the type checker accepted + /// it: acceptance derives the query's requirements from it. + pub checked: CheckedQuery, pub catalog: &'a Arc, pub snapshot: &'a Snapshot, pub params: ResolvedParams, @@ -112,17 +117,18 @@ impl<'a> QuerySource<'a> { /// memory limit, the gate policy and the indexed-path ceilings once, and /// loads the Lance statistics the planner asks for. pub(crate) async fn gather( - ir: &QueryIR, + query: &CompiledQuery, catalog: &'a Arc, snapshot: &'a Snapshot, params: &ParamMap, settings: &'a SessionSettings, ) -> Result> { - let params = resolve_params(ir, params)?; - let ir = super::constant::fold_query_constants(ir, params.shared())?; + let params = resolve_params(&query.ir, params)?; + let ir = super::constant::fold_query_constants(&query.ir, params.shared())?; let table_stats = destination_table_statistics(&ir, snapshot).await?; let mut source = QuerySource { ir, + checked: query.checked.clone(), catalog, snapshot, params, @@ -390,48 +396,113 @@ fn resolve_params(ir: &QueryIR, params: &ParamMap) -> Result { Ok(ResolvedParams(Arc::new(resolved))) } -/// Build the physical plan for execution without explain diagnostics. -pub(crate) fn plan_query(source: &QuerySource<'_>) -> Result { - omnigraph_planner::plan_query(&source.ir, source, &source.bounds()).map_err(|reason| { - match reason { - Unrouted::UnsupportedQuery { diagnostic } => unsupported_query(diagnostic), - reason => no_plan(reason.to_json()), +/// The engine's constant evaluator for acceptance: the rules its own +/// constant folding applied to the IR, under the same bound parameters. +struct BoundConstants<'p>(&'p ParamMap); + +impl ConstantEvaluator for BoundConstants<'_> { + fn evaluate(&self, expr: &IRExpr) -> Option { + super::constant::evaluate_constant(expr, self.0).ok() + } +} + +/// The failure of a read's planning or acceptance: a refusal by design is the +/// caller's error with its diagnostic, an exhausted validation limit is a +/// resource outcome, and anything else is a planner defect. +fn unaccepted(reason: Unrouted) -> OmniError { + match reason { + Unrouted::UnsupportedQuery { diagnostic } => unsupported_query(diagnostic), + Unrouted::ValidationExhausted { limit, value } => OmniError::ResourceLimitExceeded { + resource: format!("plan validation {limit}"), + limit: value, + actual: value.saturating_add(1), + }, + reason => no_plan(reason.to_json()), + } +} + +impl QuerySource<'_> { + fn accept_input<'s>(&'s self, constants: &'s BoundConstants<'s>) -> AcceptInput<'s> { + AcceptInput { + checked: &self.checked, + catalog: self.catalog, + ir: &self.ir, + params: self.params.shared(), + constants, + limits: ValidationLimits::DEFAULT, } - }) + } +} + +/// Plan the read and accept the plan, without explain diagnostics. +pub(crate) fn accept_query(source: &QuerySource<'_>) -> Result { + let constants = BoundConstants(source.params.shared()); + omnigraph_planner::accept_query(&source.accept_input(&constants), source, &source.bounds()) + .map_err(unaccepted) } -/// What the gate built for one compiled query: its explain document and the -/// physical plan the runner executes. +/// Accept a replayed plan: `query` recompiled against the replay target's +/// catalog, its constants folded under the plan's own parameter values, and +/// the plan checked against the requirements derived from it. +pub(crate) fn accept_replay( + query: &CompiledQuery, + envelope: ReplayEnvelope, + catalog: &Arc, +) -> Result { + let params = Arc::clone(&envelope.plan.values.params); + let ir = super::constant::fold_query_constants(&query.ir, ¶ms)?; + let constants = BoundConstants(¶ms); + let input = AcceptInput { + checked: &query.checked, + catalog, + ir: &ir, + params: ¶ms, + constants: &constants, + limits: ValidationLimits::DEFAULT, + }; + omnigraph_planner::accept_replay(envelope, &input).map_err(replay_refused) +} + +/// A refused replay as the caller sees it: replanning is a conflict to +/// resolve by resubmitting the query, invalid evidence is a bad request, an +/// exhausted limit is a resource outcome. +pub(crate) fn replay_refused(refusal: ReplayRefusal) -> OmniError { + match refusal { + ReplayRefusal::ReplanRequired { reason } => { + OmniError::manifest_conflict(format!("the saved plan cannot be replayed: {reason}")) + } + ReplayRefusal::IncompatibleFacts { prerequisite } => OmniError::manifest_conflict(format!( + "the saved plan's facts changed; replan the query: {prerequisite}" + )), + ReplayRefusal::InvalidEvidence { reason } => { + OmniError::manifest(format!("the saved plan is not accepted: {reason}")) + } + ReplayRefusal::Exhausted { limit, value } => OmniError::ResourceLimitExceeded { + resource: format!("plan validation {limit}"), + limit: value, + actual: value.saturating_add(1), + }, + } +} + +/// What one acceptance built for a compiled query: the explain document of +/// the accepted plan and the plan the runner executes. pub(crate) struct ExplainedQuery { pub explain: Explain, - pub physical: PhysicalPlan, + pub accepted: AcceptedPlan, } -/// A read query always gets a plan; a gate answer other than `Engine` is a -/// planner defect, never a fallback. +/// [`accept_query`] with the accepted plan's explain document, rendered from +/// the same planning run; it validates exactly what a run validates. pub(crate) fn explain_query(source: &QuerySource<'_>) -> Result { - let operation = Operation::Query(Box::new(source.ir.clone())); - match omnigraph_planner::route( - &operation, + let constants = BoundConstants(source.params.shared()); + let (accepted, explain) = omnigraph_planner::accept_query_explained( + &source.accept_input(&constants), source, - RouteOverride::Registry, &source.bounds(), - ) { - Decision::Engine { plan, explain, .. } => Ok(ExplainedQuery { - explain, - physical: plan, - }), - Decision::Executor { - reason: Unrouted::UnsupportedQuery { diagnostic }, - .. - } => Err(unsupported_query(diagnostic)), - Decision::Executor { reason, .. } => Err(no_plan(reason.to_json())), - Decision::Routed { entry, .. } => Err(OmniError::manifest_internal(format!( - "the registry routed a GQ query through entry `{}`; a read query runs only \ - the planner's own plan", - entry.name - ))), - } + ) + .map_err(unaccepted)?; + Ok(ExplainedQuery { explain, accepted }) } async fn destination_table_statistics( @@ -489,7 +560,6 @@ fn field_data_bytes(field: &Field, bytes: &HashMap) -> Option { mod tests { use super::*; use arrow_schema::DataType; - use omnigraph_compiler::query::typecheck::typecheck_query; use crate::db::{Omnigraph, ReadTarget}; use crate::engine::context::QueryContext; @@ -531,10 +601,14 @@ query likes_nothing() { query people() { match { $p: Person } return { count($p) as n } } "#; - fn compile(catalog: &Catalog, name: &str) -> QueryIR { + fn compile(catalog: &Catalog, name: &str) -> CompiledQuery { let statement = omnigraph_compiler::find_read_statement(QUERIES, name).unwrap(); - let checked = typecheck_query(catalog, statement.decl()).unwrap(); - omnigraph_compiler::lower_query(catalog, statement.decl(), &checked).unwrap() + let checked = CheckedQuery::check(catalog, statement.decl()).unwrap(); + let ir = omnigraph_compiler::lower_query(catalog, checked.decl(), checked.types()).unwrap(); + CompiledQuery { + ir: Arc::new(ir), + checked, + } } /// Rust and not `.gqt`: the claim is which map the lowering projects a bare @@ -556,17 +630,23 @@ query people() { match { $p: Person } return { count($p) as n } } ("likes_nothing", 2), ("people", 1), ] { - let ir = compile(&catalog, name); - let source = - QuerySource::gather(&ir, &catalog, &view.snapshot, &ParamMap::new(), &settings) - .await - .unwrap(); - let plan = plan_query(&source).unwrap(); + let query = compile(&catalog, name); + let source = QuerySource::gather( + &query, + &catalog, + &view.snapshot, + &ParamMap::new(), + &settings, + ) + .await + .unwrap(); + let accepted = accept_query(&source).unwrap(); + let plan = accepted.plan(); let mut from_ir = HashMap::new(); - collect_node_bindings(&ir.pipeline, &mut from_ir); + collect_node_bindings(&query.ir.pipeline, &mut from_ir); assert_eq!(from_ir.len(), bound, "{name}"); assert_eq!( - ProjectionContext::for_plan(&catalog, &plan).bindings(), + ProjectionContext::for_plan(&catalog, plan).bindings(), &from_ir, "{name}" ); @@ -587,27 +667,35 @@ query people() { match { $p: Person } return { count($p) as n } } .await .unwrap(); let settings = SessionSettings::default(); - let ir = compile(&catalog, "liked"); + let query = compile(&catalog, "liked"); let source = with_query_memory_limit( CAPTURED, - QuerySource::gather(&ir, &catalog, &view.snapshot, &ParamMap::new(), &settings), + QuerySource::gather( + &query, + &catalog, + &view.snapshot, + &ParamMap::new(), + &settings, + ), ) .await .unwrap(); with_query_memory_limit(2 * CAPTURED, async { assert_eq!(source.bounds().query_memory_pool_bytes, CAPTURED); assert_eq!(source.query_memory_pool_bytes(), CAPTURED); - let plan = plan_query(&source).unwrap(); - assert_eq!(plan.assumptions().memory_limit, CAPTURED); - let bound = crate::engine::bind::bind(plan, &source, &EmbeddingResolver::explain()) - .await - .unwrap(); + let accepted = accept_query(&source).unwrap(); + assert_eq!(accepted.plan().assumptions().memory_limit, CAPTURED); + let accepted = + crate::engine::bind::bind(accepted, &source, &EmbeddingResolver::explain()) + .await + .unwrap(); + let bound = accepted.bound(); let context = EngineContext { snapshot: &view.snapshot, catalog: &catalog, graph_index: Arc::new(GraphIndexHandle::none()), }; - let lowering = Lowering::new(&bound, &context); + let lowering = Lowering::new(bound, &context); assert_eq!(lowering.plan.assumptions().memory_limit, CAPTURED); let ctx = QueryContext::new(bound.plan.assumptions().memory_limit).unwrap(); assert_eq!(ctx.memory_limit(), CAPTURED); diff --git a/crates/omnigraph/src/engine/report.rs b/crates/omnigraph/src/engine/report.rs index 9d2914b0c..0b19171df 100644 --- a/crates/omnigraph/src/engine/report.rs +++ b/crates/omnigraph/src/engine/report.rs @@ -2,7 +2,7 @@ //! every live node of the plan, read from that operator's own metrics after //! the tree ran. A rerun appends an [`Attempt`] to every row. -use omnigraph_planner::{BoundPlan, Explain, NodeId}; +use omnigraph_planner::{BoundPlan, Evidence, Explain, NodeId}; use serde::{Deserialize, Serialize}; use super::explain::ExplainRows; @@ -128,6 +128,8 @@ impl ExecutionReport { pub struct PlanRun { pub result: omnigraph_compiler::result::QueryResult, pub plan: BoundPlan, + /// What acceptance checked of `plan`. + pub evidence: Evidence, pub report: ExecutionReport, } @@ -136,11 +138,28 @@ pub struct PlanRun { pub struct Executed { pub result: omnigraph_compiler::result::QueryResult, pub plan: BoundPlan, + /// What acceptance checked of `plan`. + pub evidence: Evidence, + /// The digest of the schema `plan` was accepted under. + pub catalog: Option, pub explain: Explain, pub report: ExecutionReport, } impl Executed { + /// The replay envelope of this run: `plan` with the source and name of + /// the query it ran, its scope and its schema, serialized. + pub fn replay_envelope(&self, source: &str, name: &str) -> Vec { + omnigraph_planner::ReplayEnvelope::new( + source, + name, + self.plan.clone(), + &self.evidence, + self.catalog.clone(), + ) + .to_bytes() + } + /// The report as actuals rows in the explain row schema: `tree` /// `profile`, no `depth`, `node` the plan node's kind, `detail` the row's /// own fields (`id`, `operator`, `status`, `attempts`). The third public diff --git a/crates/omnigraph/src/exec/query_doors.rs b/crates/omnigraph/src/exec/query_doors.rs index c87eb6fbd..9700a8d19 100644 --- a/crates/omnigraph/src/exec/query_doors.rs +++ b/crates/omnigraph/src/exec/query_doors.rs @@ -7,15 +7,14 @@ use std::sync::Arc; use omnigraph_compiler::catalog::Catalog; use omnigraph_compiler::error::CompilerError; use omnigraph_compiler::ir::{IROp, ParamMap, QueryIR}; -use omnigraph_compiler::lower_query; -use omnigraph_compiler::query::typecheck::typecheck_query; use omnigraph_compiler::result::QueryResult; use omnigraph_compiler::settings::SessionSettings; +use omnigraph_compiler::{CheckedQuery, lower_query}; use crate::db::{Omnigraph, ReadTarget, Snapshot}; use crate::engine; use crate::error::{OmniError, Result}; -use crate::runtime_cache::CompiledRead; +use crate::runtime_cache::{CompiledQuery, CompiledRead}; use crate::session::Session; /// Where a route builds its CSR graph index when the query traverses: @@ -74,15 +73,15 @@ impl Session { compiled.ir().has_wildcard_traversal(), )?; let head = resolved.graph_commit_id.clone(); - if let CompiledRead::Explain(ir) = &compiled { - let rows = - engine::explain_rows(ir, params, &resolved.snapshot, &catalog, &settings).await?; + if let CompiledRead::Explain(query) = &compiled { + let rows = engine::explain_rows(query, params, &resolved.snapshot, &catalog, &settings) + .await?; return Ok((rows, head)); } let result = self .execute_on_route( &settings, - compiled.ir(), + compiled.query(), params, &resolved.snapshot, IndexSource::Cached(&resolved), @@ -108,12 +107,12 @@ impl Session { let compiled = self.compile_named_query(&catalog, query_source, query_name)?; admit_wildcard_target(true, compiled.ir().has_wildcard_traversal())?; - if let CompiledRead::Explain(ir) = &compiled { - return engine::explain_rows(ir, params, &snapshot, &catalog, &settings).await; + if let CompiledRead::Explain(query) = &compiled { + return engine::explain_rows(query, params, &snapshot, &catalog, &settings).await; } self.execute_on_route( &settings, - compiled.ir(), + compiled.query(), params, &snapshot, IndexSource::Direct, @@ -150,7 +149,7 @@ impl Session { compiled.ir().has_wildcard_traversal(), )?; engine::explain_document( - compiled.ir(), + compiled.query(), params, &catalog, &resolved.snapshot, @@ -176,13 +175,14 @@ impl Session { ) -> Result { let settings = self.effective(query_source)?; let (resolved, catalog) = self.capture_read_view(target).await?; - let CompiledRead::Query(ir) = + let CompiledRead::Query(query) = self.compile_named_query(&catalog, query_source, query_name)? else { return Err(OmniError::manifest( "the inspection door runs no `explain` statement", )); }; + let ir = &query.ir; admit_wildcard_target( matches!(&resolved.requested, ReadTarget::Snapshot(_)), ir.has_wildcard_traversal(), @@ -202,7 +202,7 @@ impl Session { engine::GraphIndexHandle::none() }; engine::execute_query_inspected( - &ir, + &query, params, &resolved.snapshot, graph_index, @@ -213,38 +213,60 @@ impl Session { .await } - /// The replay door of the plan-replay tests: executes `bound`, a plan the - /// inspection door returned and the test serialized and read back, with - /// nothing else about its query: no session setting, no `QueryIR`. It - /// builds the engine context `query_inspected` builds (the snapshot and - /// catalog of `target`, a graph index scoped to the plan's `Expand`s), - /// refuses a snapshot whose dataset versions are not the ones the plan's - /// scans, counts and traversals pinned, and calls the same `execute`. + /// The replay door of the plan-replay tests: accepts and executes the + /// plan a replay envelope carries (an inspected run's bound plan with + /// its query's source, name and accepted scope, serialized and read + /// back), with no session setting. The envelope is decoded within the + /// evidence byte limit, its versions checked first; the query is + /// recompiled against `target`'s catalog and bound to the plan's own + /// parameter values, and the plan is accepted again against the + /// requirements derived from it. It builds the engine context + /// `query_inspected` builds (the snapshot and catalog of `target`, a + /// graph index scoped to the plan's `Expand`s), refuses a snapshot whose + /// dataset versions are not the ones the plan's scans, counts and + /// traversals pinned, and calls the same `execute`. /// /// # Errors /// - /// The errors of [`Self::query`] on the target, a snapshot the plan did - /// not pin, and every run-time error of the plan. + /// The errors of [`Self::query`] on the target; a replan-required + /// conflict for an unsupported envelope version; a conflict for a + /// snapshot the plan did not pin; a bad request for a malformed envelope + /// or a plan that fails acceptance; every run-time error of the plan. #[doc(hidden)] pub async fn replay_bound_plan( &self, target: impl Into, - bound: omnigraph_planner::BoundPlan, + envelope: &[u8], ) -> Result { + let envelope = omnigraph_planner::decode_replay( + envelope, + omnigraph_planner::ValidationLimits::DEFAULT, + ) + .map_err(engine::replay_refused)?; let (resolved, catalog) = self.capture_read_view(target).await?; - let has_wildcard = bound.plan.assumptions().has_wildcard_traversal || bound.plan.live().any(|(_, node)| { + let plan = &envelope.plan.plan; + let has_wildcard = plan.assumptions().has_wildcard_traversal || plan.live().any(|(_, node)| { matches!(node, omnigraph_planner::PhysicalNode::Expand { edges, .. } if edges.is_wildcard()) }); admit_wildcard_target( matches!(&resolved.requested, ReadTarget::Snapshot(_)), has_wildcard, )?; - engine::plan_pins_snapshot(&bound.plan, &resolved.snapshot)?; - let graph_index = if engine::plan_traverses(&bound.plan) { + engine::plan_pins_snapshot(plan, &resolved.snapshot)?; + let CompiledRead::Query(query) = + self.compile_named_query(&catalog, &envelope.query.source, &envelope.query.name)? + else { + return Err(OmniError::manifest( + "a replay envelope names an `explain` statement, which runs nothing", + )); + }; + let accepted = engine::accept_replay(&query, envelope, &catalog)?; + let plan = &accepted.bound().plan; + let graph_index = if engine::plan_traverses(plan) { engine::GraphIndexHandle::cached( Arc::clone(&**self), resolved.clone(), - engine::plan_edge_types(&bound.plan, &catalog), + engine::plan_edge_types(plan, &catalog), catalog.system_columns, ) } else { @@ -255,7 +277,7 @@ impl Session { catalog: &catalog, graph_index: Arc::new(graph_index), }; - engine::execute(bound, &context).await + engine::execute(accepted, &context).await } /// One compiled query through the engine, or through the installed @@ -264,12 +286,13 @@ impl Session { async fn execute_on_route( &self, settings: &SessionSettings, - ir: &QueryIR, + query: &CompiledQuery, params: &ParamMap, snapshot: &Snapshot, index_source: IndexSource<'_>, catalog: &Arc, ) -> Result { + let ir: &QueryIR = &query.ir; #[cfg(feature = "test-util")] if let Some(executor) = self.read_executor() { let request = omnigraph_catalog::read_executor::ReadRequest { @@ -300,7 +323,7 @@ impl Session { ), }; engine::execute_query( - ir, + query, params, snapshot, graph_index, @@ -329,12 +352,13 @@ impl Omnigraph { } let statement = omnigraph_compiler::find_read_statement(query_source, query_name) .map_err(super::query_lookup_error)?; - let type_ctx = typecheck_query(catalog, statement.decl())?; - let ir = Arc::new(lower_query(catalog, statement.decl(), &type_ctx)?); + let checked = CheckedQuery::check(catalog, statement.decl())?; + let ir = Arc::new(lower_query(catalog, checked.decl(), checked.types())?); + let query = CompiledQuery { ir, checked }; let compiled = if statement.is_explain() { - CompiledRead::Explain(ir) + CompiledRead::Explain(query) } else { - CompiledRead::Query(ir) + CompiledRead::Query(query) }; crate::instrumentation::record_query_compile(); cache.insert(catalog, key, compiled.clone()); diff --git a/crates/omnigraph/src/runtime_cache.rs b/crates/omnigraph/src/runtime_cache.rs index 50f497844..489806260 100644 --- a/crates/omnigraph/src/runtime_cache.rs +++ b/crates/omnigraph/src/runtime_cache.rs @@ -3,6 +3,7 @@ use std::hash::Hash; use std::sync::{Arc, Weak}; use lance::session::Session; +use omnigraph_compiler::CheckedQuery; use omnigraph_compiler::SystemColumns; use omnigraph_compiler::catalog::Catalog; use omnigraph_compiler::ir::QueryIR; @@ -334,20 +335,33 @@ pub(crate) struct CompiledQueryKey { name: String, } +/// A compiled read declaration: its IR, and the checked declaration the IR +/// was lowered from, which plan acceptance derives the query's requirements +/// from. +#[derive(Clone)] +pub(crate) struct CompiledQuery { + pub(crate) ir: Arc, + pub(crate) checked: CheckedQuery, +} + /// A compiled read statement: a declaration the door runs for rows, or an /// `explain` statement over one, which the door answers with the plan. #[derive(Clone)] pub(crate) enum CompiledRead { - Query(Arc), - Explain(Arc), + Query(CompiledQuery), + Explain(CompiledQuery), } impl CompiledRead { - pub(crate) fn ir(&self) -> &Arc { + pub(crate) fn query(&self) -> &CompiledQuery { match self { - Self::Query(ir) | Self::Explain(ir) => ir, + Self::Query(query) | Self::Explain(query) => query, } } + + pub(crate) fn ir(&self) -> &Arc { + &self.query().ir + } } struct CompiledQueryEntry { @@ -662,19 +676,22 @@ edge Likes: Person -> Person {} let (_, built) = db.capture_current_read_view().await.unwrap(); let compile = |catalog: &Catalog, source: &str, name: &str| { let decl = omnigraph_compiler::find_named_query(source, name).unwrap(); - let ctx = - omnigraph_compiler::query::typecheck::typecheck_query(catalog, &decl).unwrap(); - CompiledRead::Query(Arc::new( - omnigraph_compiler::lower_query(catalog, &decl, &ctx).unwrap(), - )) + let checked = CheckedQuery::check(catalog, &decl).unwrap(); + let ir = + omnigraph_compiler::lower_query(catalog, checked.decl(), checked.types()).unwrap(); + CompiledRead::Query(CompiledQuery { + ir: Arc::new(ir), + checked, + }) }; let cache = CompiledQueryCache::default(); let source = "query q() { match { $p: Person } return { $p.name } }"; let key = CompiledQueryCache::key_for(source, "q"); let first = Arc::new((*built).clone()); - let ir = Arc::clone(compile(&first, source, "q").ir()); - cache.insert(&first, key.clone(), CompiledRead::Query(Arc::clone(&ir))); + let compiled = compile(&first, source, "q"); + let ir = Arc::clone(compiled.ir()); + cache.insert(&first, key.clone(), compiled); assert!(Arc::ptr_eq(cache.get(&first, &key).unwrap().ir(), &ir)); let equal = Arc::new((*built).clone()); diff --git a/crates/omnigraph/tests/engine_v2.rs b/crates/omnigraph/tests/engine_v2.rs index 9eafd2f23..0a15da503 100644 --- a/crates/omnigraph/tests/engine_v2.rs +++ b/crates/omnigraph/tests/engine_v2.rs @@ -896,3 +896,150 @@ query total($q: Vector(4)) { ); } } + +/// Ordinary execution, the explain document, the `explain` statement and the +/// inspected run accept one plan for one query: the same physical plan and +/// validation scope, or the same refusal with the same diagnostic. Rust and +/// not `.gqt`: a case reaches one door per step, and the claim is that the +/// doors agree. +#[tokio::test] +async fn every_read_door_shares_one_acceptance() { + use arrow_array::StringArray; + use omnigraph::db::ReadTarget; + + const SCHEMA: &str = r#" +node Person { + name: String @key + age: I64 +} +node Doc { + slug: String @key + title: String @index +} +edge Likes: Person -> Doc +"#; + const ADULTS: &str = "query adults($min: I64) { + match { $p: Person $p.age >= $min } + return { $p.name } + order { $p.age desc } + limit 2 +}"; + const RANKED: &str = "query ranked($q: String) { + match { $d: Doc } + return { $d.slug, bm25($d.title, $q) as score } + order { bm25($d.title, $q) } + limit 2 +}"; + const LIKED: &str = "query liked() { + match { $p: Person $p likes $d } + return { $p.name, count($d) as n } + order { $p.name } +}"; + const RANKED_DESTINATION: &str = "query ranked_destination($q: String) { + match { $p: Person $p likes $d } + return { $d.slug } + order { bm25($d.title, $q) } +}"; + let dir = tempfile::tempdir().unwrap(); + let db = session( + Omnigraph::init(dir.path().to_str().unwrap(), SCHEMA) + .await + .unwrap(), + ); + db.load_jsonl( + r#"{"type":"Person","data":{"name":"ann","age":40}} +{"type":"Person","data":{"name":"bob","age":20}} +{"type":"Doc","data":{"slug":"d1","title":"graph engines"}} +{"type":"Doc","data":{"slug":"d2","title":"graph graph search"}} +{"edge":"Likes","from":"ann","to":"d1"} +{"edge":"Likes","from":"bob","to":"d2"}"#, + LoadMode::Overwrite, + ) + .await + .unwrap(); + db.ensure_indices().await.unwrap(); + for (source, name, values) in [ + (ADULTS, "adults", vec![("min", "18")]), + (RANKED, "ranked", vec![("q", "graph")]), + (LIKED, "liked", vec![]), + ] { + let params = params(&values); + let main = || ReadTarget::branch("main"); + let document = db + .explain_query(main(), source, name, ¶ms) + .await + .unwrap(); + let run = db + .query_inspected(main(), source, name, ¶ms) + .await + .unwrap(); + let rendered = run.explain.to_value(); + assert_eq!( + document["physical_plan"], rendered["physical_plan"], + "{name}" + ); + assert_eq!( + document["validation"], + serde_json::json!({ "scope": run.evidence.scope().as_str() }), + "{name}" + ); + assert_eq!(document["validation"], rendered["validation"], "{name}"); + let ordinary = db.query(main(), source, name, ¶ms).await.unwrap(); + assert_eq!( + ordinary.concat_batches().unwrap(), + run.result.concat_batches().unwrap(), + "{name}" + ); + let statement = db + .query(main(), &format!("explain {source}"), name, ¶ms) + .await + .unwrap() + .concat_batches() + .unwrap(); + let column = |name: &str| { + statement + .column_by_name(name) + .unwrap() + .as_any() + .downcast_ref::() + .unwrap() + .clone() + }; + let (nodes, details) = (column("node"), column("detail")); + let validation = (0..statement.num_rows()) + .find(|row| nodes.value(*row) == "validation") + .map(|row| details.value(row).to_string()); + let validation: serde_json::Value = + serde_json::from_str(&validation.expect("the statement reports its validation")) + .unwrap(); + assert_eq!(validation, document["validation"], "{name}"); + } + let params = params(&[("q", "graph")]); + let main = || ReadTarget::branch("main"); + let refusals = [ + db.explain_query(main(), RANKED_DESTINATION, "ranked_destination", ¶ms) + .await + .err(), + db.query_inspected(main(), RANKED_DESTINATION, "ranked_destination", ¶ms) + .await + .err(), + db.query(main(), RANKED_DESTINATION, "ranked_destination", ¶ms) + .await + .err(), + db.query( + main(), + &format!("explain {RANKED_DESTINATION}"), + "ranked_destination", + ¶ms, + ) + .await + .err(), + ]; + for refusal in refusals { + let refusal = refusal.expect("ranking a destination is refused at every door"); + let diagnostic = refusal + .diagnostic() + .expect("a refusal by design carries its diagnostic"); + assert_eq!(diagnostic.code.as_str(), "P001", "{refusal}"); + } +} diff --git a/crates/omnigraph/tests/engine_v2_plan_replay.rs b/crates/omnigraph/tests/engine_v2_plan_replay.rs index 07863d61a..41b44e721 100644 --- a/crates/omnigraph/tests/engine_v2_plan_replay.rs +++ b/crates/omnigraph/tests/engine_v2_plan_replay.rs @@ -81,9 +81,10 @@ async fn incomplete_rank_fuse_row_tiebreak_refuses_replay_issue_659() { let expected = vec![serde_json::json!({"edge_type":"Knows","edge_id":"shared","target":larger})]; assert_eq!(rows_of(&first.result), expected); - let encoded = serde_json::to_value(&first.plan).unwrap(); - let restored = serde_json::from_value(encoded.clone()).unwrap(); - let replay = db.replay_bound_plan("main", restored).await.unwrap(); + let encoded: Value = + serde_json::from_slice(&first.replay_envelope(query, "selected_page")).unwrap(); + let restored = serde_json::to_vec(&encoded).unwrap(); + let replay = db.replay_bound_plan("main", &restored).await.unwrap(); assert_eq!(rows_of(&replay.result), expected); let type_key = serde_json::json!({"binding":"e","property":"@type"}); let edge_key = serde_json::json!({"binding":"e","property":"@id"}); @@ -103,7 +104,7 @@ async fn incomplete_rank_fuse_row_tiebreak_refuses_replay_issue_659() { ), ] { let mut altered = encoded.clone(); - let keys = altered["body"]["plan"]["slots"] + let keys = altered["plan"]["body"]["plan"]["slots"] .as_array_mut() .unwrap() .iter_mut() @@ -113,8 +114,8 @@ async fn incomplete_rank_fuse_row_tiebreak_refuses_replay_issue_659() { .expect("the saved RankFuse carries a row_tiebreak list"); assert_eq!(*keys, expected_keys); *keys = altered_keys; - let altered = serde_json::from_value(altered).unwrap(); - let error = match db.replay_bound_plan("main", altered).await { + let altered = serde_json::to_vec(&altered).unwrap(); + let error = match db.replay_bound_plan("main", &altered).await { Err(error) => error, Ok(replay) => panic!( "{mutation}: incomplete RankFuse row_tiebreak must refuse; replay returned {:?}", @@ -149,10 +150,17 @@ async fn wildcard_replay_keeps_captured_members_and_pins_every_member_issue_659( ) .await .unwrap(); - let captured: omnigraph_planner::BoundPlan = - serde_json::from_value(serde_json::to_value(&first.plan).unwrap()).unwrap(); - let replay = db.replay_bound_plan("main", captured).await.unwrap(); - assert_eq!(rows_of(&replay.result), rows_of(&first.result)); + let captured = first.replay_envelope(WILDCARD_LIKES_QUERY, "selected"); + let error = db + .replay_bound_plan("main", &captured) + .await + .err() + .expect("a plan accepted under the old schema is not replayed under the new one"); + assert!( + matches!(&error, OmniError::Manifest(manifest) if manifest.kind == omnigraph::error::ManifestErrorKind::Conflict) + && error.to_string().contains("accepted under schema"), + "{error}" + ); let fresh = db .query_inspected("main", WILDCARD_LIKES_QUERY, "selected", &ParamMap::new()) .await @@ -195,7 +203,10 @@ async fn wildcard_replay_keeps_captured_members_and_pins_every_member_issue_659( .await .unwrap(); let error = db - .replay_bound_plan("main", fresh.plan) + .replay_bound_plan( + "main", + &fresh.replay_envelope(WILDCARD_LIKES_QUERY, "selected"), + ) .await .err() .expect("every member is pinned"); @@ -409,23 +420,44 @@ async fn historical_replay_checks_marker_and_live_wildcards_issue_659() { .query_inspected("main", WILDCARD_LIKES_QUERY, "selected", &ParamMap::new()) .await .unwrap(); + let wildcard_envelope = |plan| { + omnigraph_planner::ReplayEnvelope::new( + WILDCARD_LIKES_QUERY, + "selected", + plan, + &wildcard.evidence, + wildcard.catalog.clone(), + ) + .to_bytes() + }; let mut live_only = wildcard.plan.clone(); let mut assumptions = live_only.plan.assumptions().clone(); assumptions.has_wildcard_traversal = false; live_only.plan.set_assumptions(assumptions); - let mut marker_only = db + let count = db .query_inspected("main", PEOPLE_QUERIES, "count_people", &ParamMap::new()) .await - .unwrap() - .plan; + .unwrap(); + let mut marker_only = count.plan.clone(); let mut assumptions = marker_only.plan.assumptions().clone(); assumptions.has_wildcard_traversal = true; assumptions.traversal_work_limit = wildcard.plan.plan.assumptions().traversal_work_limit; marker_only.plan.set_assumptions(assumptions); - for bound in [wildcard.plan, live_only, marker_only] { - let bound = serde_json::from_value(serde_json::to_value(bound).unwrap()).unwrap(); + let marker_only = omnigraph_planner::ReplayEnvelope::new( + PEOPLE_QUERIES, + "count_people", + marker_only, + &count.evidence, + count.catalog.clone(), + ) + .to_bytes(); + for envelope in [ + wildcard_envelope(wildcard.plan.clone()), + wildcard_envelope(live_only), + marker_only, + ] { let error = db - .replay_bound_plan(ReadTarget::Snapshot(snapshot.clone()), bound) + .replay_bound_plan(ReadTarget::Snapshot(snapshot.clone()), &envelope) .await .err() .expect("historical wildcard replay"); @@ -664,14 +696,14 @@ async fn replayed(db: &Session, source: &str, name: &str, params: &ParamMap) -> .query_inspected(ReadTarget::branch("main"), source, name, params) .await .unwrap(); - let serialized = serde_json::to_value(&run.plan).unwrap(); - let bound: omnigraph_planner::BoundPlan = serde_json::from_value(serialized).unwrap(); - assert_eq!(bound, run.plan, "the bound plan reads back equal"); + let envelope = run.replay_envelope(source, name); + let read_back = omnigraph_planner::decode_replay(&envelope, Default::default()).unwrap(); + assert_eq!(read_back.plan, run.plan, "the bound plan reads back equal"); let result = rows_of(&run.result); let rows = report_rows(&run.report); for _ in 0..2 { let replay = db - .replay_bound_plan(ReadTarget::branch("main"), bound.clone()) + .replay_bound_plan(ReadTarget::branch("main"), &envelope) .await .unwrap(); assert_eq!(rows_of(&replay.result), result); @@ -782,11 +814,14 @@ async fn edited_replay_refusal( ) .await .unwrap(); - let mut serialized = serde_json::to_value(&run.plan).unwrap(); - edit_objects(&mut serialized, edit); - let bound: omnigraph_planner::BoundPlan = serde_json::from_value(serialized).unwrap(); + let mut serialized: Value = + serde_json::from_slice(&run.replay_envelope(CITATION_QUERIES, "cited")).unwrap(); + edit_objects(&mut serialized["plan"], edit); match db - .replay_bound_plan(ReadTarget::branch("main"), bound) + .replay_bound_plan( + ReadTarget::branch("main"), + &serde_json::to_vec(&serialized).unwrap(), + ) .await { Ok(replay) => panic!("the edited plan replays: {:?}", rows_of(&replay.result)), @@ -817,23 +852,46 @@ async fn a_marker_that_disagrees_with_its_contains_join_refuses_the_replay() { } } -/// A replayed plan whose `ContainsJoin` became a plain `CrossJoin` leaves a -/// marked scan no join fills: the lowering refuses it instead of running it. +/// A replayed plan whose `ContainsJoin` became a `CrossJoin` testing the same +/// conjuncts passes acceptance but leaves a marked scan no join fills: the +/// lowering refuses it instead of running it. The same rewrite dropping the +/// conjunct is refused by acceptance, which finds the predicate gone. #[tokio::test] async fn a_marker_with_no_contains_join_refuses_the_replay() { - let error = edited_replay_refusal(&mut |object| { - if object.get("node") == Some(&Value::from("ContainsJoin")) { + use omnigraph_compiler::ir::IRExpr; + use omnigraph_compiler::query::ast::CompOp; + use omnigraph_planner::mirror::ExprMirror; + for keep_conjunct in [true, false] { + let error = edited_replay_refusal(&mut |object| { + if object.get("node") != Some(&Value::from("ContainsJoin")) { + return; + } + let side = |key: &str| { + let pair = object[key].as_array().unwrap(); + IRExpr::PropAccess { + variable: pair[0].as_str().unwrap().to_string(), + property: pair[1].as_str().unwrap().to_string(), + } + }; + let conjunct = + IRExpr::comparison(side("haystack"), CompOp::StringContains, side("needle")); + let mut filters = Vec::new(); + if keep_conjunct { + filters.push(serde_json::to_value(ExprMirror::from(&conjunct)).unwrap()); + } + filters.extend(object["residual"].as_array().cloned().unwrap_or_default()); object.retain(|key, _| matches!(key.as_str(), "node" | "left" | "right")); - object.insert("node".to_string(), Value::from("CrossJoin")); - } - }) - .await; - assert!( - error - .to_string() - .contains("carries a runtime filter no contains join fills"), - "{error}" - ); + object.insert("node".to_string(), Value::from("FilteredCrossJoin")); + object.insert("filters".to_string(), Value::from(filters)); + }) + .await; + let expected = if keep_conjunct { + "carries a runtime filter no contains join fills" + } else { + "predicate retention: the plan tests no conjunct for `$p.text contains $m.number`" + }; + assert!(error.to_string().contains(expected), "{error}"); + } } #[tokio::test] @@ -953,7 +1011,10 @@ async fn an_edge_write_after_planning_refuses_the_replay() { .await .unwrap(); let refused = db - .replay_bound_plan(ReadTarget::branch("main"), run.plan.clone()) + .replay_bound_plan( + ReadTarget::branch("main"), + &run.replay_envelope(PEOPLE_QUERIES, "liked"), + ) .await .err() .expect("the edge table moved"); @@ -987,7 +1048,10 @@ async fn a_write_to_an_unread_table_leaves_the_replay_accepted() { .await .unwrap(); let replay = db - .replay_bound_plan(ReadTarget::branch("main"), run.plan.clone()) + .replay_bound_plan( + ReadTarget::branch("main"), + &run.replay_envelope(PEOPLE_QUERIES, "count_people"), + ) .await .expect("the counted table did not move"); assert_eq!(rows_of(&replay.result), rows_of(&run.result)); @@ -1130,7 +1194,10 @@ async fn an_insert_after_planning_refuses_the_replay_of_a_count() { .await .unwrap(); let refused = db - .replay_bound_plan(ReadTarget::branch("main"), run.plan.clone()) + .replay_bound_plan( + ReadTarget::branch("main"), + &run.replay_envelope(PEOPLE_QUERIES, "count_people"), + ) .await .err() .expect("the counted table moved"); @@ -1200,15 +1267,14 @@ async fn sibling_plan_replay_is_refused( .await .unwrap(); assert_ne!(rows_of(&run.result), rows_of(&other.result)); - let bound: omnigraph_planner::BoundPlan = - serde_json::from_value(serde_json::to_value(&run.plan).unwrap()).unwrap(); + let envelope = run.replay_envelope(PEOPLE_QUERIES, query_name); let same = db - .replay_bound_plan(ReadTarget::branch("left"), bound.clone()) + .replay_bound_plan(ReadTarget::branch("left"), &envelope) .await .unwrap(); assert_eq!(rows_of(&same.result), rows_of(&run.result)); let refused = db - .replay_bound_plan(ReadTarget::branch("right"), bound) + .replay_bound_plan(ReadTarget::branch("right"), &envelope) .await .err() .expect("equal lineage counters must not admit a different detached pin"); diff --git a/crates/omnigraph/tests/engine_v2_scrubbed_replay.rs b/crates/omnigraph/tests/engine_v2_scrubbed_replay.rs index d64116d92..e9bcb1bd8 100644 --- a/crates/omnigraph/tests/engine_v2_scrubbed_replay.rs +++ b/crates/omnigraph/tests/engine_v2_scrubbed_replay.rs @@ -163,7 +163,10 @@ async fn the_replay_runs_under_the_captured_memory_limit_not_the_ambient_one() { ); let replay = with_query_memory_limit( 1, - v2.replay_bound_plan(ReadTarget::branch("main"), run.plan.clone()), + v2.replay_bound_plan( + ReadTarget::branch("main"), + &run.replay_envelope(QUERY, "liked"), + ), ) .await .expect("the ambient 1-byte limit is not read; the plan's 64 MiB is"); @@ -209,7 +212,10 @@ async fn the_replay_reads_the_plans_env_pair_not_the_process_environment() { // SAFETY: the test is `#[serial]` and restores the variable before it returns. unsafe { std::env::set_var(EXPAND_INDEXED_MAX_FRONTIER_ENV, "1") }; let replay = v2 - .replay_bound_plan(ReadTarget::branch("main"), run.plan.clone()) + .replay_bound_plan( + ReadTarget::branch("main"), + &run.replay_envelope(QUERY, "liked"), + ) .await; let replanned = v2 .query_inspected(ReadTarget::branch("main"), QUERY, "liked", &ParamMap::new()) @@ -301,13 +307,14 @@ async fn ann_nprobes_is_a_field_of_the_ranked_scan_and_the_rows_do_not_move() { .contains(&format!("\"nprobes\":{nprobes}")), "{explain}" ); + let envelope = run.replay_envelope(NEAREST, "by_vector"); let replay = session - .replay_bound_plan(ReadTarget::branch("main"), run.plan.clone()) + .replay_bound_plan(ReadTarget::branch("main"), &envelope) .await .unwrap(); assert_eq!(rows_of(&replay.result), rows_of(&run.result)); runs.push(rows_of(&run.result)); - plans.push(run.plan.clone()); + plans.push(envelope); } assert_eq!( runs[0], runs[1], @@ -315,7 +322,7 @@ async fn ann_nprobes_is_a_field_of_the_ranked_scan_and_the_rows_do_not_move() { ); assert_eq!(runs[0].len(), 3); let crossed = with_setting(&v2, "ann_nprobes", "64") - .replay_bound_plan(ReadTarget::branch("main"), plans[0].clone()) + .replay_bound_plan(ReadTarget::branch("main"), &plans[0]) .await .expect("the door takes a plan gathered under another session's cap"); assert_eq!( From 13d7e69335aeb7d4184754456e4f8c28e3ddda13 Mon Sep 17 00:00:00 2001 From: Ragnor Comerford Date: Thu, 1 Oct 2026 17:23:59 +0200 Subject: [PATCH 04/29] planner: typed order keys; the validator recomputes the order it checks A node's declared ordering is typed order keys (column, expression with direction and null placement, identity, fused rank), derived by one rule from the operator and its inputs: a sort establishes exactly its comparator, identity keys included, and a filter keeps its input's order. The bound plan format carries them (version 2). Acceptance recomputes the root's order from the operators rather than reading declared properties, and each check has a test that breaks the plan it guards. --- crates/omnigraph-planner/src/bound.rs | 5 +- crates/omnigraph-planner/src/lib.rs | 6 +- crates/omnigraph-planner/src/mirror.rs | 77 +++- crates/omnigraph-planner/src/optimizer.rs | 172 +++++---- crates/omnigraph-planner/src/physical.rs | 60 ++- .../src/validate/invariants.rs | 9 +- crates/omnigraph-planner/src/validate/mod.rs | 3 + .../omnigraph-planner/src/validate/tests.rs | 343 ++++++++++++++++++ crates/omnigraph-planner/tests/bound_plan.rs | 26 +- .../tests/query_plan/explain.rs | 3 +- 10 files changed, 592 insertions(+), 112 deletions(-) create mode 100644 crates/omnigraph-planner/src/validate/tests.rs diff --git a/crates/omnigraph-planner/src/bound.rs b/crates/omnigraph-planner/src/bound.rs index c0d4ed373..683c0bf7f 100644 --- a/crates/omnigraph-planner/src/bound.rs +++ b/crates/omnigraph-planner/src/bound.rs @@ -15,7 +15,10 @@ use crate::mirror::BoundPlanMirror; use crate::physical::{NodeId, PhysicalPlan}; /// Saved execution plans use a separate version from rendered explain output. -pub const BOUND_PLAN_VERSION: u32 = 1; +/// Version 2: a node's declared ordering is typed order keys (column, +/// expression with direction and null placement, identity, fused rank) +/// instead of their text. +pub const BOUND_PLAN_VERSION: u32 = 2; /// The envelope omits top-level `plan` and `values` so unversioned readers reject it. #[derive(Serialize, Deserialize)] diff --git a/crates/omnigraph-planner/src/lib.rs b/crates/omnigraph-planner/src/lib.rs index 37071f3c3..85b1eda82 100644 --- a/crates/omnigraph-planner/src/lib.rs +++ b/crates/omnigraph-planner/src/lib.rs @@ -58,9 +58,9 @@ pub use lower::{ pub use operation::{Operation, PageBudgetSpec, ScopeSpec, Side, TableRef}; pub use optimizer::{Bounds, physical_plan, rewrite}; pub use physical::{ - Assumptions, DatasetPin, Estimate, GatePolicy, Hop, NodeId, OverfetchRung, PhysicalNode, - PhysicalPlan, Prefilter, PrefilterMode, Properties, RankArm, RankKind, RankScope, RankedAccess, - ScanInput, StatisticSource, + Assumptions, DatasetPin, Estimate, GatePolicy, Hop, NodeId, OrderKey, OverfetchRung, + PhysicalNode, PhysicalPlan, Prefilter, PrefilterMode, Properties, RankArm, RankKind, RankScope, + RankedAccess, ScanInput, StatisticSource, }; pub use registry::{Coverage, Entry, Route, Shape}; pub use route::RouteOverride; diff --git a/crates/omnigraph-planner/src/mirror.rs b/crates/omnigraph-planner/src/mirror.rs index 478d77647..480aba05b 100644 --- a/crates/omnigraph-planner/src/mirror.rs +++ b/crates/omnigraph-planner/src/mirror.rs @@ -23,8 +23,8 @@ use crate::error::PlanError; use crate::logical::{ColumnRef, GqFilter, KeyJoinKind, Predicate, RuntimeFilterSpec, ScanSpec}; use crate::operation::TableRef; use crate::physical::{ - Assumptions, Estimate, Hop, NodeId, OverfetchRung, PhysicalNode, PhysicalPlan, Prefilter, - Properties, RankArm, RankKind, RankScope, RankedAccess, ScanInput, StatisticSource, + Assumptions, Estimate, Hop, NodeId, OrderKey, OverfetchRung, PhysicalNode, PhysicalPlan, + Prefilter, Properties, RankArm, RankKind, RankScope, RankedAccess, ScanInput, StatisticSource, }; use crate::source::SideId; @@ -1437,12 +1437,74 @@ impl From for IROrdering { } } +/// One typed order key of a node's declared output order. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +#[serde(tag = "key", rename_all = "snake_case")] +pub enum OrderKeyMirror { + Column { + name: String, + }, + Expr { + expr: ExprMirror, + descending: bool, + nulls_first: bool, + }, + Identity { + column: ColumnRef, + }, + Fused { + bindings: Vec, + }, +} + +impl From<&OrderKey> for OrderKeyMirror { + fn from(key: &OrderKey) -> Self { + match key { + OrderKey::Column(name) => Self::Column { name: name.clone() }, + OrderKey::Expr { + expr, + descending, + nulls_first, + } => Self::Expr { + expr: ExprMirror::from(expr), + descending: *descending, + nulls_first: *nulls_first, + }, + OrderKey::Identity(column) => Self::Identity { + column: column.clone(), + }, + OrderKey::Fused { bindings } => Self::Fused { + bindings: bindings.clone(), + }, + } + } +} + +impl From for OrderKey { + fn from(mirror: OrderKeyMirror) -> Self { + match mirror { + OrderKeyMirror::Column { name } => Self::Column(name), + OrderKeyMirror::Expr { + expr, + descending, + nulls_first, + } => Self::Expr { + expr: IRExpr::from(expr), + descending, + nulls_first, + }, + OrderKeyMirror::Identity { column } => Self::Identity(column), + OrderKeyMirror::Fused { bindings } => Self::Fused { bindings }, + } + } +} + #[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] pub struct PropertiesMirror { pub schema: Vec, #[serde(default)] pub schema_metadata: HashMap, - pub ordering: Option>, + pub ordering: Option>, pub rows: Estimate, pub work_bytes: Estimate, pub retained_limit: Option, @@ -1459,7 +1521,10 @@ impl From<&Properties> for PropertiesMirror { .map(|field| FieldMirror::from(field.as_ref())) .collect(), schema_metadata: properties.schema.metadata().clone(), - ordering: properties.ordering.clone(), + ordering: properties + .ordering + .as_ref() + .map(|keys| keys.iter().map(OrderKeyMirror::from).collect()), rows: properties.rows, work_bytes: properties.work_bytes, retained_limit: properties.retained_limit, @@ -1484,7 +1549,9 @@ impl TryFrom for Properties { let schema: SchemaRef = Arc::new(Schema::new_with_metadata(fields, mirror.schema_metadata)); Ok(Self { schema, - ordering: mirror.ordering, + ordering: mirror + .ordering + .map(|keys| keys.into_iter().map(OrderKey::from).collect()), rows: mirror.rows, work_bytes: mirror.work_bytes, retained_limit: mirror.retained_limit, diff --git a/crates/omnigraph-planner/src/optimizer.rs b/crates/omnigraph-planner/src/optimizer.rs index 186fba1f3..da17145d8 100644 --- a/crates/omnigraph-planner/src/optimizer.rs +++ b/crates/omnigraph-planner/src/optimizer.rs @@ -22,13 +22,13 @@ use crate::error::{PlanError, SET_TRAVERSAL_WORK_LIMIT}; use crate::logical::{ ColumnRef, EDGE_TYPE_MEMBER, GqFilter, IDENTITY_MEMBER, KeyJoinKind, LOGICAL_ID, LogicalId, LogicalNode, LogicalPlan, Predicate, RuntimeFilterKind, RuntimeFilterSpec, ScanSpec, SearchArm, - ordering_text, tiebreak_text, + tiebreak_text, }; use crate::lower::ContainsJoinFields; use crate::operation::{Operation, Side}; use crate::physical::{ - Assumptions, Estimate, Hop, NodeId, OverfetchRung, PhysicalNode, PhysicalPlan, Prefilter, - Properties, RankArm, RankKind, RankScope, RankedAccess, ScanInput, StatisticSource, + Assumptions, Estimate, Hop, NodeId, OrderKey, OverfetchRung, PhysicalNode, PhysicalPlan, + Prefilter, Properties, RankArm, RankKind, RankScope, RankedAccess, ScanInput, StatisticSource, TextContains, }; use crate::source::{NodeTypeSpec, PlanSource, SideId}; @@ -2612,8 +2612,9 @@ impl Lowering<'_> { } LogicalNode::Ordered { input, keys } => { let lowered = self.lower(*input)?; - let declared = declared_ordering(&self.physical, lowered).unwrap_or_default(); - if declared.starts_with(keys) { + let declared = derived_order(&self.physical, lowered).unwrap_or_default(); + let wanted: Vec = keys.iter().cloned().map(OrderKey::Column).collect(); + if declared.starts_with(&wanted) { Ok(lowered) } else { Err(PlanError::Internal(format!( @@ -3071,83 +3072,75 @@ fn variable_offset_width(data_type: &DataType) -> u64 { /// no limit. pub(crate) const RRF_NEAREST_ARM_K: usize = 100; -/// The ordering a ranked scan or a fusion carries: `nearest` ranks by -/// ascending `_distance`, `bm25` by descending `_score`, `rrf` by the fused rank. -fn search_ordering(node: &PhysicalNode) -> Option> { +/// The order a node's output leaves in, from the operator's definition and +/// its inputs' orders (`input`): a ranking scan leaves in score order +/// (`nearest` ascending `_distance`, `bm25` descending `_score`), a fusion in +/// fused rank, an id-ordered scan or a key merge in its key's order, a sort +/// in exactly its comparator (its keys, then its identity keys), and an +/// order-preserving operator (a dependent scan, a hash join's probe, a +/// filter, a projection, a limit, a page, a hydration, a row compare) in its +/// input's order; every other operator declares none. +pub fn node_order( + node: &PhysicalNode, + input: impl Fn(NodeId) -> Option>, +) -> Option> { match node { PhysicalNode::Scan { - spec, - ranked: Some(ranked), - .. - } => { - let binding = spec.binding.as_deref()?; - Some(vec![ordering_text(&ranked.ordering(binding))]) - } - PhysicalNode::RankFuse { arms, .. } => { - let targets: Vec = arms.iter().map(|arm| format!("${}", arm.binding)).collect(); - Some(vec![format!("rrf({}) desc", targets.join(", "))]) - } - _ => None, - } -} - -/// `keys` appended to `input`'s ordering, each key once: a sort's ordering -/// is its keys, and a ranked input's score key leads them already. -fn sorted_ordering(input: Option>, keys: &[IROrdering]) -> Vec { - let mut ordering = input.unwrap_or_default(); - for key in keys.iter().map(ordering_text) { - if !ordering.contains(&key) { - ordering.push(key); - } - } - ordering -} - -/// The ordering a node declares by construction, in logical column names. -pub fn declared_ordering(plan: &PhysicalPlan, id: NodeId) -> Option> { - match plan.node(id)? { - PhysicalNode::Scan { - source: ScanInput::Dependent { input, .. }, - .. - } => declared_ordering(plan, *input), - node @ PhysicalNode::Scan { - source: ScanInput::Table, - ranked: Some(_), + source: ScanInput::Dependent { input: probe, .. }, .. - } => search_ordering(node), + } => input(*probe), PhysicalNode::Scan { source: ScanInput::Table, - ordered: true, + spec, + ranked: Some(ranked), .. - } => Some(vec![LOGICAL_ID.to_string()]), + } => Some(vec![OrderKey::of( + &ranked.ordering(spec.binding.as_deref()?), + )]), PhysicalNode::Scan { source: ScanInput::Table, - ordered: false, + ordered, .. - } => None, - PhysicalNode::SortMergeJoin { on, .. } => Some(vec![on.clone()]), - PhysicalNode::HashJoin { probe, .. } => declared_ordering(plan, *probe), - PhysicalNode::HydrateByAddress { input, .. } - | PhysicalNode::RowCompare { input, .. } - | PhysicalNode::ClassifyThreeWay { input } - | PhysicalNode::Page { input, .. } - | PhysicalNode::Limit { input, .. } - | PhysicalNode::Projection { input, .. } => declared_ordering(plan, *input), - node @ PhysicalNode::RankFuse { .. } => search_ordering(node), + } => ordered.then(|| vec![OrderKey::Column(LOGICAL_ID.to_string())]), + PhysicalNode::SortMergeJoin { on, .. } => Some(vec![OrderKey::Column(on.clone())]), + PhysicalNode::ClassifyThreeWay { .. } => { + Some(vec![OrderKey::Column(LOGICAL_ID.to_string())]) + } + PhysicalNode::HashJoin { probe, .. } => input(*probe), + PhysicalNode::HydrateByAddress { input: from, .. } + | PhysicalNode::RowCompare { input: from } + | PhysicalNode::Page { input: from, .. } + | PhysicalNode::Limit { input: from, .. } + | PhysicalNode::Filter { input: from, .. } + | PhysicalNode::Projection { input: from, .. } => input(*from), + PhysicalNode::RankFuse { arms, .. } => Some(vec![OrderKey::Fused { + bindings: arms.iter().map(|arm| arm.binding.clone()).collect(), + }]), PhysicalNode::Sort { - input, order_by, .. - } => Some(sorted_ordering(declared_ordering(plan, *input), order_by)), + order_by, tiebreak, .. + } => Some( + order_by + .iter() + .map(OrderKey::of) + .chain(tiebreak.iter().cloned().map(OrderKey::Identity)) + .collect(), + ), PhysicalNode::MetadataCount { .. } | PhysicalNode::CrossJoin { .. } | PhysicalNode::ContainsJoin { .. } | PhysicalNode::OuterReference { .. } - | PhysicalNode::Filter { .. } | PhysicalNode::Expand { .. } | PhysicalNode::AntiJoin { .. } | PhysicalNode::Aggregate { .. } => None, } } +/// [`node_order`] of `id`, recomputed from the operators of its subtree and +/// never read from declared properties. +pub fn derived_order(plan: &PhysicalPlan, id: NodeId) -> Option> { + node_order(plan.node(id)?, |input| derived_order(plan, input)) +} + /// Stage 3. Bottom-up: output schema, ordering, row estimate, estimated work /// bytes, retained-memory limit and the statistic each derives from. fn derive_properties( @@ -3161,10 +3154,14 @@ fn derive_properties( .node(id) .cloned() .ok_or_else(|| PlanError::Internal(format!("physical node {id} is a tombstone")))?; + let ordering = node_order(&node, |input| { + plan.properties(input) + .and_then(|properties| properties.ordering.clone()) + }); let properties = match &node { PhysicalNode::MetadataCount { return_exprs, .. } => Properties { schema: metadata_count_schema(return_exprs)?, - ordering: None, + ordering: ordering.clone(), rows: Estimate::Known(1), work_bytes: Estimate::Unknown, retained_limit: None, @@ -3178,7 +3175,7 @@ fn derive_properties( let input = props(plan, *input)?; Properties { schema: input.schema.clone(), - ordering: input.ordering.clone(), + ordering: ordering.clone(), rows: Estimate::Unknown, work_bytes: Estimate::Unknown, retained_limit: None, @@ -3188,7 +3185,6 @@ fn derive_properties( PhysicalNode::Scan { source: ScanInput::Table, spec, - ordered, keys_only: true, .. } => { @@ -3206,7 +3202,7 @@ fn derive_properties( }); Properties { schema: key_schema(spec), - ordering: ordered.then(|| vec![LOGICAL_ID.to_string()]), + ordering: ordering.clone(), rows, work_bytes, retained_limit: None, @@ -3216,9 +3212,9 @@ fn derive_properties( PhysicalNode::Scan { source: ScanInput::Table, spec, - ordered, keys_only: false, ranked, + .. } => { let (rows, sources) = if spec.binding.is_some() { query_scan_rows(spec, source) @@ -3243,8 +3239,7 @@ fn derive_properties( }; Properties { schema, - ordering: search_ordering(&node) - .or_else(|| ordered.then(|| vec![LOGICAL_ID.to_string()])), + ordering: ordering.clone(), rows, work_bytes: Estimate::Unknown, retained_limit: None, @@ -3255,7 +3250,6 @@ fn derive_properties( left, right, kind, - on, build, .. } => { @@ -3294,7 +3288,7 @@ fn derive_properties( &right_props.schema, physical_prefix(plan, *right), ), - ordering: Some(vec![on.clone()]), + ordering: ordering.clone(), rows, work_bytes: sum_estimates(left_props.work_bytes, right_props.work_bytes), retained_limit, @@ -3306,7 +3300,7 @@ fn derive_properties( let schema = expand_side(&input_props.schema, *side, &source.schema(*side)?); Properties { schema, - ordering: input_props.ordering.clone(), + ordering: ordering.clone(), rows: input_props.rows, work_bytes: Estimate::Unknown, retained_limit: Some(bounds.hydration_chunk_hard_bytes), @@ -3321,7 +3315,7 @@ fn derive_properties( let input = props(plan, *input)?; Properties { schema: diff_schema(&input.schema), - ordering: input.ordering.clone(), + ordering: ordering.clone(), rows: input.rows, work_bytes: input.work_bytes, retained_limit: None, @@ -3332,7 +3326,7 @@ fn derive_properties( let input = props(plan, *input)?; Properties { schema: classify_schema(&source.schema(SideId::Base)?), - ordering: Some(vec![LOGICAL_ID.to_string()]), + ordering: ordering.clone(), rows: input.rows, work_bytes: input.work_bytes, retained_limit: None, @@ -3348,7 +3342,7 @@ fn derive_properties( }; Properties { schema: input.schema.clone(), - ordering: input.ordering.clone(), + ordering: ordering.clone(), rows: bounded, work_bytes: input.work_bytes, retained_limit: None, @@ -3375,7 +3369,7 @@ fn derive_properties( &right_props.schema, physical_prefix(plan, *right), ), - ordering: None, + ordering: ordering.clone(), rows, work_bytes: Estimate::Unknown, retained_limit: None, @@ -3392,7 +3386,7 @@ fn derive_properties( &right_props.schema, physical_prefix(plan, *right), ), - ordering: None, + ordering: ordering.clone(), rows: Estimate::Unknown, work_bytes: Estimate::Unknown, retained_limit: None, @@ -3401,19 +3395,17 @@ fn derive_properties( } PhysicalNode::OuterReference { .. } => Properties { schema: Arc::new(Schema::empty()), - ordering: None, + ordering: ordering.clone(), rows: Estimate::Unknown, work_bytes: Estimate::Unknown, retained_limit: None, sources: Vec::new(), }, - PhysicalNode::Sort { - input, order_by, .. - } => { + PhysicalNode::Sort { input, .. } => { let input = props(plan, *input)?; Properties { schema: input.schema.clone(), - ordering: Some(sorted_ordering(input.ordering.clone(), order_by)), + ordering: ordering.clone(), rows: input.rows, work_bytes: Estimate::Unknown, retained_limit: None, @@ -3427,7 +3419,7 @@ fn derive_properties( let input = props(plan, *input)?; Properties { schema: input.schema.clone(), - ordering: None, + ordering: ordering.clone(), rows: Estimate::Unknown, work_bytes: Estimate::Unknown, retained_limit: None, @@ -3438,7 +3430,7 @@ fn derive_properties( let input = props(plan, *input)?; Properties { schema: input.schema.clone(), - ordering: input.ordering.clone(), + ordering: ordering.clone(), rows: input.rows, work_bytes: Estimate::Unknown, retained_limit: None, @@ -3455,7 +3447,7 @@ fn derive_properties( }; Properties { schema: input.schema.clone(), - ordering: search_ordering(&node), + ordering: ordering.clone(), rows, work_bytes: Estimate::Unknown, retained_limit: None, @@ -3725,7 +3717,10 @@ mod tests { .properties(optimized.physical.root()) .expect("root props"); assert_eq!(root.rows, Estimate::Known(1)); - assert_eq!(root.ordering.as_deref(), Some(&["id".to_string()][..])); + assert_eq!( + root.ordering.as_deref(), + Some(&[OrderKey::Column("id".to_string())][..]) + ); let pipelines = optimized.physical.pipelines_json(); assert_eq!(pipelines.as_array().map(Vec::len), Some(2)); assert_eq!(pipelines[0]["sink"], "SortMergeJoin(build)"); @@ -3918,7 +3913,10 @@ mod tests { .properties(optimized.physical.root()) .expect("root props"); assert_eq!(root.rows, Estimate::Known(15)); - assert_eq!(root.ordering.as_deref(), Some(&["id".to_string()][..])); + assert_eq!( + root.ordering.as_deref(), + Some(&[OrderKey::Column("id".to_string())][..]) + ); let names: Vec<&str> = root .schema .fields() diff --git a/crates/omnigraph-planner/src/physical.rs b/crates/omnigraph-planner/src/physical.rs index 11e9f0317..e866f4ea5 100644 --- a/crates/omnigraph-planner/src/physical.rs +++ b/crates/omnigraph-planner/src/physical.rs @@ -357,13 +357,65 @@ pub struct StatisticSource { pub origin: &'static str, } +/// One key of the order a node's output leaves in, typed: the comparator a +/// sort establishes, the score order a ranking scan produces, or the logical +/// id order of a diff or merge plan. +#[derive(Debug, Clone, PartialEq)] +pub enum OrderKey { + /// A diff or merge plan's logical column, ascending. + Column(String), + /// An expression with its direction and null placement. + Expr { + expr: IRExpr, + descending: bool, + nulls_first: bool, + }, + /// A binding's identity metadata key (`@id`, or `@type` of a selected + /// edge): ascending, nulls first. + Identity(ColumnRef), + /// The fused reciprocal rank of an `rrf()` over its arms' bindings, + /// descending. + Fused { bindings: Vec }, +} + +impl OrderKey { + /// A written or score sort key: ascending keys place nulls first, + /// descending keys nulls last (RFC 0047, "One total order"). + pub fn of(ordering: &IROrdering) -> Self { + Self::Expr { + expr: ordering.expr.clone(), + descending: ordering.descending, + nulls_first: !ordering.descending, + } + } +} + +impl std::fmt::Display for OrderKey { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + Self::Column(column) => f.write_str(column), + Self::Expr { + expr, descending, .. + } => write!(f, "{expr} {}", if *descending { "desc" } else { "asc" }), + Self::Identity(column) => write!(f, "${column} asc"), + Self::Fused { bindings } => { + let targets: Vec = bindings + .iter() + .map(|binding| format!("${binding}")) + .collect(); + write!(f, "rrf({}) desc", targets.join(", ")) + } + } + } +} + /// The properties a physical node declares: derived after selection, /// recomputed when a later pass changes the node, never recomputed by an /// executor. Each is a bound the executor adapts within, never above. #[derive(Debug, Clone)] pub struct Properties { pub schema: SchemaRef, - pub ordering: Option>, + pub ordering: Option>, pub rows: Estimate, pub work_bytes: Estimate, pub retained_limit: Option, @@ -374,8 +426,12 @@ impl Properties { /// A query plan prints no `schema`: its run-time schemas are the /// engine's to derive, and the planner's are conservative input schemas. fn to_json(&self, query: bool) -> Value { + let ordering: Option> = self + .ordering + .as_ref() + .map(|keys| keys.iter().map(ToString::to_string).collect()); let mut value = json!({ - "ordering": self.ordering, + "ordering": ordering, "rows": self.rows, "work_bytes": self.work_bytes, "retained_limit": self.retained_limit, diff --git a/crates/omnigraph-planner/src/validate/invariants.rs b/crates/omnigraph-planner/src/validate/invariants.rs index 0b21cd333..a06fbd210 100644 --- a/crates/omnigraph-planner/src/validate/invariants.rs +++ b/crates/omnigraph-planner/src/validate/invariants.rs @@ -23,7 +23,7 @@ use super::requirements::{ use super::{AcceptInput, ValidationError}; use crate::logical::{ColumnRef, EDGE_TYPE_MEMBER, IDENTITY_MEMBER}; use crate::lower::ContainsJoinFields; -use crate::optimizer::RRF_NEAREST_ARM_K; +use crate::optimizer::{RRF_NEAREST_ARM_K, derived_order}; use crate::physical::{ NodeId, OverfetchRung, PhysicalNode, PhysicalPlan, RankKind, RankScope, RankedAccess, }; @@ -631,6 +631,13 @@ impl Requirements { else { unreachable!("the loop stops at a sort"); }; + budget.visit(u64::try_from(top.len()).unwrap_or(u64::MAX))?; + if derived_order(plan, plan.root()) != derived_order(plan, id) { + return Err(ValidationError::violated( + "order", + "the plan's root does not leave its rows in the final sort's order", + )); + } let expected_fetch = self.limit.and_then(|limit| usize::try_from(limit).ok()); if *fetch != expected_fetch { return Err(ValidationError::violated( diff --git a/crates/omnigraph-planner/src/validate/mod.rs b/crates/omnigraph-planner/src/validate/mod.rs index 7eeb0589c..3006b283f 100644 --- a/crates/omnigraph-planner/src/validate/mod.rs +++ b/crates/omnigraph-planner/src/validate/mod.rs @@ -214,3 +214,6 @@ pub(crate) fn accept( }, }) } + +#[cfg(test)] +mod tests; diff --git a/crates/omnigraph-planner/src/validate/tests.rs b/crates/omnigraph-planner/src/validate/tests.rs new file mode 100644 index 000000000..1da3ecb63 --- /dev/null +++ b/crates/omnigraph-planner/src/validate/tests.rs @@ -0,0 +1,343 @@ +//! Each acceptance check refuses the plan it exists for. Rust and not +//! `.gqt`: a case only reaches plans the planner builds, and the planner +//! builds none of these; each test plans a query, breaks one thing its query +//! requires, and asserts the check that names it. + +use std::sync::Arc; + +use omnigraph_compiler::CheckedQuery; +use omnigraph_compiler::catalog::{Catalog, build_catalog}; +use omnigraph_compiler::ir::{IRExpr, ParamMap, QueryIR, fold}; +use omnigraph_compiler::query::ast::Literal; +use omnigraph_compiler::schema::parser::parse_schema; + +use super::*; +use crate::fixture_bounds::BOUNDS; +use crate::logical::{ColumnRef, IDENTITY_MEMBER}; +use crate::operation::TableRef; +use crate::physical::{PhysicalNode, PhysicalPlan}; +use crate::source::{MemorySource, NodeTypeSpec}; + +const SCHEMA: &str = r#" +node Doc { + slug: String @key + title: String + year: I64 + open: Bool +} +"#; + +/// The bound-literal evaluator: literals and bound parameters through the +/// compiler's fold rules. +struct Bound<'p>(&'p ParamMap); + +impl ConstantEvaluator for Bound<'_> { + fn evaluate(&self, expr: &IRExpr) -> Option { + match expr { + IRExpr::Literal(literal) => Some(literal.clone()), + IRExpr::Param(name) => self.0.get(name).cloned(), + IRExpr::Binary { left, op, right } => { + fold::evaluate(*op, &self.evaluate(left)?, &self.evaluate(right)?) + } + IRExpr::Not(inner) => match self.evaluate(inner)? { + Literal::Bool(value) => Some(Literal::Bool(!value)), + Literal::Null => Some(Literal::Null), + _ => None, + }, + IRExpr::IsNull { expr, negated } => Some(Literal::Bool( + matches!(self.evaluate(expr)?, Literal::Null) != *negated, + )), + _ => None, + } + } +} + +struct Fixture { + catalog: Catalog, + checked: CheckedQuery, + ir: QueryIR, + params: ParamMap, + source: MemorySource, +} + +impl Fixture { + fn new(query: &str, params: &[(&str, Literal)]) -> Self { + let catalog = build_catalog(&parse_schema(SCHEMA).unwrap()).unwrap(); + let decl = omnigraph_compiler::find_named_query(query, "q").unwrap(); + let checked = CheckedQuery::check(&catalog, &decl).unwrap(); + let ir = + omnigraph_compiler::lower_query(&catalog, checked.decl(), checked.types()).unwrap(); + let mut source = MemorySource::default(); + for (name, node_type) in &catalog.node_types { + source = source.with_node_type( + name, + NodeTypeSpec { + table: TableRef { + type_key: format!("node:{name}"), + dataset_path: format!("node/{name}"), + native_branch: None, + }, + version: Some(1), + columns: catalog.system_columns, + schema: Arc::clone(&node_type.arrow_schema), + key: node_type.key.clone().unwrap_or_default(), + object_columns: node_type + .arrow_schema + .fields() + .iter() + .map(|field| field.name().clone()) + .collect(), + row_count: None, + }, + ); + } + Self { + catalog, + checked, + ir, + params: params + .iter() + .map(|(name, value)| (name.to_string(), value.clone())) + .collect(), + source, + } + } + + fn plan(&self) -> PhysicalPlan { + crate::gate::plan_query(&self.ir, &self.source, &BOUNDS).expect("the query plans") + } + + fn accept(&self, plan: PhysicalPlan) -> Result { + let constants = Bound(&self.params); + let input = AcceptInput { + checked: &self.checked, + catalog: &self.catalog, + ir: &self.ir, + params: &self.params, + constants: &constants, + limits: ValidationLimits::DEFAULT, + }; + accept(plan, &input) + } + + /// The check that refuses `plan`, which must be refused. + fn refused(&self, plan: PhysicalPlan) -> (&'static str, String) { + match self.accept(plan) { + Err(ValidationError::Violated { check, detail }) => (check, detail), + other => panic!("the broken plan was not refused by a check: {other:?}"), + } + } +} + +fn node_ids(plan: &PhysicalPlan, want: impl Fn(&PhysicalNode) -> bool) -> Vec { + plan.live() + .filter(|(_, node)| want(node)) + .map(|(id, _)| id) + .collect() +} + +const FILTERED: &str = r#"query q($min: I64) { + match { $d: Doc { open: true } $d.year >= $min } + return { $d.slug, $d.year as year } + order { $d.year desc } + limit 2 +}"#; + +const RANKED: &str = r#"query q($q: String) { + match { $d: Doc $d.year > 2000 } + return { $d.slug, bm25($d.title, $q) as score } + order { bm25($d.title, $q), $d.year } + limit 3 +}"#; + +fn filtered() -> Fixture { + Fixture::new(FILTERED, &[("min", Literal::Integer(2001))]) +} + +fn ranked() -> Fixture { + Fixture::new(RANKED, &[("q", Literal::String("graph".into()))]) +} + +#[test] +fn the_planned_plans_are_accepted() { + for fixture in [filtered(), ranked()] { + let accepted = fixture + .accept(fixture.plan()) + .expect("a planned plan is accepted"); + assert_eq!(accepted.scope(), ValidationScope::InvariantsOnly); + } +} + +#[test] +fn a_dropped_conjunct_fails_predicate_retention() { + let fixture = filtered(); + let mut plan = fixture.plan(); + for id in node_ids(&plan, |node| matches!(node, PhysicalNode::Scan { .. })) { + if let Some(PhysicalNode::Scan { spec, .. }) = plan.node_mut(id) { + spec.filter = None; + } + } + for id in node_ids(&plan, |node| matches!(node, PhysicalNode::Filter { .. })) { + if let Some(PhysicalNode::Filter { filters, .. }) = plan.node_mut(id) { + filters.clear(); + } + } + let (check, detail) = fixture.refused(plan); + assert_eq!(check, "predicate retention", "{detail}"); +} + +#[test] +fn a_scan_of_another_binding_fails_binding_identity() { + let fixture = filtered(); + let mut plan = fixture.plan(); + for id in node_ids(&plan, |node| matches!(node, PhysicalNode::Scan { .. })) { + if let Some(PhysicalNode::Scan { spec, .. }) = plan.node_mut(id) { + spec.binding = Some("e".to_string()); + } + } + let (check, _) = fixture.refused(plan); + assert_eq!(check, "binding identity"); +} + +#[test] +fn another_query_argument_fails_search_identity() { + let fixture = ranked(); + let mut plan = fixture.plan(); + for id in node_ids(&plan, |node| node.ranked().is_some()) { + if let Some(PhysicalNode::Scan { + ranked: Some(ranked), + .. + }) = plan.node_mut(id) + { + ranked.query = IRExpr::Literal(Literal::String("other".into())); + } + } + let (check, detail) = fixture.refused(plan); + assert_eq!(check, "search identity", "{detail}"); +} + +#[test] +fn a_capped_bm25_scan_fails_the_row_cut() { + let fixture = ranked(); + let mut plan = fixture.plan(); + for id in node_ids(&plan, |node| node.ranked().is_some()) { + if let Some(PhysicalNode::Scan { + ranked: Some(ranked), + .. + }) = plan.node_mut(id) + { + ranked.fetch = Some(3); + } + } + let (check, detail) = fixture.refused(plan); + assert_eq!(check, "row cut", "{detail}"); +} + +#[test] +fn another_limit_fails_the_row_cut() { + let fixture = filtered(); + let mut plan = fixture.plan(); + let root = plan.root(); + if let Some(PhysicalNode::Limit { rows, .. }) = plan.node_mut(root) { + *rows = 3; + } + let (check, _) = fixture.refused(plan); + assert_eq!(check, "row cut"); +} + +#[test] +fn a_dropped_or_reversed_key_fails_the_order() { + let fixture = ranked(); + for edit in [ + |order_by: &mut Vec| { + order_by.pop(); + }, + |order_by: &mut Vec| { + let last = order_by.last_mut().unwrap(); + last.descending = !last.descending; + }, + |order_by: &mut Vec| { + order_by.remove(0); + }, + ] { + let mut plan = fixture.plan(); + for id in node_ids(&plan, |node| matches!(node, PhysicalNode::Sort { .. })) { + if let Some(PhysicalNode::Sort { order_by, .. }) = plan.node_mut(id) { + edit(order_by); + } + } + let (check, detail) = fixture.refused(plan); + assert_eq!(check, "order", "{detail}"); + } +} + +#[test] +fn a_missing_identity_key_fails_the_order() { + let fixture = filtered(); + let mut plan = fixture.plan(); + for id in node_ids(&plan, |node| matches!(node, PhysicalNode::Sort { .. })) { + if let Some(PhysicalNode::Sort { tiebreak, .. }) = plan.node_mut(id) { + assert_eq!(*tiebreak, vec![ColumnRef::property("d", IDENTITY_MEMBER)]); + tiebreak.clear(); + } + } + let (check, detail) = fixture.refused(plan); + assert_eq!(check, "order", "{detail}"); +} + +#[test] +fn another_alias_or_item_fails_the_projection() { + let fixture = filtered(); + let mut plan = fixture.plan(); + for id in node_ids(&plan, |node| { + matches!(node, PhysicalNode::Projection { .. }) + }) { + if let Some(PhysicalNode::Projection { return_exprs, .. }) = plan.node_mut(id) { + return_exprs[1].alias = Some("age".to_string()); + } + } + let (check, _) = fixture.refused(plan); + assert_eq!(check, "projection"); +} + +#[test] +fn a_score_read_from_another_column_fails_the_projection() { + let fixture = ranked(); + let mut plan = fixture.plan(); + for id in node_ids(&plan, |node| { + matches!(node, PhysicalNode::Projection { .. }) + }) { + if let Some(PhysicalNode::Projection { return_exprs, .. }) = plan.node_mut(id) { + return_exprs[1].expr = IRExpr::PropAccess { + variable: "d".to_string(), + property: "_distance".to_string(), + }; + } + } + let (check, _) = fixture.refused(plan); + assert_eq!(check, "projection"); +} + +#[test] +fn an_exhausted_budget_is_a_resource_outcome() { + let fixture = ranked(); + let constants = Bound(&fixture.params); + let input = AcceptInput { + checked: &fixture.checked, + catalog: &fixture.catalog, + ir: &fixture.ir, + params: &fixture.params, + constants: &constants, + limits: ValidationLimits { + work: 3, + ..ValidationLimits::DEFAULT + }, + }; + assert_eq!( + accept(fixture.plan(), &input).unwrap_err(), + ValidationError::Exhausted { + limit: "work", + value: 3 + } + ); +} diff --git a/crates/omnigraph-planner/tests/bound_plan.rs b/crates/omnigraph-planner/tests/bound_plan.rs index 0e8b8a834..c9de76420 100644 --- a/crates/omnigraph-planner/tests/bound_plan.rs +++ b/crates/omnigraph-planner/tests/bound_plan.rs @@ -10,8 +10,8 @@ use omnigraph_compiler::SYSTEM_COLUMNS_V3; use omnigraph_compiler::ir::{IRExpr, IROp, IROrdering, IRProjection, QueryIR}; use omnigraph_compiler::query::ast::{CompOp, Literal}; use omnigraph_planner::{ - BoundPlan, Bounds, MemorySource, NodeTypeSpec, PhysicalNode, PhysicalPlan, RankKind, RankScope, - TableRef, ValueTable, plan_query, + BOUND_PLAN_VERSION, BoundPlan, Bounds, MemorySource, NodeTypeSpec, PhysicalNode, PhysicalPlan, + RankKind, RankScope, TableRef, ValueTable, plan_query, }; #[path = "support/bounds.rs"] @@ -101,7 +101,7 @@ fn saved_plan_version_refuses_both_legacy_and_future_readers() { values: Default::default(), }; let encoded = serde_json::to_value(&bound).unwrap(); - assert_eq!(encoded["bound_plan_version"], 1); + assert_eq!(encoded["bound_plan_version"], BOUND_PLAN_VERSION); assert!(encoded.get("plan").is_none()); assert!(encoded["body"]["plan"].is_object()); assert_eq!(round_trip(&bound), bound); @@ -109,15 +109,17 @@ fn saved_plan_version_refuses_both_legacy_and_future_readers() { let legacy = encoded["body"].clone(); let error = serde_json::from_value::(legacy).unwrap_err(); assert!(error.to_string().contains("regenerate"), "{error}"); - let mut future = encoded.clone(); - future["bound_plan_version"] = serde_json::json!(2); - let error = serde_json::from_value::(future).unwrap_err(); - assert!( - error - .to_string() - .contains("unsupported bound plan version 2"), - "{error}" - ); + for version in [BOUND_PLAN_VERSION - 1, BOUND_PLAN_VERSION + 1] { + let mut other = encoded.clone(); + other["bound_plan_version"] = serde_json::json!(version); + let error = serde_json::from_value::(other).unwrap_err(); + assert!( + error + .to_string() + .contains(&format!("unsupported bound plan version {version}")), + "{error}" + ); + } #[derive(serde::Deserialize)] #[allow(dead_code)] diff --git a/crates/omnigraph-planner/tests/query_plan/explain.rs b/crates/omnigraph-planner/tests/query_plan/explain.rs index c9f203eb3..bf4cb99f7 100644 --- a/crates/omnigraph-planner/tests/query_plan/explain.rs +++ b/crates/omnigraph-planner/tests/query_plan/explain.rs @@ -277,7 +277,8 @@ fn the_physical_document_prints_gq_orderings_and_no_query_schema() { assert_eq!(sort["fetch"], 10); assert_eq!( sort["properties"]["ordering"], - serde_json::json!(["$c._distance asc", "$c.rank desc"]) + serde_json::json!(["$c._distance asc", "$c.rank desc", "$c.@id asc"]), + "a sort's output order is its whole comparator, identity keys included" ); assert!(sort["properties"].get("schema").is_none()); let projection = &sort["inputs"][0]; From ee3f2858c070462692d53394cd0ead96f500a818 Mon Sep 17 00:00:00 2001 From: Ragnor Comerford Date: Thu, 1 Oct 2026 17:35:14 +0200 Subject: [PATCH 05/29] engine: a BM25 scan filters before scoring only under full full-text coverage Lance scores fragments no full-text segment covers flat, from statistics over the rows its prefilter admits, so a filter applied before scoring changes those rows' BM25 scores. The plan source reads the ranked property's full-text coverage at the pinned dataset version (full, partial or absent), the plan records it in its assumptions, and each ranked scan declares where it applies its eligibility: a bm25 scan before scoring only under full coverage, else after, as Lance's postfilter. Acceptance refuses a bm25 scan that filters before scoring without recorded full coverage. GQT gains the claim `ranked bm25 eligibility `. --- ...es_a_filter_over_an_unindexed_fragment.gqt | 87 +++++++++++++++++++ crates/omnigraph-gqt/src/plan.rs | 64 ++++++++++++-- crates/omnigraph-planner/src/bound.rs | 3 +- crates/omnigraph-planner/src/gate.rs | 11 ++- crates/omnigraph-planner/src/lib.rs | 8 +- crates/omnigraph-planner/src/mirror.rs | 8 +- crates/omnigraph-planner/src/optimizer.rs | 27 +++++- crates/omnigraph-planner/src/physical.rs | 36 +++++++- crates/omnigraph-planner/src/source.rs | 41 +++++++++ .../src/validate/invariants.rs | 46 ++++++++-- crates/omnigraph/src/engine/lower.rs | 2 + crates/omnigraph/src/engine/mod.rs | 1 + crates/omnigraph/src/engine/plan_source.rs | 79 ++++++++++++++++- crates/omnigraph/src/engine/scan.rs | 13 ++- crates/omnigraph/src/engine/search.rs | 3 + crates/omnigraph/src/table_store.rs | 31 ++++++- 16 files changed, 425 insertions(+), 35 deletions(-) create mode 100644 crates/omnigraph-gqt/cases/v2/bm25_score_ignores_a_filter_over_an_unindexed_fragment.gqt diff --git a/crates/omnigraph-gqt/cases/v2/bm25_score_ignores_a_filter_over_an_unindexed_fragment.gqt b/crates/omnigraph-gqt/cases/v2/bm25_score_ignores_a_filter_over_an_unindexed_fragment.gqt new file mode 100644 index 000000000..49cf6ec18 --- /dev/null +++ b/crates/omnigraph-gqt/cases/v2/bm25_score_ignores_a_filter_over_an_unindexed_fragment.gqt @@ -0,0 +1,87 @@ +# issue: none +# notes: A BM25 score is the retrieval's, not the filter's (RFC 0047, "Checked rewrites for exact +# notes: search"). The seed is indexed, so the filter may run before scoring. The insert lands in a +# notes: fragment no full-text segment covers, which Lance scores flat from statistics over the rows +# notes: its prefilter admits; under that partial coverage the plan applies the filter after scoring, +# notes: so `t1` scores the same with and without a filter that excludes `t2`, the other new row. + +--- runner +timeout_ms: 10000 +environments: + - target: omnigraph-engine + storage: local-filesystem + +--- schema +node Doc { + slug: String @key + text: String @index + year: I64 +} + +--- seed +{"type":"Doc","data":{"slug":"d1","text":"graph engines","year":2020}} +{"type":"Doc","data":{"slug":"d2","text":"graph search","year":2021}} +{"type":"Doc","data":{"slug":"d3","text":"databases","year":2019}} + +--- query +query covered() { + match { $d: Doc $d.year >= 2020 } + return { $d.slug, bm25($d.text, "graph") as score } + order { bm25($d.text, "graph") } +} + +--- expect ordered +{"d.slug":"d1","score":0.43445715} +{"d.slug":"d2","score":0.43445715} + +--- expect shape +d.slug: String +score: F32 + +--- expect plan +scan Doc as $d: ranked bm25 eligibility before_scoring +validation scope invariants_only + +--- mutate +query tail() { + insert Doc { slug: "t1", text: "graph", year: 2022 } + insert Doc { slug: "t2", text: "graph graph graph graph graph graph graph graph", year: 1990 } +} +--- expect ok + +--- query +query unfiltered() { + match { $d: Doc } + return { $d.slug, bm25($d.text, "graph") as score } + order { bm25($d.text, "graph") } +} + +--- expect ordered +{"d.slug":"t2","score":0.46573505} +{"d.slug":"d1","score":0.43445715} +{"d.slug":"d2","score":0.43445715} +{"d.slug":"t1","score":0.3903352} + +--- expect shape +d.slug: String +score: F32 + +--- query +query filtered() { + match { $d: Doc $d.year >= 2000 } + return { $d.slug, bm25($d.text, "graph") as score } + order { bm25($d.text, "graph") } +} + +--- expect ordered +{"d.slug":"d1","score":0.43445715} +{"d.slug":"d2","score":0.43445715} +{"d.slug":"t1","score":0.3903352} + +--- expect shape +d.slug: String +score: F32 + +--- expect plan +scan Doc as $d: ranked bm25 eligibility after_scoring +validation scope invariants_only diff --git a/crates/omnigraph-gqt/src/plan.rs b/crates/omnigraph-gqt/src/plan.rs index 41725486a..21297db42 100644 --- a/crates/omnigraph-gqt/src/plan.rs +++ b/crates/omnigraph-gqt/src/plan.rs @@ -14,6 +14,7 @@ //! scan Doc as $d: ranked bm25 //! scan Doc as $d: ranked nearest fetch 10 //! scan Doc as $d: ranked nearest fetch 10 nprobes 20 +//! scan Doc as $d: ranked bm25 eligibility after_scoring //! expand $d Knows $e: mode indexed_scan //! expand $d Knows $e: mode indexed_scan ran csr //! contains join $p.text contains $m.number @@ -42,7 +43,7 @@ use serde_json::Value; use crate::report::Row; -const FORMS: &str = "forms: `scan [ as $var]: columns [a, b]`, `scan [ as $var]: not columns [a, b]`, `scan [ as $var]: filter reads [v.a]`, `scan [ as $var]: no filter`, `scan [ as $var]: access id_lookup`, `hash join $var[ ran ]`, `scan [ as $var]: ranked [ fetch ][ nprobes ]`, `scan [ as $var]: runtime filter `, `scan [ as $var]: no runtime filter`, `contains join $h.x contains $n.y`, `cross join $h.x contains $n.y`, `expand $src $dst: mode [ ran ]`, `filter reads [a.x, b.y]`, `sort tiebreak [$a.@id, $e.@type]`, `rank fuse row tiebreak [$e.@type, $e.@id]`, `expand $a $b: selection alternation [Knows out, Likes in]`, `sort no tiebreak`, `pass `, `not pass `, `validation scope `"; +const FORMS: &str = "forms: `scan [ as $var]: columns [a, b]`, `scan [ as $var]: not columns [a, b]`, `scan [ as $var]: filter reads [v.a]`, `scan [ as $var]: no filter`, `scan [ as $var]: access id_lookup`, `hash join $var[ ran ]`, `scan [ as $var]: ranked [ fetch ][ nprobes ][ eligibility ]`, `scan [ as $var]: runtime filter `, `scan [ as $var]: no runtime filter`, `contains join $h.x contains $n.y`, `cross join $h.x contains $n.y`, `expand $src $dst: mode [ ran ]`, `filter reads [a.x, b.y]`, `sort tiebreak [$a.@id, $e.@type]`, `rank fuse row tiebreak [$e.@type, $e.@id]`, `expand $a $b: selection alternation [Knows out, Likes in]`, `sort no tiebreak`, `pass `, `not pass `, `validation scope `"; const ID_LOOKUP: &str = "id_lookup"; const JOIN_SIDES: [&str; 2] = ["hash_join", "id_lookup"]; @@ -92,6 +93,8 @@ pub(crate) enum PlanLine { index: String, fetch: Option, nprobes: Option, + /// Where the scan applies its eligibility, when claimed. + eligibility: Option, }, /// Every selected physical scan is marked with a runtime filter on /// `column`, or with none when `column` is `None`. @@ -404,18 +407,28 @@ pub(crate) fn parse_plan_body(body: &[(usize, &str)]) -> Result, S .ok_or_else(|| refused("claims `ranked nearest` or `ranked bm25`"))?; let mut fetch = None; let mut nprobes = None; + let mut eligibility = None; while let Some(key) = words.next() { - let count = words.next().and_then(|count| count.parse::().ok()); - match (key, count) { - ("fetch", Some(count)) if fetch.is_none() && nprobes.is_none() => { + let value = words.next(); + let count = value.and_then(|count| count.parse::().ok()); + match (key, count, value) { + ("fetch", Some(count), _) + if fetch.is_none() && nprobes.is_none() && eligibility.is_none() => + { fetch = Some(count); } - ("nprobes", Some(count)) if nprobes.is_none() => { + ("nprobes", Some(count), _) if nprobes.is_none() && eligibility.is_none() => { nprobes = Some(count); } + ("eligibility", _, Some(placement)) + if eligibility.is_none() + && ["before_scoring", "after_scoring"].contains(&placement) => + { + eligibility = Some(placement.to_string()); + } _ => { return Err(refused( - "claims `ranked [ fetch ][ nprobes ]`: each key optional, at most once, in that order, followed by a whole number", + "claims `ranked [ fetch ][ nprobes ][ eligibility ]`: each key optional, at most once, in that order", )); } } @@ -426,6 +439,7 @@ pub(crate) fn parse_plan_body(body: &[(usize, &str)]) -> Result, S index: kind.to_string(), fetch, nprobes, + eligibility, }); continue; } else if let Some(access) = claim.strip_prefix("access ") { @@ -614,6 +628,7 @@ struct PlannedRanking { kind: String, fetch: Option, nprobes: Option, + eligibility: Option, } /// `[ ran ]` over the words `allowed`: the planned choice @@ -723,6 +738,10 @@ fn planned_physical(node: &Value, out: &mut PlannedNodes) { nprobes: ranked .get("nprobes") .map(|nprobes| nprobes.as_u64().unwrap_or(0)), + eligibility: ranked + .get("eligibility") + .and_then(Value::as_str) + .map(str::to_string), }), runtime_filter: node .get("runtime_filter") @@ -1011,6 +1030,7 @@ pub(crate) fn plan_mismatch( index: kind, fetch, nprobes, + eligibility, } => { let selected = match physical_scans(&nodes, type_name, binding.as_deref()) { Ok(selected) => selected, @@ -1051,6 +1071,15 @@ pub(crate) fn plan_mismatch( }); } } + if let Some(eligibility) = eligibility { + let placements: Vec> = + of_kind.iter().map(|r| r.eligibility.as_deref()).collect(); + if !placements.contains(&Some(eligibility.as_str())) { + return Some(format!( + "expect plan: the `{kind}` scans of `{type_name}` apply eligibility {placements:?}, expected {eligibility}" + )); + } + } if let Some(nprobes) = nprobes { let caps: Vec> = of_kind.iter().map(|r| r.nprobes).collect(); if !caps.contains(&Some(*nprobes)) { @@ -1469,6 +1498,27 @@ mod tests { assert!(parse_plan_body(&[(0, "validation scope everything")]).is_err()); } + /// The eligibility claim reads the ranked scan's declared placement. + #[test] + fn eligibility_claims_read_the_ranked_scans_placement() { + let explain = json!({"physical_plan": {"node": "Scan", "table": "node:Doc", "binding": "d", + "ranked": {"kind": "bm25", "fetch": null, "eligibility": "after_scoring"}}}); + let lines = + parse_plan_body(&[(0, "scan Doc as $d: ranked bm25 eligibility after_scoring")]) + .unwrap(); + assert_eq!(check(&lines, &explain), None); + let lines = + parse_plan_body(&[(0, "scan Doc as $d: ranked bm25 eligibility before_scoring")]) + .unwrap(); + let mismatch = check(&lines, &explain).unwrap(); + assert!(mismatch.contains("expected before_scoring"), "{mismatch}"); + assert!(parse_plan_body(&[(0, "scan Doc: ranked bm25 eligibility sometimes")]).is_err()); + assert!( + parse_plan_body(&[(0, "scan Doc: ranked bm25 eligibility after_scoring fetch 1")]) + .is_err() + ); + } + #[test] fn a_read_column_fails_the_negative_form() { let lines = parse_plan_body(&[(0, "scan Doc: not columns [slug]")]).unwrap(); @@ -1899,6 +1949,7 @@ mod tests { index: "nearest".to_string(), fetch: Some(10), nprobes: None, + eligibility: None, } ); assert_eq!(lines[2].clone(), { @@ -1972,6 +2023,7 @@ mod tests { index: "nearest".to_string(), fetch: None, nprobes: Some(20), + eligibility: None, } ); assert_eq!( diff --git a/crates/omnigraph-planner/src/bound.rs b/crates/omnigraph-planner/src/bound.rs index 683c0bf7f..f716fb6ca 100644 --- a/crates/omnigraph-planner/src/bound.rs +++ b/crates/omnigraph-planner/src/bound.rs @@ -17,7 +17,8 @@ use crate::physical::{NodeId, PhysicalPlan}; /// Saved execution plans use a separate version from rendered explain output. /// Version 2: a node's declared ordering is typed order keys (column, /// expression with direction and null placement, identity, fused rank) -/// instead of their text. +/// instead of their text; a ranked scan declares where it applies its +/// eligibility, and the assumptions record full-text coverage. pub const BOUND_PLAN_VERSION: u32 = 2; /// The envelope omits top-level `plan` and `values` so unversioned readers reject it. diff --git a/crates/omnigraph-planner/src/gate.rs b/crates/omnigraph-planner/src/gate.rs index 6d37da49b..6394dc24c 100644 --- a/crates/omnigraph-planner/src/gate.rs +++ b/crates/omnigraph-planner/src/gate.rs @@ -21,7 +21,7 @@ use crate::registry::{Coverage, Entry, Route, coverage, lookup}; use crate::route::RouteOverride; use crate::source::{ AdjacencyProof, EXPAND_INDEXED_MAX_FRONTIER_ENV, EXPAND_INDEXED_MAX_HOPS_ENV, ExpandStatistics, - FragmentStat, NodeTypeSpec, PlanSource, SideId, + FragmentStat, FullTextCoverage, NodeTypeSpec, PlanSource, SideId, }; use crate::validate::{self, AcceptInput, AcceptedPlan, ValidationError}; @@ -160,6 +160,15 @@ impl PlanSource for Recorded<'_> { fn gate_policy(&self) -> GatePolicy { self.source.gate_policy() } + + fn full_text_coverage(&self, type_key: &str, property: &str) -> FullTextCoverage { + let coverage = self.source.full_text_coverage(type_key, property); + self.read + .borrow_mut() + .full_text + .insert(Assumptions::full_text_key(type_key, property), coverage); + coverage + } } /// Every parameter name `expr` names. diff --git a/crates/omnigraph-planner/src/lib.rs b/crates/omnigraph-planner/src/lib.rs index 85b1eda82..b1a3e2df2 100644 --- a/crates/omnigraph-planner/src/lib.rs +++ b/crates/omnigraph-planner/src/lib.rs @@ -58,15 +58,15 @@ pub use lower::{ pub use operation::{Operation, PageBudgetSpec, ScopeSpec, Side, TableRef}; pub use optimizer::{Bounds, physical_plan, rewrite}; pub use physical::{ - Assumptions, DatasetPin, Estimate, GatePolicy, Hop, NodeId, OrderKey, OverfetchRung, - PhysicalNode, PhysicalPlan, Prefilter, PrefilterMode, Properties, RankArm, RankKind, RankScope, - RankedAccess, ScanInput, StatisticSource, + Assumptions, DatasetPin, Eligibility, Estimate, GatePolicy, Hop, NodeId, OrderKey, + OverfetchRung, PhysicalNode, PhysicalPlan, Prefilter, PrefilterMode, Properties, RankArm, + RankKind, RankScope, RankedAccess, ScanInput, StatisticSource, }; pub use registry::{Coverage, Entry, Route, Shape}; pub use route::RouteOverride; pub use source::{ AdjacencyProof, EXPAND_INDEXED_MAX_FRONTIER_ENV, EXPAND_INDEXED_MAX_HOPS_ENV, ExpandStatistics, - FragmentStat, MemorySource, NodeTypeSpec, PlanSource, SideId, + FragmentStat, FullTextCoverage, MemorySource, NodeTypeSpec, PlanSource, SideId, }; pub use validate::{ AcceptInput, AcceptedBoundPlan, AcceptedPlan, ConstantEvaluator, Evidence, ReplayEnvelope, diff --git a/crates/omnigraph-planner/src/mirror.rs b/crates/omnigraph-planner/src/mirror.rs index 480aba05b..8ddacf33d 100644 --- a/crates/omnigraph-planner/src/mirror.rs +++ b/crates/omnigraph-planner/src/mirror.rs @@ -23,8 +23,9 @@ use crate::error::PlanError; use crate::logical::{ColumnRef, GqFilter, KeyJoinKind, Predicate, RuntimeFilterSpec, ScanSpec}; use crate::operation::TableRef; use crate::physical::{ - Assumptions, Estimate, Hop, NodeId, OrderKey, OverfetchRung, PhysicalNode, PhysicalPlan, - Prefilter, Properties, RankArm, RankKind, RankScope, RankedAccess, ScanInput, StatisticSource, + Assumptions, Eligibility, Estimate, Hop, NodeId, OrderKey, OverfetchRung, PhysicalNode, + PhysicalPlan, Prefilter, Properties, RankArm, RankKind, RankScope, RankedAccess, ScanInput, + StatisticSource, }; use crate::source::SideId; @@ -738,6 +739,7 @@ pub struct RankedMirror { pub scope: RankScope, pub overfetch: Vec, pub prefilter: Option, + pub eligibility: Eligibility, } impl From<&RankedAccess> for RankedMirror { @@ -751,6 +753,7 @@ impl From<&RankedAccess> for RankedMirror { scope: ranked.scope, overfetch: ranked.overfetch.clone(), prefilter: ranked.prefilter.as_ref().map(PrefilterMirror::from), + eligibility: ranked.eligibility, } } } @@ -766,6 +769,7 @@ impl From for RankedAccess { scope: mirror.scope, overfetch: mirror.overfetch, prefilter: mirror.prefilter.map(Prefilter::from), + eligibility: mirror.eligibility, } } } diff --git a/crates/omnigraph-planner/src/optimizer.rs b/crates/omnigraph-planner/src/optimizer.rs index da17145d8..209013d3f 100644 --- a/crates/omnigraph-planner/src/optimizer.rs +++ b/crates/omnigraph-planner/src/optimizer.rs @@ -27,11 +27,11 @@ use crate::logical::{ use crate::lower::ContainsJoinFields; use crate::operation::{Operation, Side}; use crate::physical::{ - Assumptions, Estimate, Hop, NodeId, OrderKey, OverfetchRung, PhysicalNode, PhysicalPlan, - Prefilter, Properties, RankArm, RankKind, RankScope, RankedAccess, ScanInput, StatisticSource, - TextContains, + Assumptions, Eligibility, Estimate, Hop, NodeId, OrderKey, OverfetchRung, PhysicalNode, + PhysicalPlan, Prefilter, Properties, RankArm, RankKind, RankScope, RankedAccess, ScanInput, + StatisticSource, TextContains, }; -use crate::source::{NodeTypeSpec, PlanSource, SideId}; +use crate::source::{FullTextCoverage, NodeTypeSpec, PlanSource, SideId}; pub const ROW_ID: &str = "_rowid"; pub const ROW_ADDR: &str = "_rowaddr"; @@ -2036,6 +2036,22 @@ impl Lowering<'_> { ))); } self.unmark(scan); + let mut access = access; + access.eligibility = match access.kind { + RankKind::Nearest => Eligibility::BeforeScoring, + RankKind::Bm25 => { + let type_key = match self.physical.node(scan) { + Some(PhysicalNode::Scan { spec, .. }) => spec.table.type_key.clone(), + _ => String::new(), + }; + match self.source.full_text_coverage(&type_key, &access.property) { + FullTextCoverage::Full => Eligibility::BeforeScoring, + FullTextCoverage::Partial | FullTextCoverage::Absent => { + Eligibility::AfterScoring + } + } + } + }; match self.physical.node_mut(scan) { Some(PhysicalNode::Scan { ranked, .. }) if ranked.is_none() => { *ranked = Some(access); @@ -2497,6 +2513,7 @@ impl Lowering<'_> { scope: RankScope::Order, overfetch: fetch.map(OverfetchRung::ladder).unwrap_or_default(), prefilter: None, + eligibility: Eligibility::BeforeScoring, }; let (score, scan) = self.rank(lowered, binding, access)?; let top = top_level(&self.physical, lowered); @@ -2533,6 +2550,7 @@ impl Lowering<'_> { scope: RankScope::Order, overfetch: Vec::new(), prefilter: None, + eligibility: Eligibility::BeforeScoring, }; let (score, _) = self.rank(lowered, binding, access)?; self.ranking = Some(Ranking::Scores(vec![score])); @@ -2574,6 +2592,7 @@ impl Lowering<'_> { scope, overfetch: Vec::new(), prefilter: None, + eligibility: Eligibility::BeforeScoring, }; let (_, scan) = self.rank(root, &arm.binding, access)?; if arm.kind == RankKind::Bm25 { diff --git a/crates/omnigraph-planner/src/physical.rs b/crates/omnigraph-planner/src/physical.rs index e866f4ea5..3e516e9e0 100644 --- a/crates/omnigraph-planner/src/physical.rs +++ b/crates/omnigraph-planner/src/physical.rs @@ -16,7 +16,7 @@ use crate::logical::{ tiebreak_text, }; use crate::mirror::EdgeSelectionMirror; -use crate::source::SideId; +use crate::source::{FullTextCoverage, SideId}; use omnigraph_compiler::query::codes::{P002, P003}; /// The index of a node in a [`PhysicalPlan`]. @@ -130,6 +130,18 @@ pub struct Assumptions { /// Retained even when rewrites remove an expansion, for historical replay admission. #[serde(default, skip_serializing_if = "is_false")] pub has_wildcard_traversal: bool, + /// The full-text coverage of every property a ranked scan's eligibility + /// placement read, keyed `.`. A pinned dataset + /// version fixes it, so the dataset pins re-establish it on replay. + #[serde(default, skip_serializing_if = "BTreeMap::is_empty")] + pub full_text: BTreeMap, +} + +impl Assumptions { + /// The key a property's full-text coverage is recorded under. + pub fn full_text_key(type_key: &str, property: &str) -> String { + format!("{type_key}.{property}") + } } impl Assumptions { @@ -195,6 +207,19 @@ impl Prefilter { } } +/// Where a ranked scan applies its eligibility (its pushed filter, a gate's +/// eligible set and the members of its search predicates): before the index +/// scores rows, as Lance's prefilter, or after, on the scored rows. A BM25 +/// score is independent of a filter applied before scoring only when the +/// property's full-text index covers every fragment; elsewhere eligibility +/// applies after scoring, so the filter never changes a score. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum Eligibility { + BeforeScoring, + AfterScoring, +} + /// One rerun of a `nearest` scan's overfetch ladder, taken in order after a /// full scan left the answer short of the limit. Report rung `r >= 1` names /// `overfetch[r - 1]`; rung 0 is the scan as planned. @@ -253,6 +278,10 @@ pub struct RankedAccess { /// the ranked binding, and on every arm of `rrf()`, whose pre-pass the /// `RankFuse` declares. pub prefilter: Option, + /// Where the scan applies its eligibility: always before scoring on a + /// `nearest` scan, whose candidate window is drawn from eligible rows; + /// on a `bm25` scan before scoring only under recorded full coverage. + pub eligibility: Eligibility, } impl RankedAccess { @@ -276,8 +305,9 @@ impl RankedAccess { "fetch": self.fetch, "scope": self.scope, }); - if self.kind == RankKind::Nearest { - value["nprobes"] = json!(self.nprobes); + match self.kind { + RankKind::Nearest => value["nprobes"] = json!(self.nprobes), + RankKind::Bm25 => value["eligibility"] = json!(self.eligibility), } value } diff --git a/crates/omnigraph-planner/src/source.rs b/crates/omnigraph-planner/src/source.rs index a101968a1..aa0a2f593 100644 --- a/crates/omnigraph-planner/src/source.rs +++ b/crates/omnigraph-planner/src/source.rs @@ -191,6 +191,28 @@ pub trait PlanSource { fn gate_policy(&self) -> GatePolicy { GatePolicy::default() } + + /// How far the full-text index of `property` covers the pinned table + /// `type_key` names. The default claims no coverage, the answer under + /// which every plan stays correct. + fn full_text_coverage(&self, _type_key: &str, _property: &str) -> FullTextCoverage { + FullTextCoverage::Absent + } +} + +/// How far a property's full-text index covers the pinned dataset's +/// fragments. Only full coverage makes a BM25 score independent of a filter +/// applied before scoring: Lance scores the unindexed fragments flat, from +/// statistics over the rows the filter admits. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum FullTextCoverage { + /// Every fragment of the pinned version is in a full-text segment. + Full, + /// A full-text index exists; some fragment is not in it. + Partial, + /// The property has no full-text index. + Absent, } /// An in-memory [`PlanSource`] for planner tests and registry fixtures. @@ -208,9 +230,21 @@ pub struct MemorySource { table_data_bytes: HashMap, column_data_bytes: HashMap>, query_memory_pool_bytes: u64, + full_text: HashMap<(String, String), FullTextCoverage>, } impl MemorySource { + pub fn with_full_text_coverage( + mut self, + type_key: &str, + property: &str, + coverage: FullTextCoverage, + ) -> Self { + self.full_text + .insert((type_key.to_string(), property.to_string()), coverage); + self + } + pub fn with_query_memory_pool_bytes(mut self, bytes: u64) -> Self { self.query_memory_pool_bytes = bytes; self @@ -349,4 +383,11 @@ impl PlanSource for MemorySource { fn query_memory_pool_bytes(&self) -> u64 { self.query_memory_pool_bytes } + + fn full_text_coverage(&self, type_key: &str, property: &str) -> FullTextCoverage { + self.full_text + .get(&(type_key.to_string(), property.to_string())) + .copied() + .unwrap_or(FullTextCoverage::Absent) + } } diff --git a/crates/omnigraph-planner/src/validate/invariants.rs b/crates/omnigraph-planner/src/validate/invariants.rs index a06fbd210..441516692 100644 --- a/crates/omnigraph-planner/src/validate/invariants.rs +++ b/crates/omnigraph-planner/src/validate/invariants.rs @@ -25,8 +25,10 @@ use crate::logical::{ColumnRef, EDGE_TYPE_MEMBER, IDENTITY_MEMBER}; use crate::lower::ContainsJoinFields; use crate::optimizer::{RRF_NEAREST_ARM_K, derived_order}; use crate::physical::{ - NodeId, OverfetchRung, PhysicalNode, PhysicalPlan, RankKind, RankScope, RankedAccess, + Assumptions, Eligibility, NodeId, OverfetchRung, PhysicalNode, PhysicalPlan, RankKind, + RankScope, RankedAccess, }; +use crate::source::FullTextCoverage; /// The nodes of one pipeline: every node reachable from `root` without /// entering a correlated block's inner tree. @@ -132,9 +134,9 @@ impl Requirements { for nodes in &pipelines { self.check_bindings(plan, nodes)?; self.check_eligibility(plan, nodes, &matcher, budget)?; + self.check_blocks(plan, nodes, &matcher, budget)?; } let top = pipeline(plan, plan.root(), budget)?; - self.check_blocks(plan, &top, &matcher, budget)?; self.check_search(plan, &matcher, budget)?; self.check_returns(plan, &top, &matcher, budget)?; self.check_order_and_cut(plan, &top, &matcher, budget)?; @@ -238,8 +240,9 @@ impl Requirements { Ok(()) } - /// One correlated block of the top level per written block, each with - /// the aggregate and comparison the query wrote. + /// One correlated block per written block in each pipeline the query's + /// rows come from, each with the aggregate and comparison the query + /// wrote. fn check_blocks( &self, plan: &PhysicalPlan, @@ -364,7 +367,15 @@ impl Requirements { ), )); } - self.check_access(plan, access)?; + let type_key = ranked + .iter() + .find(|(_, _, other)| std::ptr::eq(*other, *access)) + .and_then(|(id, _, _)| match plan.node(*id) { + Some(PhysicalNode::Scan { spec, .. }) => Some(spec.table.type_key.as_str()), + _ => None, + }) + .unwrap_or_default(); + self.check_access(plan, type_key, access)?; } match (&self.search, fuse) { (Some(Search::Fuse { arms, k }), Some((planned, planned_k))) => { @@ -416,8 +427,33 @@ impl Requirements { fn check_access( &self, plan: &PhysicalPlan, + type_key: &str, access: &RankedAccess, ) -> Result<(), ValidationError> { + match (access.kind, access.eligibility) { + (RankKind::Nearest, Eligibility::AfterScoring) => { + return Err(ValidationError::violated( + "declared policy", + "a nearest scan draws its candidates from eligible rows; it filters before scoring", + )); + } + (RankKind::Bm25, Eligibility::BeforeScoring) => { + let recorded = plan + .assumptions() + .full_text + .get(&Assumptions::full_text_key(type_key, &access.property)); + if recorded != Some(&FullTextCoverage::Full) { + return Err(ValidationError::violated( + "prerequisite", + format!( + "the bm25 scan of `{type_key}.{}` filters before scoring under recorded coverage {recorded:?}; only full coverage keeps its scores independent of the filter", + access.property + ), + )); + } + } + _ => {} + } let limit = self.limit.and_then(|limit| usize::try_from(limit).ok()); let (fetch, overfetch) = match (access.kind, access.scope) { (RankKind::Bm25, _) => (None, Vec::new()), diff --git a/crates/omnigraph/src/engine/lower.rs b/crates/omnigraph/src/engine/lower.rs index 5648a1313..0782621b9 100644 --- a/crates/omnigraph/src/engine/lower.rs +++ b/crates/omnigraph/src/engine/lower.rs @@ -170,6 +170,8 @@ impl<'a> Lowering<'a> { property: ranked.property.clone(), text, }); + mode.eligibility_after_scoring = + ranked.eligibility == omnigraph_planner::Eligibility::AfterScoring; } } Ok(mode) diff --git a/crates/omnigraph/src/engine/mod.rs b/crates/omnigraph/src/engine/mod.rs index 76f1aabe5..d605d6b40 100644 --- a/crates/omnigraph/src/engine/mod.rs +++ b/crates/omnigraph/src/engine/mod.rs @@ -747,6 +747,7 @@ mod traversal_admission_tests { scope: RankScope::Order, overfetch: vec![], prefilter: None, + eligibility: omnigraph_planner::Eligibility::BeforeScoring, }), }; validate_traversal_admission(&plan).unwrap(); diff --git a/crates/omnigraph/src/engine/plan_source.rs b/crates/omnigraph/src/engine/plan_source.rs index 063ff49cb..8d734e4c9 100644 --- a/crates/omnigraph/src/engine/plan_source.rs +++ b/crates/omnigraph/src/engine/plan_source.rs @@ -18,8 +18,9 @@ use omnigraph_compiler::types::Direction; use omnigraph_planner::{ AcceptInput, AcceptedBoundPlan, AcceptedPlan, AdjacencyProof, Bounds, ConstantEvaluator, DatasetPin, EXPAND_INDEXED_MAX_FRONTIER_ENV, EXPAND_INDEXED_MAX_HOPS_ENV, ExpandStatistics, - Explain, FragmentStat, GatePolicy, NodeTypeSpec, Operation, PlanError, PlanSource, - PrefilterMode, ReplayEnvelope, ReplayRefusal, SideId, TableRef, Unrouted, ValidationLimits, + Explain, FragmentStat, FullTextCoverage, GatePolicy, NodeTypeSpec, Operation, PlanError, + PlanSource, PrefilterMode, ReplayEnvelope, ReplayRefusal, SideId, TableRef, Unrouted, + ValidationLimits, }; use super::ResolvedParams; @@ -105,6 +106,9 @@ pub(crate) struct QuerySource<'a> { gate_policy: GatePolicy, expand_caps: ExpandCaps, table_stats: HashMap, + /// The full-text coverage of every property a `bm25()` order key ranks, + /// by `(table key, property)`, read at the pinned dataset version. + full_text: HashMap<(String, String), FullTextCoverage>, } struct TableStatistics { @@ -126,6 +130,7 @@ impl<'a> QuerySource<'a> { let params = resolve_params(&query.ir, params)?; let ir = super::constant::fold_query_constants(&query.ir, params.shared())?; let table_stats = destination_table_statistics(&ir, snapshot).await?; + let full_text = ranked_full_text_coverage(&ir, snapshot).await?; let mut source = QuerySource { ir, checked: query.checked.clone(), @@ -137,6 +142,7 @@ impl<'a> QuerySource<'a> { gate_policy: gate_policy(settings), expand_caps: ExpandCaps::from_env(), table_stats, + full_text, }; source .load_column_statistics(&Operation::Query(Box::new(source.ir.clone()))) @@ -196,6 +202,13 @@ impl<'a> QuerySource<'a> { } impl PlanSource for QuerySource<'_> { + fn full_text_coverage(&self, type_key: &str, property: &str) -> FullTextCoverage { + self.full_text + .get(&(type_key.to_string(), property.to_string())) + .copied() + .unwrap_or(FullTextCoverage::Absent) + } + fn traversal_work_limit(&self) -> Option { self.ir .has_edge_selections() @@ -505,6 +518,68 @@ pub(crate) fn explain_query(source: &QuerySource<'_>) -> Result Ok(ExplainedQuery { explain, accepted }) } +/// The full-text coverage of every property the leading `order` key ranks +/// by `bm25()` (alone or as an `rrf()` arm), at the snapshot's pinned +/// version: the fact that decides where the ranked scan applies its +/// eligibility. +async fn ranked_full_text_coverage( + ir: &QueryIR, + snapshot: &Snapshot, +) -> Result> { + fn bm25_targets(expr: &IRExpr, out: &mut Vec<(String, String)>) { + match expr { + IRExpr::Bm25 { field, .. } => { + if let IRExpr::PropAccess { variable, property } = field.as_ref() { + out.push((variable.clone(), property.clone())); + } + } + IRExpr::Rrf { + primary, secondary, .. + } => { + bm25_targets(primary, out); + bm25_targets(secondary, out); + } + _ => {} + } + } + fn binding_type(ops: &[IROp], binding: &str) -> Option { + ops.iter().find_map(|op| match op { + IROp::NodeScan { + variable, + type_name, + .. + } if variable == binding => Some(type_name.clone()), + IROp::Expand { + dst_var, dst_type, .. + } if dst_var == binding => Some(dst_type.clone()), + _ => None, + }) + } + let mut targets = Vec::new(); + if let Some(leading) = ir.order_by.first() { + bm25_targets(&leading.expr, &mut targets); + } + let mut coverage = HashMap::new(); + for (binding, property) in targets { + let Some(type_name) = binding_type(&ir.pipeline, &binding) else { + continue; + }; + let type_key = format!("node:{type_name}"); + if coverage.contains_key(&(type_key.clone(), property.clone())) { + continue; + } + let known = match snapshot.dataset(&type_key) { + Some(_) => { + let dataset = snapshot.open_lance_dataset(&type_key).await?; + crate::table_store::TableStore::fts_coverage(&dataset, &property).await? + } + None => FullTextCoverage::Absent, + }; + coverage.insert((type_key, property), known); + } + Ok(coverage) +} + async fn destination_table_statistics( ir: &QueryIR, snapshot: &Snapshot, diff --git a/crates/omnigraph/src/engine/scan.rs b/crates/omnigraph/src/engine/scan.rs index a09a52f61..7a1b302cc 100644 --- a/crates/omnigraph/src/engine/scan.rs +++ b/crates/omnigraph/src/engine/scan.rs @@ -45,6 +45,9 @@ pub(super) struct NodeRead<'n> { /// The gate proved the answer empty, or a BM25 filter matched no row: no /// Lance read runs. pub(super) proven_empty: bool, + /// The pushed filter runs after the full-text search scores rows + /// (Lance's postfilter) instead of before it. + filter_after_scoring: bool, } impl<'n> NodeRead<'n> { @@ -142,12 +145,15 @@ impl<'n> NodeRead<'n> { fts_query, columns, proven_empty, + filter_after_scoring: search_mode.eligibility_after_scoring, }) } /// The Lance plan of this read: the projection, the pushed filter as a - /// prefilter, the `(rows, bytes)` batch override and bounded readahead of - /// a pipelined read, the full-text query, then `configure` (nearest). + /// prefilter (as a postfilter on the scored rows where the plan places a + /// BM25 scan's eligibility after scoring), the `(rows, bytes)` batch + /// override and bounded readahead of a pipelined read, the full-text + /// query, then `configure` (nearest). pub(super) fn plan( &self, pipelined_batch: Option<(usize, usize)>, @@ -157,10 +163,11 @@ impl<'n> NodeRead<'n> { Result>, > { let projection = self.columns.read_projection(); + let filter_after_scoring = self.filter_after_scoring && self.fts_query.is_some(); TableStore::scan_plan_with(&self.ds, projection.as_deref(), None, false, |scanner| { if let Some(expr) = &self.filter_expr { scanner.filter_expr(expr.clone()); - scanner.prefilter(true); + scanner.prefilter(!filter_after_scoring); } if let Some((rows, bytes)) = pipelined_batch { scanner.batch_size(rows); diff --git a/crates/omnigraph/src/engine/search.rs b/crates/omnigraph/src/engine/search.rs index 79f978254..1db1cf4de 100644 --- a/crates/omnigraph/src/engine/search.rs +++ b/crates/omnigraph/src/engine/search.rs @@ -28,6 +28,9 @@ pub(super) struct SearchMode { pub(super) answer_proven_empty: bool, /// BM25 full-text search on the scan's binding. pub(super) bm25: Option, + /// The plan places the scan's eligibility after scoring: its filter + /// runs as Lance's postfilter, so BM25 statistics read every row. + pub(super) eligibility_after_scoring: bool, /// The set a gate ANDs into the ranked scan as `id IN (...)`, read off the /// `Pass` by `Lowering::search_mode`; the arms an `rrf()` prefilter feeds /// are the plan's `Prefilter.feeds`. diff --git a/crates/omnigraph/src/table_store.rs b/crates/omnigraph/src/table_store.rs index ee6441539..66c0c40fd 100644 --- a/crates/omnigraph/src/table_store.rs +++ b/crates/omnigraph/src/table_store.rs @@ -4749,8 +4749,20 @@ impl TableStore { /// runs the postfilter plan — so every unprovable case (no FTS index, /// an entry without a fragment bitmap) contributes nothing. pub(crate) async fn fts_covers_all_fragments(ds: &Dataset, column: &str) -> Result { + Ok(Self::fts_coverage(ds, column).await? == omnigraph_planner::FullTextCoverage::Full) + } + + /// How far `column`'s full-text index covers `ds`'s fragments: every + /// fragment (the union of the entries' fragment bitmaps holds them all), + /// some (an entry exists, but the bitmaps do not prove every fragment), + /// or no full-text entry at all. + pub(crate) async fn fts_coverage( + ds: &Dataset, + column: &str, + ) -> Result { + use omnigraph_planner::FullTextCoverage; let indices = user_indices_for_column(ds, column).await?; - let fts_bitmaps: Vec<_> = indices + let fts_entries: Vec<_> = indices .iter() .filter(|index| { index @@ -4759,16 +4771,27 @@ impl TableStore { .map(|details| IndexDetails(details.clone()).supports_fts()) .unwrap_or(false) }) + .collect(); + if fts_entries.is_empty() { + return Ok(FullTextCoverage::Absent); + } + let fts_bitmaps: Vec<_> = fts_entries + .iter() .filter_map(|index| index.fragment_bitmap.as_ref()) .collect(); if fts_bitmaps.is_empty() { - return Ok(false); + return Ok(FullTextCoverage::Partial); } - Ok(ds.fragments().iter().all(|f| { + let covered = ds.fragments().iter().all(|f| { fts_bitmaps .iter() .any(|bitmap| bitmap.contains(f.id as u32)) - })) + }); + Ok(if covered { + FullTextCoverage::Full + } else { + FullTextCoverage::Partial + }) } pub async fn has_vector_index(&self, ds: &Dataset, column: &str) -> Result { From 95a6b9a3a5ff30c3e2baf74096ed72103550bb08 Mon Sep 17 00:00:00 2001 From: Ragnor Comerford Date: Thu, 1 Oct 2026 17:51:05 +0200 Subject: [PATCH 06/29] planner: check the exact fragment's derivation from its canonical form A query of one binding with Boolean, integer and String eligibility, an optional leading bm25() over a String property, direct property sort keys, a limit and a projection of properties and the selected score is a member of the exact fragment. The optimizer records the rules it applies to such a chain (absorb a conjunct into the scan, prune the scan's columns, lower each node, rank the scan by bm25 with its eligibility placement); the validator checks that the IR is the declaration's lowering, rebuilds the canonical chain, re-applies every rule against its precondition and requires the result to equal the plan. A member is accepted as exact_subset, everything else as invariants_only. Replay envelopes carry the derivation, and replay checks it again. --- ...es_a_filter_over_an_unindexed_fragment.gqt | 4 +- crates/omnigraph-planner/src/gate.rs | 54 +- crates/omnigraph-planner/src/optimizer.rs | 85 +- crates/omnigraph-planner/src/validate/mod.rs | 52 +- .../omnigraph-planner/src/validate/replay.rs | 17 +- .../omnigraph-planner/src/validate/subset.rs | 1230 +++++++++++++++++ .../omnigraph-planner/src/validate/tests.rs | 144 +- .../omnigraph/tests/engine_v2_plan_replay.rs | 58 + 8 files changed, 1603 insertions(+), 41 deletions(-) create mode 100644 crates/omnigraph-planner/src/validate/subset.rs diff --git a/crates/omnigraph-gqt/cases/v2/bm25_score_ignores_a_filter_over_an_unindexed_fragment.gqt b/crates/omnigraph-gqt/cases/v2/bm25_score_ignores_a_filter_over_an_unindexed_fragment.gqt index 49cf6ec18..6dbc4e9c9 100644 --- a/crates/omnigraph-gqt/cases/v2/bm25_score_ignores_a_filter_over_an_unindexed_fragment.gqt +++ b/crates/omnigraph-gqt/cases/v2/bm25_score_ignores_a_filter_over_an_unindexed_fragment.gqt @@ -40,7 +40,7 @@ score: F32 --- expect plan scan Doc as $d: ranked bm25 eligibility before_scoring -validation scope invariants_only +validation scope exact_subset --- mutate query tail() { @@ -84,4 +84,4 @@ score: F32 --- expect plan scan Doc as $d: ranked bm25 eligibility after_scoring -validation scope invariants_only +validation scope exact_subset diff --git a/crates/omnigraph-planner/src/gate.rs b/crates/omnigraph-planner/src/gate.rs index 6394dc24c..c4df3efed 100644 --- a/crates/omnigraph-planner/src/gate.rs +++ b/crates/omnigraph-planner/src/gate.rs @@ -15,7 +15,9 @@ use crate::error::PlanError; use crate::explain::{EntrySummary, Explain, OperationSummary}; use crate::logical::{Census, LogicalPlan}; use crate::operation::Operation; -use crate::optimizer::{Bounds, Optimized, physical_plan, resolve, rewrite}; +use crate::optimizer::{ + Bounds, Optimized, physical_plan, physical_plan_with, resolve, rewrite, rewrite_with, +}; use crate::physical::{Assumptions, DatasetPin, GatePolicy, NodeId, PhysicalNode, PhysicalPlan}; use crate::registry::{Coverage, Entry, Route, coverage, lookup}; use crate::route::RouteOverride; @@ -23,6 +25,7 @@ use crate::source::{ AdjacencyProof, EXPAND_INDEXED_MAX_FRONTIER_ENV, EXPAND_INDEXED_MAX_HOPS_ENV, ExpandStatistics, FragmentStat, FullTextCoverage, NodeTypeSpec, PlanSource, SideId, }; +use crate::validate::subset::{Derivation, Tracer}; use crate::validate::{self, AcceptInput, AcceptedPlan, ValidationError}; /// A `PlanSource` that records what the planner read through it: the @@ -374,22 +377,53 @@ pub fn accept_query_explained( Ok((accepted, explain)) } -fn plan_and_accept( - input: &AcceptInput<'_>, +/// One traced planning run of a read query: the rewritten logical plan, the +/// optimized physical plan with its assumptions, the passes that fired, and +/// the derivation the optimizer recorded for an exact-fragment chain. +pub(crate) struct Traced { + pub logical: LogicalPlan, + pub optimized: Optimized, + pub fired: Vec<&'static str>, + pub derivation: Option, +} + +pub(crate) fn plan_traced( + query: &QueryIR, source: &dyn PlanSource, bounds: &Bounds, - explain: bool, -) -> Result<(AcceptedPlan, Option), Unrouted> { - let query = input.ir; +) -> Result { let operation = Operation::Query(Box::new(query.clone())); let recorded = Recorded::new(source, query.has_wildcard_traversal()); let mut logical = resolve(&operation, &recorded).map_err(Unrouted::of)?; - let fired = rewrite(&mut logical, &recorded).map_err(Unrouted::of)?; + let mut tracer = Tracer::for_plan(&logical); + let fired = rewrite_with(&mut logical, &recorded, &mut tracer).map_err(Unrouted::of)?; let mut optimized = - physical_plan(&mut logical, &recorded, bounds, fired.clone()).map_err(Unrouted::of)?; + physical_plan_with(&mut logical, &recorded, bounds, fired.clone(), &mut tracer) + .map_err(Unrouted::of)?; optimized .physical .set_assumptions(recorded.assumptions(bounds)); + Ok(Traced { + logical, + optimized, + fired, + derivation: tracer.finish(), + }) +} + +fn plan_and_accept( + input: &AcceptInput<'_>, + source: &dyn PlanSource, + bounds: &Bounds, + explain: bool, +) -> Result<(AcceptedPlan, Option), Unrouted> { + let operation = Operation::Query(Box::new(input.ir.clone())); + let Traced { + logical, + optimized, + fired, + derivation, + } = plan_traced(input.ir, source, bounds)?; let rendered = explain.then(|| { LogicalView::of(&logical) .explain( @@ -401,8 +435,8 @@ fn plan_and_accept( ) .engine() }); - let accepted = - validate::accept(optimized.physical, input).map_err(ValidationError::into_unrouted)?; + let accepted = validate::accept(optimized.physical, input, derivation) + .map_err(ValidationError::into_unrouted)?; let rendered = rendered.map(|mut explain| { explain.pipelines = None; explain.validation = Some(accepted.summary()); diff --git a/crates/omnigraph-planner/src/optimizer.rs b/crates/omnigraph-planner/src/optimizer.rs index 209013d3f..227a24f32 100644 --- a/crates/omnigraph-planner/src/optimizer.rs +++ b/crates/omnigraph-planner/src/optimizer.rs @@ -32,6 +32,7 @@ use crate::physical::{ StatisticSource, TextContains, }; use crate::source::{FullTextCoverage, NodeTypeSpec, PlanSource, SideId}; +use crate::validate::subset::{Role, Rule, Tracer}; pub const ROW_ID: &str = "_rowid"; pub const ROW_ADDR: &str = "_rowaddr"; @@ -447,7 +448,7 @@ fn schema_of(plan: &LogicalPlan, id: LogicalId) -> Result } /// `filters` as one `Filter` node over `input`, its conjunct list in written -/// order; `input` itself when there is none. Where each conjunct runs is the +/// order, a repeated conjunct kept once; `input` itself when there is none. Where each conjunct runs is the /// placement pass's decision, not this builder's. fn filter_over( plan: &mut LogicalPlan, @@ -455,11 +456,12 @@ fn filter_over( filters: &[IRExpr], schema: SchemaRef, ) -> LogicalId { - let conjuncts: Vec = filters - .iter() - .cloned() - .flat_map(IRExpr::into_conjuncts) - .collect(); + let mut conjuncts: Vec = Vec::new(); + for conjunct in filters.iter().cloned().flat_map(IRExpr::into_conjuncts) { + if !conjuncts.contains(&conjunct) { + conjuncts.push(conjunct); + } + } if conjuncts.is_empty() { input } else { @@ -1022,11 +1024,20 @@ pub fn optimize( pub fn rewrite( plan: &mut LogicalPlan, source: &dyn PlanSource, +) -> Result, PlanError> { + rewrite_with(plan, source, &mut Tracer::default()) +} + +/// [`rewrite`], recording the rules it applies to an exact-fragment chain. +pub(crate) fn rewrite_with( + plan: &mut LogicalPlan, + source: &dyn PlanSource, + tracer: &mut Tracer, ) -> Result, PlanError> { let mut fired = vec![PASS_RESOLVE]; let query = is_query_plan(plan); let pushed = if query { - place_query_filters(plan, source) + place_query_filters(plan, source, tracer) } else { resume_pushdown(plan) }; @@ -1036,7 +1047,7 @@ pub fn rewrite( if query && aggregate_pushdown(plan)? { fired.push(PASS_AGGREGATE_PUSHDOWN); } - if query && projection_pushdown(plan, source)? { + if query && projection_pushdown(plan, source, tracer)? { fired.push(PASS_PROJECTION_PUSHDOWN); } Ok(fired) @@ -1055,10 +1066,22 @@ fn is_query_plan(plan: &LogicalPlan) -> bool { /// Stages 2 and 3: physical selection and property derivation over a /// rewritten plan; `fired` carries stage 1's record forward. pub fn physical_plan( + plan: &mut LogicalPlan, + source: &dyn PlanSource, + bounds: &Bounds, + fired: Vec<&'static str>, +) -> Result { + physical_plan_with(plan, source, bounds, fired, &mut Tracer::default()) +} + +/// [`physical_plan`], recording the rules it applies to an exact-fragment +/// chain. +pub(crate) fn physical_plan_with( plan: &mut LogicalPlan, source: &dyn PlanSource, bounds: &Bounds, mut fired: Vec<&'static str>, + tracer: &mut Tracer, ) -> Result { let query = is_query_plan(plan); let before = plan @@ -1084,8 +1107,11 @@ pub fn physical_plan( decisions: Vec::new(), ranking: None, limit: None, + tracer: std::mem::take(tracer), }; - let root = lowering.lower(plan.root())?; + let root = lowering.lower(plan.root()); + *tracer = std::mem::take(&mut lowering.tracer); + let root = root?; let Lowering { mut physical, late_materialization, @@ -1282,7 +1308,11 @@ fn and_filter(existing: Option, added: Predicate) -> Predicate { /// Stage 1, pass 2 on a query plan, per scope (the top-level tree and each /// `not { … }` inner tree): adjacent `Filter` nodes coalesce, each conjunct moves /// into its `placement_target` scan or onto its `join_target`, an emptied node goes. -fn place_query_filters(plan: &mut LogicalPlan, source: &dyn PlanSource) -> bool { +fn place_query_filters( + plan: &mut LogicalPlan, + source: &dyn PlanSource, + tracer: &mut Tracer, +) -> bool { let mut fired = false; let mut scopes = vec![plan.root()]; while let Some(root) = scopes.pop() { @@ -1328,6 +1358,7 @@ fn place_query_filters(plan: &mut LogicalPlan, source: &dyn PlanSource) -> bool match target.and_then(|target| plan.node_mut(target)) { Some(LogicalNode::TableScan { spec, .. }) => { spec.filter = Some(and_filter(spec.filter.take(), gq_conjunct(&conjunct))); + tracer.absorb(&conjunct); fired = true; } _ => match join_target(plan, input, &conjunct) { @@ -1643,7 +1674,11 @@ fn metadata_count_schema(return_exprs: &[IRProjection]) -> Result Result { +fn projection_pushdown( + plan: &mut LogicalPlan, + source: &dyn PlanSource, + tracer: &mut Tracer, +) -> Result { let scans = plan .live() .filter_map(|(id, node)| match node { @@ -1701,6 +1736,13 @@ fn projection_pushdown(plan: &mut LogicalPlan, source: &dyn PlanSource) -> Resul }) .map(str::to_string) .collect(); + tracer.record( + &[Role::Scan], + Rule::PruneScanColumns { + columns: projection.clone(), + }, + &[], + ); match plan.node_mut(id) { Some(LogicalNode::TableScan { spec, .. }) => { spec.projection = Some(projection); @@ -1983,6 +2025,8 @@ struct Lowering<'a> { ranking: Option, /// The `limit` above the node being lowered: the fetch of the score sort. limit: Option, + /// The record of the rules lowering applies to an exact-fragment chain. + tracer: Tracer, } /// What a leading search function became in the physical plan: the score @@ -2343,6 +2387,7 @@ impl Lowering<'_> { ranked: None, }; let Some(input) = input else { + self.tracer.record(&[Role::Scan], Rule::Lower, &[]); return Ok(self.physical.add(scan(ScanInput::Table))); }; let access = self.access_path(*input, spec)?; @@ -2378,6 +2423,7 @@ impl Lowering<'_> { let Some(order_by) = self.sort_keys(lowered, order_by) else { return Ok(lowered); }; + self.tracer.record(&[Role::Sort], Rule::Lower, &[]); Ok(self.physical.add(PhysicalNode::Sort { input: lowered, order_by, @@ -2403,6 +2449,7 @@ impl Lowering<'_> { return self.filtered_cross_join(left_lowered, right_lowered, conjuncts); } let lowered = self.lower(*input)?; + self.tracer.record(&[Role::Filter], Rule::Lower, &[]); Ok(self.physical.add(PhysicalNode::Filter { input: lowered, filters: conjuncts.clone(), @@ -2414,6 +2461,7 @@ impl Lowering<'_> { .. } => { let lowered = self.lower(*input)?; + self.tracer.record(&[Role::Projection], Rule::Lower, &[]); Ok(self.physical.add(PhysicalNode::Projection { input: lowered, return_exprs: return_exprs.clone(), @@ -2552,7 +2600,19 @@ impl Lowering<'_> { prefilter: None, eligibility: Eligibility::BeforeScoring, }; - let (score, _) = self.rank(lowered, binding, access)?; + let (score, scan) = self.rank(lowered, binding, access)?; + if let Some(PhysicalNode::Scan { + ranked: Some(ranked), + .. + }) = self.physical.node(scan) + { + let eligibility = ranked.eligibility; + self.tracer.record( + &[Role::Search, Role::Scan], + Rule::RankBm25Scan { eligibility }, + &[Role::Search], + ); + } self.ranking = Some(Ranking::Scores(vec![score])); Ok(lowered) } @@ -2644,6 +2704,7 @@ impl Lowering<'_> { LogicalNode::Limit { input, rows } => { self.limit = Some(*rows); let lowered = self.lower(*input)?; + self.tracer.record(&[Role::Limit], Rule::Lower, &[]); Ok(self.physical.add(PhysicalNode::Limit { input: lowered, rows: *rows, diff --git a/crates/omnigraph-planner/src/validate/mod.rs b/crates/omnigraph-planner/src/validate/mod.rs index 3006b283f..bcbea2678 100644 --- a/crates/omnigraph-planner/src/validate/mod.rs +++ b/crates/omnigraph-planner/src/validate/mod.rs @@ -16,6 +16,7 @@ mod budget; mod invariants; mod replay; mod requirements; +pub(crate) mod subset; pub use budget::{Budget, ValidationLimits}; pub use replay::{ @@ -23,6 +24,7 @@ pub use replay::{ accept_replay, catalog_digest, decode_replay, }; pub use requirements::{ConstantEvaluator, Requirements}; +pub use subset::{ChainNode, ChainRanking, Derivation, Rule, Step}; use omnigraph_compiler::CheckedQuery; use omnigraph_compiler::catalog::Catalog; @@ -116,13 +118,14 @@ pub struct AcceptInput<'a> { pub limits: ValidationLimits, } -/// The internal record of one acceptance: its scope and the limits it ran -/// under. Inspection and replay read it; ordinary responses carry only the -/// scope. -#[derive(Debug, Clone, PartialEq, Eq)] +/// The internal record of one acceptance: its scope, the limits it ran +/// under and, for a member of the exact fragment, the checked derivation. +/// Inspection and replay read it; ordinary responses carry only the scope. +#[derive(Debug, Clone, PartialEq)] pub struct Evidence { scope: ValidationScope, limits: ValidationLimits, + derivation: Option, } impl Evidence { @@ -133,6 +136,10 @@ impl Evidence { pub fn limits(&self) -> ValidationLimits { self.limits } + + pub fn derivation(&self) -> Option<&Derivation> { + self.derivation.as_ref() + } } /// A physical plan the validator accepted, with its evidence. The fields are @@ -197,23 +204,50 @@ impl AcceptedBoundPlan { } } -/// Validate `plan`, built for `input`, and wrap it. Every check reads the -/// requirements the checked declaration states and the plan alone. +/// Validate `plan`, built for `input` with the optimizer's `derivation`, and +/// wrap it. Every check reads the requirements the checked declaration +/// states, the plan, and for a member of the exact fragment the derivation. pub(crate) fn accept( plan: PhysicalPlan, input: &AcceptInput<'_>, + derivation: Option, ) -> Result { let mut budget = Budget::new(input.limits); - let requirements = Requirements::derive(input, &mut budget)?; - requirements.check(&plan, input, &mut budget)?; + let (scope, derivation) = check(&plan, input, derivation, &mut budget)?; Ok(AcceptedPlan { plan, evidence: Evidence { - scope: ValidationScope::InvariantsOnly, + scope, limits: input.limits, + derivation, }, }) } +/// The checks of one acceptance, shared by fresh planning and replay: the +/// invariants for every plan, and for a member of the exact fragment its +/// derivation, which a member must carry. +pub(crate) fn check( + plan: &PhysicalPlan, + input: &AcceptInput<'_>, + derivation: Option, + budget: &mut Budget, +) -> Result<(ValidationScope, Option), ValidationError> { + let requirements = Requirements::derive(input, budget)?; + requirements.check(plan, input, budget)?; + let Some(member) = subset::membership(input) else { + return Ok((ValidationScope::InvariantsOnly, None)); + }; + let derivation = derivation.ok_or_else(|| { + ValidationError::violated( + "exact subset", + "the query is a member of the exact fragment; the plan carries no derivation", + ) + })?; + budget.visit(u64::try_from(derivation.steps.len()).unwrap_or(u64::MAX))?; + subset::check(&derivation, plan, input, &requirements, &member, budget)?; + Ok((ValidationScope::ExactSubset, Some(derivation))) +} + #[cfg(test)] mod tests; diff --git a/crates/omnigraph-planner/src/validate/replay.rs b/crates/omnigraph-planner/src/validate/replay.rs index 2c9bc259a..bd0ea6f94 100644 --- a/crates/omnigraph-planner/src/validate/replay.rs +++ b/crates/omnigraph-planner/src/validate/replay.rs @@ -12,9 +12,10 @@ use omnigraph_compiler::catalog::Catalog; use serde::{Deserialize, Serialize}; use super::budget::Budget; -use super::requirements::Requirements; +use super::subset::Derivation; use super::{ AcceptInput, AcceptedBoundPlan, Evidence, ValidationError, ValidationLimits, ValidationScope, + check, }; use crate::bound::BoundPlan; @@ -52,6 +53,9 @@ pub struct ReplayEnvelope { /// different schema is refused as incompatible facts. pub catalog: Option, pub plan: BoundPlan, + /// The checked derivation of a member of the exact fragment. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub derivation: Option, } /// The versions of an envelope, read before anything else is decoded. @@ -83,6 +87,7 @@ impl ReplayEnvelope { scope: evidence.scope(), catalog, plan, + derivation: evidence.derivation().cloned(), } } @@ -185,9 +190,12 @@ pub fn accept_replay( }); } let mut budget = Budget::new(input.limits); - let requirements = Requirements::derive(input, &mut budget)?; - requirements.check(&envelope.plan.plan, input, &mut budget)?; - let scope = ValidationScope::InvariantsOnly; + let (scope, derivation) = check( + &envelope.plan.plan, + input, + envelope.derivation.clone(), + &mut budget, + )?; if scope != envelope.scope { return Err(ReplayRefusal::InvalidEvidence { reason: format!( @@ -202,6 +210,7 @@ pub fn accept_replay( evidence: Evidence { scope, limits: input.limits, + derivation, }, }) } diff --git a/crates/omnigraph-planner/src/validate/subset.rs b/crates/omnigraph-planner/src/validate/subset.rs new file mode 100644 index 000000000..ae120efea --- /dev/null +++ b/crates/omnigraph-planner/src/validate/subset.rs @@ -0,0 +1,1230 @@ +//! The exact fragment and its checked derivation (RFC 0047, "Checked +//! rewrites for exact search"). Membership is a syntax-and-type check over +//! the checked declaration: one binding of one node type, eligibility over +//! Boolean, integer and String properties, literals and parameters, an +//! optional leading `bm25()` over a String property, direct property sort +//! keys, a limit, and a projection of properties and the selected score. +//! +//! For a member, the validator first checks that the IR is the lowering of +//! the declaration, then builds the canonical chain from it +//! (`Limit(n, Sort(K, Project(V, [Search(a)] Filter(p, Scan(C)))))`, leaf +//! first), applies the optimizer's recorded rule applications one by one, +//! each checked against its precondition and reconstructed by the validator, +//! and requires the result to equal the candidate plan node for node. A +//! derivation records rule ids, node references and typed substitutions; +//! the successors are the validator's to reconstruct, never the optimizer's +//! to assert. The rule catalogue is closed and versioned +//! ([`super::RULES_VERSION`]). + +use std::collections::BTreeSet; + +use omnigraph_compiler::catalog::NodeType; +use omnigraph_compiler::ir::{IRExpr, IROp, IROrdering, IRProjection, QueryIR}; +use omnigraph_compiler::query::ast::{BinaryOp, Clause, CompOp, Expr, Literal}; +use omnigraph_compiler::types::{PropType, ScalarType}; +use serde::{Deserialize, Serialize}; + +use super::budget::Budget; +use super::requirements::{Matcher, Position, Requirements}; +use super::{AcceptInput, ValidationError}; +use crate::logical::{ColumnRef, IDENTITY_MEMBER, LogicalNode, LogicalPlan}; +use crate::mirror::{ExprMirror, OrderingMirror, ProjectionMirror}; +use crate::physical::{Eligibility, PhysicalNode, PhysicalPlan, RankKind, RankScope, ScanInput}; +use crate::source::FullTextCoverage; + +/// A member of the exact fragment: its one binding and node type. +#[derive(Debug, Clone, PartialEq, Eq)] +pub(crate) struct Member { + pub binding: String, + pub type_name: String, +} + +/// The scalar types the fragment's comparisons and keys admit. +fn admitted(property: &PropType) -> bool { + !property.list + && matches!( + property.scalar, + ScalarType::Bool + | ScalarType::I32 + | ScalarType::I64 + | ScalarType::U32 + | ScalarType::U64 + | ScalarType::String + ) +} + +/// An operand's type as the fragment compares it: an integer literal +/// compares with any integer property, every other operand only with its +/// own scalar type. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum Operand { + Scalar(ScalarType), + IntegerLiteral, + Null, +} + +impl Operand { + fn integer(self) -> bool { + matches!( + self, + Self::IntegerLiteral + | Self::Scalar( + ScalarType::I32 | ScalarType::I64 | ScalarType::U32 | ScalarType::U64 + ) + ) + } + + fn string(self) -> bool { + self == Self::Scalar(ScalarType::String) + } + + fn same_type(self, other: Self) -> bool { + match (self, other) { + (Self::Null, _) | (_, Self::Null) => true, + (Self::IntegerLiteral, other) | (other, Self::IntegerLiteral) => other.integer(), + (left, right) => left == right, + } + } +} + +struct Fragment<'a> { + binding: &'a str, + node_type: &'a NodeType, + params: &'a [omnigraph_compiler::query::ast::Param], +} + +impl Fragment<'_> { + fn property(&self, variable: &str, property: &str) -> Option { + if variable != self.binding { + return None; + } + let declared = self.node_type.properties.get(property)?; + admitted(declared).then_some(declared.scalar) + } + + fn operand(&self, expr: &Expr) -> Option { + match expr { + Expr::PropAccess { variable, property } => { + self.property(variable, property).map(Operand::Scalar) + } + Expr::Literal(Literal::Bool(_)) => Some(Operand::Scalar(ScalarType::Bool)), + Expr::Literal(Literal::String(_)) => Some(Operand::Scalar(ScalarType::String)), + Expr::Literal(Literal::Integer(_)) => Some(Operand::IntegerLiteral), + Expr::Literal(Literal::Null) => Some(Operand::Null), + Expr::Variable(name) => { + let param = self.params.iter().find(|param| param.name == *name)?; + let declared = PropType::from_param_type_name(¶m.type_name, param.nullable)?; + admitted(&declared).then_some(Operand::Scalar(declared.scalar)) + } + _ => None, + } + } + + /// Whether `expr` is an admitted eligibility predicate. + fn predicate(&self, expr: &Expr) -> bool { + match expr { + Expr::Binary { + left, + op: BinaryOp::And | BinaryOp::Or, + right, + } => self.predicate(left) && self.predicate(right), + Expr::Not(inner) => self.predicate(inner), + Expr::IsNull { expr, .. } => self.operand(expr).is_some(), + Expr::Binary { + left, + op: BinaryOp::Compare(op), + right, + } => { + let (Some(left), Some(right)) = (self.operand(left), self.operand(right)) else { + return false; + }; + left.same_type(right) + && match op { + CompOp::Eq | CompOp::Ne => true, + CompOp::Lt | CompOp::Le | CompOp::Gt | CompOp::Ge => { + (left.integer() || left.string() || left == Operand::Null) + && (right.integer() || right.string() || right == Operand::Null) + } + CompOp::Contains | CompOp::StartsWith | CompOp::StringContains => false, + } + } + other => self.operand(other) == Some(Operand::Scalar(ScalarType::Bool)), + } + } + + /// Whether `expr` is the leading `bm25()` the fragment admits: over a + /// String property of the binding, with a String literal or parameter. + fn bm25(&self, expr: &Expr) -> bool { + let Expr::Bm25 { field, query } = expr else { + return false; + }; + let Expr::PropAccess { variable, property } = field.as_ref() else { + return false; + }; + self.property(variable, property) == Some(ScalarType::String) + && self.operand(query) == Some(Operand::Scalar(ScalarType::String)) + } +} + +/// The member the checked declaration of `input` is, or `None` for a query +/// outside the fragment. +pub(crate) fn membership(input: &AcceptInput<'_>) -> Option { + let decl = input.checked.decl(); + let mut binding = None; + for clause in &decl.match_clause { + match clause { + Clause::Binding(found) if binding.is_none() => binding = Some(found), + Clause::Filter(_) => {} + _ => return None, + } + } + let binding = binding?; + let node_type = input.catalog.node_types.get(&binding.type_name)?; + let fragment = Fragment { + binding: &binding.variable, + node_type, + params: &decl.params, + }; + for matched in &binding.prop_matches { + let declared = node_type.properties.get(&matched.prop_name)?; + let comparison = Expr::comparison( + Expr::PropAccess { + variable: binding.variable.clone(), + property: matched.prop_name.clone(), + }, + CompOp::Eq, + matched.value.clone(), + ); + if !admitted(declared) || !fragment.predicate(&comparison) { + return None; + } + } + for clause in &decl.match_clause { + if let Clause::Filter(filter) = clause + && !fragment.predicate(filter) + { + return None; + } + } + let mut order = decl.order_clause.iter(); + let leading = decl.order_clause.first().map(|key| &key.expr); + let ranked = leading.is_some_and(|lead| fragment.bm25(lead)); + if ranked { + order.next(); + } + for key in order { + let Expr::PropAccess { variable, property } = &key.expr else { + return None; + }; + fragment.property(variable, property)?; + } + for projection in &decl.return_clause { + let direct = match &projection.expr { + Expr::PropAccess { variable, property } => { + fragment.property(variable, property).is_some() + } + call => ranked && Some(call) == leading, + }; + if !direct { + return None; + } + } + Some(Member { + binding: binding.variable.clone(), + type_name: binding.type_name.clone(), + }) +} + +/// The ranking a scan of the chain carries: `bm25` over `property` with +/// `query`, placing eligibility as declared. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +pub struct ChainRanking { + pub property: String, + pub query: ExprMirror, + pub eligibility: Eligibility, +} + +/// One node of a member's chain, logical until a lowering rule marks it +/// physical (`lowered`). A filter with no conjunct is no node. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +#[serde(tag = "node", rename_all = "snake_case")] +pub enum ChainNode { + Scan { + type_key: String, + version: Option, + binding: String, + filter: Vec, + projection: Option>, + ranked: Option, + lowered: bool, + }, + Filter { + conjuncts: Vec, + lowered: bool, + }, + Search { + property: String, + query: ExprMirror, + }, + Projection { + exprs: Vec, + lowered: bool, + }, + Sort { + keys: Vec, + fetch: Option, + tiebreak: Vec, + lowered: bool, + }, + Limit { + rows: usize, + lowered: bool, + }, +} + +/// A chain node's place, leaf first; a chain holds at most one of each. +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)] +pub(crate) enum Role { + Scan, + Filter, + Search, + Projection, + Sort, + Limit, +} + +impl ChainNode { + pub(crate) fn role(&self) -> Role { + match self { + Self::Scan { .. } => Role::Scan, + Self::Filter { .. } => Role::Filter, + Self::Search { .. } => Role::Search, + Self::Projection { .. } => Role::Projection, + Self::Sort { .. } => Role::Sort, + Self::Limit { .. } => Role::Limit, + } + } + + fn lowered(&self) -> bool { + match self { + Self::Scan { lowered, .. } + | Self::Filter { lowered, .. } + | Self::Projection { lowered, .. } + | Self::Sort { lowered, .. } + | Self::Limit { lowered, .. } => *lowered, + Self::Search { .. } => false, + } + } + + fn lower(&mut self) { + match self { + Self::Scan { lowered, .. } + | Self::Filter { lowered, .. } + | Self::Projection { lowered, .. } + | Self::Sort { lowered, .. } + | Self::Limit { lowered, .. } => *lowered = true, + Self::Search { .. } => {} + } + } +} + +/// One rule of the closed catalogue, with its typed substitution. Each +/// application names the arena nodes it rewrites (`at`) and appends one +/// successor per named node, a removed node as an empty slot. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +#[serde(tag = "rule", rename_all = "snake_case")] +pub enum Rule { + /// `Filter(p ∧ c, Scan(C, f))` → `Filter(p, Scan(C, f ∧ c))`, `at` + /// `[filter, scan]`; `c` must be one of the filter's conjuncts and + /// within the fragment's predicate forms, which the scan evaluates with + /// the same typed semantics. + AbsorbScanFilter { conjunct: ExprMirror }, + /// The scan reads `columns`, which must hold every column the nodes + /// above it read and only columns of its table; `at` `[scan]`. + PruneScanColumns { columns: Vec }, + /// A logical node becomes its physical operator unchanged; `at` `[node]`. + Lower, + /// `Search(a, …Scan(C))` → a ranked physical scan of `C` by `a`, + /// placing eligibility before scoring only under recorded full + /// full-text coverage; `at` `[search, scan]`. + RankBm25Scan { eligibility: Eligibility }, +} + +/// One recorded rule application. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +pub struct Step { + pub at: Vec, + #[serde(flatten)] + pub rule: Rule, +} + +/// The optimizer's record of how it rewrote a member's canonical chain: +/// rule applications over arena references. The arena starts with the +/// canonical chain, leaf first, and every application appends its +/// successors. +#[derive(Debug, Clone, Default, PartialEq, Serialize, Deserialize)] +pub struct Derivation { + pub steps: Vec, +} + +/// The canonical chain's roles for a plan shape: the optimizer and the +/// validator number the arena the same way. +pub(crate) fn canonical_roles(filter: bool, search: bool, sort: bool, limit: bool) -> Vec { + let mut roles = vec![Role::Scan]; + if filter { + roles.push(Role::Filter); + } + if search { + roles.push(Role::Search); + } + roles.push(Role::Projection); + if sort { + roles.push(Role::Sort); + } + if limit { + roles.push(Role::Limit); + } + roles +} + +/// The optimizer's side of a derivation: it numbers the arena as the +/// validator does and records each rule it applies. Inactive (the default) +/// on every plan that is no single-binding chain. +#[derive(Debug, Default)] +pub(crate) struct Tracer { + derivation: Option, + current: std::collections::HashMap, + next: usize, + /// Conjuncts the chain's filter still holds; absorbing the last one + /// removes the filter. + conjuncts: usize, +} + +impl Tracer { + /// Start recording over the canonical chain of `plan`, a resolved + /// logical plan, when it is one binding's chain: + /// `[Limit] [Sort] Projection [TextSearch] Filter* TableScan`. + pub(crate) fn for_plan(plan: &LogicalPlan) -> Self { + let mut id = plan.root(); + let mut limit = false; + let mut sort = false; + let mut search = false; + let mut conjuncts: Vec<&IRExpr> = Vec::new(); + let mut projection = false; + loop { + match plan.node(id) { + Some(LogicalNode::Limit { input, .. }) if !sort && !projection && !limit => { + limit = true; + id = *input; + } + Some(LogicalNode::Sort { input, .. }) if !projection && !sort => { + sort = true; + id = *input; + } + Some(LogicalNode::Projection { input, .. }) if !projection => { + projection = true; + id = *input; + } + Some(LogicalNode::TextSearch { input, .. }) if projection && !search => { + search = true; + id = *input; + } + Some(LogicalNode::Filter { + input, + conjuncts: held, + }) if projection => { + for conjunct in held { + if !conjuncts.contains(&conjunct) { + conjuncts.push(conjunct); + } + } + id = *input; + } + Some(LogicalNode::TableScan { input: None, spec }) + if projection && spec.binding.is_some() => + { + break; + } + _ => return Self::default(), + } + } + let roles = canonical_roles(!conjuncts.is_empty(), search, sort, limit); + Self { + derivation: Some(Derivation::default()), + current: roles + .iter() + .enumerate() + .map(|(index, role)| (*role, index)) + .collect(), + next: roles.len(), + conjuncts: conjuncts.len(), + } + } + + /// Record the placement pass moving `conjunct` from the chain's filter + /// into its scan. + pub(crate) fn absorb(&mut self, conjunct: &IRExpr) { + self.conjuncts = self.conjuncts.saturating_sub(1); + let removed: &[Role] = if self.conjuncts == 0 { + &[Role::Filter] + } else { + &[] + }; + self.record( + &[Role::Filter, Role::Scan], + Rule::AbsorbScanFilter { + conjunct: mirror(conjunct), + }, + removed, + ); + } + + /// Record `rule` applied to the current nodes of `roles`; a role in + /// `removed` has no successor. + pub(crate) fn record(&mut self, roles: &[Role], rule: Rule, removed: &[Role]) { + let Some(derivation) = self.derivation.as_mut() else { + return; + }; + let mut at = Vec::with_capacity(roles.len()); + for role in roles { + let Some(index) = self.current.get(role).copied() else { + self.derivation = None; + return; + }; + at.push(index); + } + for role in roles { + if removed.contains(role) { + self.current.remove(role); + } else { + self.current.insert(*role, self.next); + } + self.next += 1; + } + derivation.steps.push(Step { at, rule }); + } + + pub(crate) fn finish(self) -> Option { + self.derivation + } +} + +/// The checker's arena: every node the derivation reached, `None` where a +/// rule removed one. +struct Arena { + nodes: Vec>, + current: std::collections::HashMap, +} + +impl Arena { + fn node(&self, index: usize, role: Role) -> Result<&ChainNode, ValidationError> { + let node = self.nodes.get(index).and_then(Option::as_ref); + match node { + Some(node) if node.role() == role && self.current.get(&role) == Some(&index) => { + Ok(node) + } + _ => Err(invalid(format!( + "a rule names node {index} as the current {role:?}, which it is not" + ))), + } + } +} + +fn invalid(detail: String) -> ValidationError { + ValidationError::violated("exact subset", detail) +} + +fn mirror(expr: &IRExpr) -> ExprMirror { + ExprMirror::from(expr) +} + +/// The physical scan's table and pinned version recorded for `type_key`. +fn pinned(plan: &PhysicalPlan, type_key: &str) -> Option> { + plan.assumptions() + .datasets + .get(type_key) + .map(|pin| pin.as_ref().map(|pin| pin.version)) +} + +/// Check `derivation` for `member` against the candidate `plan`. +pub(crate) fn check( + derivation: &Derivation, + plan: &PhysicalPlan, + input: &AcceptInput<'_>, + requirements: &Requirements, + member: &Member, + budget: &mut Budget, +) -> Result<(), ValidationError> { + let ir = input.ir; + let matcher = Matcher::new(input, requirements); + lowered_ir(ir, input, member, &matcher, budget)?; + let mut arena = canonical(ir, plan, member, input)?; + for step in &derivation.steps { + budget.step()?; + apply(&mut arena, step, plan, member, input, budget)?; + } + let reached: Vec<&ChainNode> = [ + Role::Limit, + Role::Sort, + Role::Projection, + Role::Search, + Role::Filter, + Role::Scan, + ] + .iter() + .filter_map(|role| arena.current.get(role)) + .filter_map(|index| arena.nodes[*index].as_ref()) + .filter(|node| !matches!(node, ChainNode::Filter { conjuncts, .. } if conjuncts.is_empty())) + .collect(); + if let Some(node) = reached.iter().find(|node| !node.lowered()) { + return Err(invalid(format!( + "the derivation leaves the {:?} logical", + node.role() + ))); + } + let candidate = chain_of(plan, input)?; + if reached.len() != candidate.len() + || reached + .iter() + .zip(&candidate) + .any(|(left, right)| *left != right) + { + return Err(invalid(format!( + "the derivation reconstructs {reached:?}; the plan is {candidate:?}" + ))); + } + Ok(()) +} + +/// Stage one: the IR is the lowering of the member's declaration, the +/// pattern the compiler defines for one binding. +fn lowered_ir( + ir: &QueryIR, + input: &AcceptInput<'_>, + member: &Member, + matcher: &Matcher<'_>, + budget: &mut Budget, +) -> Result<(), ValidationError> { + let decl = input.checked.decl(); + let mut pipeline = ir.pipeline.iter(); + let Some(IROp::NodeScan { + variable, + type_name, + filters, + }) = pipeline.next() + else { + return Err(invalid( + "the IR does not start with the binding's scan".into(), + )); + }; + if *variable != member.binding || *type_name != member.type_name { + return Err(invalid(format!( + "the IR scans `${variable}: {type_name}`; the query binds `${}: {}`", + member.binding, member.type_name + ))); + } + let binding = decl.match_clause.iter().find_map(|clause| match clause { + Clause::Binding(binding) => Some(binding), + _ => None, + }); + let matches = binding.map_or(&[][..], |binding| binding.prop_matches.as_slice()); + if filters.len() != matches.len() { + return Err(invalid( + "the IR scan's inline filters are not the binding's matches".into(), + )); + } + for (matched, filter) in matches.iter().zip(filters) { + let written = Expr::comparison( + Expr::PropAccess { + variable: member.binding.clone(), + property: matched.prop_name.clone(), + }, + CompOp::Eq, + matched.value.clone(), + ); + if !matcher.matches(&written, filter, Position::Filter, budget)? { + return Err(invalid(format!("the IR tests `{filter}` for `{written}`"))); + } + } + let clauses: Vec<&Expr> = decl + .match_clause + .iter() + .filter_map(|clause| match clause { + Clause::Filter(filter) => Some(filter), + _ => None, + }) + .collect(); + let rest: Vec<&IROp> = pipeline.collect(); + if rest.len() != clauses.len() { + return Err(invalid( + "the IR's filters are not the query's clauses".into(), + )); + } + for (clause, op) in clauses.iter().zip(rest) { + let IROp::Filter(filter) = op else { + return Err(invalid("the IR holds an op the fragment does not".into())); + }; + if !matcher.matches(clause, filter, Position::Filter, budget)? { + return Err(invalid(format!("the IR tests `{filter}` for `{clause}`"))); + } + } + if ir.return_exprs.len() != decl.return_clause.len() { + return Err(invalid("the IR's return is not the query's".into())); + } + for (written, lowered) in decl.return_clause.iter().zip(&ir.return_exprs) { + if written.alias != lowered.alias + || !matcher.matches(&written.expr, &lowered.expr, Position::Return, budget)? + { + return Err(invalid(format!( + "the IR returns `{}`; the query returns `{}`", + lowered.expr, written.expr + ))); + } + } + if ir.order_by.len() != decl.order_clause.len() { + return Err(invalid("the IR's order is not the query's".into())); + } + for (written, lowered) in decl.order_clause.iter().zip(&ir.order_by) { + if written.descending != lowered.descending + || !matcher.matches(&written.expr, &lowered.expr, Position::Plain, budget)? + { + return Err(invalid(format!( + "the IR orders by `{}`; the query by `{}`", + lowered.expr, written.expr + ))); + } + } + if ir.limit != decl.limit { + return Err(invalid("the IR's limit is not the query's".into())); + } + Ok(()) +} + +/// The canonical chain of a member, from its checked IR: the scan of the +/// pinned table, every conjunct in written order (a repeated one once), the +/// leading `bm25()`, the projection, the sort of the remaining keys with the +/// identity tie-break the comparator requires, and the limit. +fn canonical( + ir: &QueryIR, + plan: &PhysicalPlan, + member: &Member, + input: &AcceptInput<'_>, +) -> Result { + let type_key = format!("node:{}", member.type_name); + let version = pinned(plan, &type_key) + .ok_or_else(|| invalid(format!("the plan records no pin of `{type_key}`")))?; + let mut conjuncts: Vec = Vec::new(); + for op in &ir.pipeline { + let filters: Vec = match op { + IROp::NodeScan { filters, .. } => filters.clone(), + IROp::Filter(filter) => vec![filter.clone()], + _ => Vec::new(), + }; + for conjunct in filters.into_iter().flat_map(IRExpr::into_conjuncts) { + if !conjuncts.contains(&conjunct) { + conjuncts.push(conjunct); + } + } + } + let mut keys: &[IROrdering] = &ir.order_by; + let search = match keys.first().map(|key| &key.expr) { + Some(IRExpr::Bm25 { field, query }) => match field.as_ref() { + IRExpr::PropAccess { property, .. } => { + keys = &keys[1..]; + Some(ChainNode::Search { + property: property.clone(), + query: mirror(query), + }) + } + _ => None, + }, + _ => None, + }; + let sorted = search.is_some() || !ir.order_by.is_empty(); + let mut nodes = vec![ChainNode::Scan { + type_key, + version, + binding: member.binding.clone(), + filter: Vec::new(), + projection: None, + ranked: None, + lowered: false, + }]; + if !conjuncts.is_empty() { + nodes.push(ChainNode::Filter { + conjuncts: conjuncts.iter().map(mirror).collect(), + lowered: false, + }); + } + nodes.extend(search); + nodes.push(ChainNode::Projection { + exprs: ir.return_exprs.iter().map(ProjectionMirror::from).collect(), + lowered: false, + }); + if sorted { + let written: Vec<&IROrdering> = keys + .iter() + .filter(|key| !matches!(key.expr, IRExpr::Literal(_))) + .collect(); + nodes.push(ChainNode::Sort { + keys: written + .iter() + .map(|key| OrderingMirror::from(*key)) + .collect(), + fetch: ir.limit.and_then(|limit| usize::try_from(limit).ok()), + tiebreak: canonical_tiebreak(ir, member, input), + lowered: false, + }); + } + if let Some(limit) = ir.limit { + nodes.push(ChainNode::Limit { + rows: usize::try_from(limit).unwrap_or(usize::MAX), + lowered: false, + }); + } + Ok(Arena { + current: nodes + .iter() + .enumerate() + .map(|(index, node)| (node.role(), index)) + .collect(), + nodes: nodes.into_iter().map(Some).collect(), + }) +} + +/// The identity tie-break of the canonical sort: the binding's id unless +/// every returned expression is an order key (rows that tie are then +/// indistinguishable) or the id is a key itself. +fn canonical_tiebreak(ir: &QueryIR, member: &Member, input: &AcceptInput<'_>) -> Vec { + let keys: Vec = ir.order_by.iter().map(|key| key.expr.to_string()).collect(); + let covered = ir + .return_exprs + .iter() + .all(|projection| keys.contains(&projection.expr.to_string())); + let id = input.catalog.system_columns.id; + let keyed = ir.order_by.iter().any(|key| { + matches!(&key.expr, IRExpr::PropAccess { variable, property } + if *variable == member.binding && property == id) + }); + if covered || keyed { + Vec::new() + } else { + vec![ColumnRef::property(&member.binding, IDENTITY_MEMBER)] + } +} + +/// The columns the nodes above the scan read: returned and sorted +/// properties, residual conjuncts, and the id when a tie-break reads it. +fn demanded(arena: &Arena, member: &Member, input: &AcceptInput<'_>) -> BTreeSet { + let mut reads = Vec::new(); + for (role, index) in &arena.current { + let Some(node) = arena.nodes[*index].as_ref() else { + continue; + }; + match (role, node) { + (Role::Filter, ChainNode::Filter { conjuncts, .. }) => { + for conjunct in conjuncts { + crate::optimizer::reads_of_expr(&IRExpr::from(conjunct.clone()), &mut reads); + } + } + (Role::Projection, ChainNode::Projection { exprs, .. }) => { + for projection in exprs { + let projection = IRProjection::from(projection.clone()); + crate::optimizer::reads_of_expr(&projection.expr, &mut reads); + } + } + (Role::Sort, ChainNode::Sort { keys, tiebreak, .. }) => { + for key in keys { + let key = IROrdering::from(key.clone()); + crate::optimizer::reads_of_expr(&key.expr, &mut reads); + } + if !tiebreak.is_empty() { + reads.push(ColumnRef::property(&member.binding, IDENTITY_MEMBER)); + } + } + _ => {} + } + } + let id = input.catalog.system_columns.id; + reads + .into_iter() + .filter(|read| read.binding == member.binding) + .filter_map(|read| match read.property.as_deref() { + Some(IDENTITY_MEMBER) => Some(id.to_string()), + Some(property) => Some(property.to_string()), + None => None, + }) + .collect() +} + +/// Apply one recorded rule: check its references and precondition, then +/// reconstruct its successors. +fn apply( + arena: &mut Arena, + step: &Step, + plan: &PhysicalPlan, + member: &Member, + input: &AcceptInput<'_>, + budget: &mut Budget, +) -> Result<(), ValidationError> { + let successors: Vec<(Role, Option)> = match (&step.rule, step.at.as_slice()) { + (Rule::AbsorbScanFilter { conjunct }, [filter, scan]) => { + let ChainNode::Filter { conjuncts, lowered } = + arena.node(*filter, Role::Filter)?.clone() + else { + unreachable!("the arena checked the role"); + }; + let ChainNode::Scan { + type_key, + version, + binding, + mut filter, + projection, + ranked, + lowered: scan_lowered, + } = arena.node(*scan, Role::Scan)?.clone() + else { + unreachable!("the arena checked the role"); + }; + if lowered || scan_lowered || ranked.is_some() { + return Err(invalid( + "a filter is absorbed only between logical nodes".into(), + )); + } + let Some(position) = conjuncts.iter().position(|held| held == conjunct) else { + return Err(invalid(format!( + "the filter holds no conjunct `{}` to absorb", + IRExpr::from(conjunct.clone()) + ))); + }; + if !translatable(&IRExpr::from(conjunct.clone()), member, input) { + return Err(invalid(format!( + "`{}` has no admitted scan translation", + IRExpr::from(conjunct.clone()) + ))); + } + let mut rest = conjuncts; + let moved = rest.remove(position); + filter.push(moved); + let filter_node = (!rest.is_empty()).then_some(ChainNode::Filter { + conjuncts: rest, + lowered: false, + }); + vec![ + (Role::Filter, filter_node), + ( + Role::Scan, + Some(ChainNode::Scan { + type_key, + version, + binding, + filter, + projection, + ranked, + lowered: false, + }), + ), + ] + } + (Rule::PruneScanColumns { columns }, [scan]) => { + let mut node = arena.node(*scan, Role::Scan)?.clone(); + let schema = &input + .catalog + .node_types + .get(&member.type_name) + .ok_or_else(|| invalid("the member's type left the catalog".into()))? + .arrow_schema; + budget.visit(u64::try_from(columns.len()).unwrap_or(u64::MAX))?; + if let Some(unknown) = columns + .iter() + .find(|column| schema.field_with_name(column).is_err()) + { + return Err(invalid(format!("the scan cannot read `{unknown}`"))); + } + let held: BTreeSet<&String> = columns.iter().collect(); + if let Some(missing) = demanded(arena, member, input) + .into_iter() + .find(|column| schema.field_with_name(column).is_ok() && !held.contains(column)) + { + return Err(invalid(format!( + "the pruned scan drops `{missing}`, which a node above it reads" + ))); + } + if let ChainNode::Scan { projection, .. } = &mut node { + *projection = Some(columns.clone()); + } + vec![(Role::Scan, Some(node))] + } + (Rule::Lower, [at]) => { + let role = arena + .nodes + .get(*at) + .and_then(Option::as_ref) + .map(ChainNode::role) + .ok_or_else(|| invalid(format!("no node {at} to lower")))?; + let mut node = arena.node(*at, role)?.clone(); + match (&mut node, role) { + (ChainNode::Search { .. }, _) => { + return Err(invalid("a search lowers only into its ranked scan".into())); + } + (ChainNode::Sort { keys, .. }, Role::Sort) => { + if arena.current.contains_key(&Role::Search) { + return Err(invalid("the sort lowers after its ranking".into())); + } + let ranked = arena + .current + .get(&Role::Scan) + .and_then(|index| arena.nodes[*index].as_ref()) + .and_then(|scan| match scan { + ChainNode::Scan { ranked, .. } => ranked.clone(), + _ => None, + }); + if ranked.is_some() { + let (column, descending) = RankKind::Bm25.score(); + keys.insert( + 0, + OrderingMirror::from(&IROrdering { + expr: IRExpr::PropAccess { + variable: member.binding.clone(), + property: column.to_string(), + }, + descending, + }), + ); + } + } + _ => {} + } + if node.lowered() { + return Err(invalid(format!("the {role:?} is lowered twice"))); + } + node.lower(); + vec![(role, Some(node))] + } + (Rule::RankBm25Scan { eligibility }, [search, scan]) => { + let ChainNode::Search { property, query } = arena.node(*search, Role::Search)?.clone() + else { + unreachable!("the arena checked the role"); + }; + let mut node = arena.node(*scan, Role::Scan)?.clone(); + let ChainNode::Scan { + type_key, + ranked, + lowered, + .. + } = &mut node + else { + unreachable!("the arena checked the role"); + }; + if !*lowered || ranked.is_some() { + return Err(invalid("only a physical, unranked scan is ranked".into())); + } + let coverage = + plan.assumptions() + .full_text + .get(&crate::physical::Assumptions::full_text_key( + type_key, &property, + )); + if *eligibility == Eligibility::BeforeScoring + && coverage != Some(&FullTextCoverage::Full) + { + return Err(invalid(format!( + "the bm25 scan filters before scoring under recorded coverage {coverage:?}" + ))); + } + *ranked = Some(ChainRanking { + property, + query, + eligibility: *eligibility, + }); + vec![(Role::Search, None), (Role::Scan, Some(node))] + } + (rule, at) => { + return Err(invalid(format!("rule {rule:?} names {} nodes", at.len()))); + } + }; + for (role, successor) in successors { + budget.node()?; + let index = arena.nodes.len(); + match successor { + Some(node) => { + arena.nodes.push(Some(node)); + arena.current.insert(role, index); + } + None => { + arena.nodes.push(None); + arena.current.remove(&role); + } + } + } + Ok(()) +} + +/// Whether the scan evaluates `conjunct` with the same typed semantics as +/// the in-memory filter: the fragment's predicate forms over the member's +/// properties, literals and bound parameters, with no cast. +fn translatable(conjunct: &IRExpr, member: &Member, input: &AcceptInput<'_>) -> bool { + let Some(node_type) = input.catalog.node_types.get(&member.type_name) else { + return false; + }; + fn operand(expr: &IRExpr, member: &Member, node_type: &NodeType) -> bool { + match expr { + IRExpr::PropAccess { variable, property } => { + *variable == member.binding + && node_type.properties.get(property).is_some_and(admitted) + } + IRExpr::Literal( + Literal::Bool(_) | Literal::String(_) | Literal::Integer(_) | Literal::Null, + ) + | IRExpr::Param(_) => true, + _ => false, + } + } + match conjunct { + IRExpr::Binary { + left, + op: BinaryOp::And | BinaryOp::Or, + right, + } => translatable(left, member, input) && translatable(right, member, input), + IRExpr::Not(inner) => translatable(inner, member, input), + IRExpr::IsNull { expr, .. } => operand(expr, member, node_type), + IRExpr::Binary { + left, + op: + BinaryOp::Compare( + CompOp::Eq | CompOp::Ne | CompOp::Lt | CompOp::Le | CompOp::Gt | CompOp::Ge, + ), + right, + } => operand(left, member, node_type) && operand(right, member, node_type), + other => operand(other, member, node_type), + } +} + +/// The candidate plan as a chain, root first, or why it is none. +fn chain_of( + plan: &PhysicalPlan, + input: &AcceptInput<'_>, +) -> Result, ValidationError> { + let mut chain = Vec::new(); + let mut id = plan.root(); + loop { + let node = plan + .node(id) + .ok_or_else(|| invalid(format!("node {id} is a tombstone")))?; + let (converted, next) = match node { + PhysicalNode::Limit { input, rows } => ( + ChainNode::Limit { + rows: *rows, + lowered: true, + }, + Some(*input), + ), + PhysicalNode::Sort { + input, + order_by, + fetch, + tiebreak, + } => ( + ChainNode::Sort { + keys: order_by.iter().map(OrderingMirror::from).collect(), + fetch: *fetch, + tiebreak: tiebreak.clone(), + lowered: true, + }, + Some(*input), + ), + PhysicalNode::Projection { + input, + return_exprs, + } => ( + ChainNode::Projection { + exprs: return_exprs.iter().map(ProjectionMirror::from).collect(), + lowered: true, + }, + Some(*input), + ), + PhysicalNode::Filter { input, filters } => ( + ChainNode::Filter { + conjuncts: filters.iter().map(mirror).collect(), + lowered: true, + }, + Some(*input), + ), + PhysicalNode::Scan { + source: ScanInput::Table, + spec, + ordered: false, + keys_only: false, + ranked, + } => { + let same_table = match plan.assumptions().datasets.get(&spec.table.type_key) { + Some(Some(pin)) => { + pin.dataset_path == spec.table.dataset_path + && pin.native_branch == spec.table.native_branch + } + Some(None) => spec.version.is_none(), + None => false, + }; + if !same_table + || spec.fragments.is_some() + || spec.runtime_filter.is_some() + || spec.columns != input.catalog.system_columns + { + return Err(invalid(format!( + "the scan of `{}` reads another table image than the one pinned", + spec.table.type_key + ))); + } + let ranked = match ranked { + None => None, + Some(access) + if access.kind == RankKind::Bm25 + && access.fetch.is_none() + && access.nprobes.is_none() + && access.scope == RankScope::Order + && access.overfetch.is_empty() + && access.prefilter.is_none() => + { + Some(ChainRanking { + property: access.property.clone(), + query: mirror(&access.query), + eligibility: access.eligibility, + }) + } + Some(_) => { + return Err(invalid( + "the scan's ranking is no exact bm25 retrieval".into(), + )); + } + }; + ( + ChainNode::Scan { + type_key: spec.table.type_key.clone(), + version: spec.version, + binding: spec.binding.clone().unwrap_or_default(), + filter: spec + .filter + .as_ref() + .map(|filter| filter.gq_filters().iter().map(mirror).collect()) + .unwrap_or_default(), + projection: spec.projection.clone(), + ranked, + lowered: true, + }, + None, + ) + } + other => { + return Err(invalid(format!( + "the plan holds a {} the fragment's chain does not", + other.name() + ))); + } + }; + chain.push(converted); + match next { + Some(next) => id = next, + None => return Ok(chain), + } + } +} diff --git a/crates/omnigraph-planner/src/validate/tests.rs b/crates/omnigraph-planner/src/validate/tests.rs index 1da3ecb63..bc0fe1d1a 100644 --- a/crates/omnigraph-planner/src/validate/tests.rs +++ b/crates/omnigraph-planner/src/validate/tests.rs @@ -103,11 +103,27 @@ impl Fixture { } } + fn traced(&self) -> crate::gate::Traced { + crate::gate::plan_traced(&self.ir, &self.source, &BOUNDS).expect("the query plans") + } + fn plan(&self) -> PhysicalPlan { - crate::gate::plan_query(&self.ir, &self.source, &BOUNDS).expect("the query plans") + self.traced().optimized.physical + } + + fn derivation(&self) -> Option { + self.traced().derivation } fn accept(&self, plan: PhysicalPlan) -> Result { + self.accept_with(plan, self.derivation()) + } + + fn accept_with( + &self, + plan: PhysicalPlan, + derivation: Option, + ) -> Result { let constants = Bound(&self.params); let input = AcceptInput { checked: &self.checked, @@ -117,7 +133,7 @@ impl Fixture { constants: &constants, limits: ValidationLimits::DEFAULT, }; - accept(plan, &input) + accept(plan, &input, derivation) } /// The check that refuses `plan`, which must be refused. @@ -164,7 +180,127 @@ fn the_planned_plans_are_accepted() { let accepted = fixture .accept(fixture.plan()) .expect("a planned plan is accepted"); - assert_eq!(accepted.scope(), ValidationScope::InvariantsOnly); + assert_eq!(accepted.scope(), ValidationScope::ExactSubset); + assert!(accepted.evidence().derivation().is_some()); + } + let outside = Fixture::new( + r#"query q() { + match { $d: Doc $d.title contains "graph" } + return { $d.slug } +}"#, + &[], + ); + let accepted = outside + .accept(outside.plan()) + .expect("a planned plan is accepted"); + assert_eq!(accepted.scope(), ValidationScope::InvariantsOnly); + assert!(accepted.evidence().derivation().is_none()); +} + +/// The rules each fixture's derivation applies, in order. +fn rules(derivation: &Derivation) -> Vec<&'static str> { + derivation + .steps + .iter() + .map(|step| match step.rule { + Rule::AbsorbScanFilter { .. } => "absorb", + Rule::PruneScanColumns { .. } => "prune", + Rule::Lower => "lower", + Rule::RankBm25Scan { .. } => "rank", + }) + .collect() +} + +#[test] +fn the_derivations_record_every_rule() { + assert_eq!( + rules(&filtered().derivation().unwrap()), + [ + "absorb", "absorb", "prune", "lower", "lower", "lower", "lower" + ] + ); + assert_eq!( + rules(&ranked().derivation().unwrap()), + [ + "absorb", "prune", "lower", "rank", "lower", "lower", "lower" + ] + ); +} + +#[test] +fn a_member_without_its_derivation_is_refused() { + let fixture = filtered(); + let (check, detail) = match fixture.accept_with(fixture.plan(), None) { + Err(ValidationError::Violated { check, detail }) => (check, detail), + other => panic!("{other:?}"), + }; + assert_eq!(check, "exact subset", "{detail}"); +} + +#[test] +fn a_dropped_step_fails_the_reconstruction() { + let fixture = filtered(); + let mut derivation = fixture.derivation().unwrap(); + derivation.steps.remove(0); + let error = fixture + .accept_with(fixture.plan(), Some(derivation)) + .unwrap_err(); + assert!( + matches!( + &error, + ValidationError::Violated { + check: "exact subset", + .. + } + ), + "{error:?}" + ); +} + +#[test] +fn a_scan_pruned_below_its_readers_is_refused() { + let fixture = filtered(); + let plan = fixture.plan(); + let mut derivation = fixture.derivation().unwrap(); + for step in &mut derivation.steps { + if let Rule::PruneScanColumns { columns } = &mut step.rule { + columns.retain(|column| column != "year"); + } + } + match fixture.accept_with(plan, Some(derivation)) { + Err(ValidationError::Violated { check, detail }) => { + assert_eq!(check, "exact subset"); + assert!(detail.contains("drops `year`"), "{detail}"); + } + other => panic!("{other:?}"), + } +} + +#[test] +fn a_bm25_rank_before_scoring_needs_full_coverage() { + let mut fixture = ranked(); + fixture.source = std::mem::take(&mut fixture.source).with_full_text_coverage( + "node:Doc", + "title", + crate::source::FullTextCoverage::Full, + ); + let traced = fixture.traced(); + let accepted = fixture + .accept_with(traced.optimized.physical.clone(), traced.derivation.clone()) + .expect("full coverage admits filtering before scoring"); + assert_eq!(accepted.scope(), ValidationScope::ExactSubset); + let mut plan = traced.optimized.physical; + let mut assumptions = plan.assumptions().clone(); + assumptions.full_text.insert( + crate::physical::Assumptions::full_text_key("node:Doc", "title"), + crate::source::FullTextCoverage::Partial, + ); + plan.set_assumptions(assumptions); + match fixture.accept_with(plan, traced.derivation) { + Err(ValidationError::Violated { check, detail }) => { + assert_eq!(check, "prerequisite", "{detail}"); + } + other => panic!("{other:?}"), } } @@ -334,7 +470,7 @@ fn an_exhausted_budget_is_a_resource_outcome() { }, }; assert_eq!( - accept(fixture.plan(), &input).unwrap_err(), + accept(fixture.plan(), &input, fixture.derivation()).unwrap_err(), ValidationError::Exhausted { limit: "work", value: 3 diff --git a/crates/omnigraph/tests/engine_v2_plan_replay.rs b/crates/omnigraph/tests/engine_v2_plan_replay.rs index 41b44e721..75118ba66 100644 --- a/crates/omnigraph/tests/engine_v2_plan_replay.rs +++ b/crates/omnigraph/tests/engine_v2_plan_replay.rs @@ -1308,3 +1308,61 @@ async fn sibling_detached_edge_pins_refuse_traversal_plan_replay() { ) .await; } + +/// A member of the exact fragment replays with its checked derivation; an +/// envelope whose derivation was dropped or edited is refused as invalid +/// evidence, before anything runs. +#[tokio::test] +async fn an_exact_subset_member_replays_only_with_its_derivation() { + const SOURCE: &str = r#"query elders($min: I64) { + match { $p: Person $p.age >= $min } + return { $p.name, $p.age as age } + order { $p.age desc } + limit 2 +}"#; + let dir = tempfile::tempdir().unwrap(); + let db = people(&dir).await; + let params = ParamMap::from([("min".to_string(), Literal::Integer(30))]); + let run = db + .query_inspected(ReadTarget::branch("main"), SOURCE, "elders", ¶ms) + .await + .unwrap(); + assert_eq!( + run.evidence.scope(), + omnigraph_planner::ValidationScope::ExactSubset + ); + let envelope = run.replay_envelope(SOURCE, "elders"); + let replay = db + .replay_bound_plan(ReadTarget::branch("main"), &envelope) + .await + .unwrap(); + assert_eq!(rows_of(&replay.result), rows_of(&run.result)); + assert_eq!( + replay.evidence.scope(), + omnigraph_planner::ValidationScope::ExactSubset + ); + let saved: Value = serde_json::from_slice(&envelope).unwrap(); + let mut dropped = saved.clone(); + dropped.as_object_mut().unwrap().remove("derivation"); + let mut edited = saved.clone(); + edited["derivation"]["steps"] + .as_array_mut() + .unwrap() + .remove(0); + for (case, altered) in [("dropped", dropped), ("edited", edited)] { + let error = db + .replay_bound_plan( + ReadTarget::branch("main"), + &serde_json::to_vec(&altered).unwrap(), + ) + .await + .err() + .unwrap_or_else(|| panic!("{case}: the replay must be refused")); + assert!( + matches!(&error, OmniError::Manifest(manifest) + if manifest.kind == omnigraph::error::ManifestErrorKind::BadRequest) + && error.to_string().contains("exact subset"), + "{case}: {error}" + ); + } +} From 7583516e1f45f429c73b1e56efb122753960bf3e Mon Sep 17 00:00:00 2001 From: Ragnor Comerford Date: Thu, 1 Oct 2026 20:10:20 +0200 Subject: [PATCH 07/29] engine: the plan declares the search policies the run adapts within A nearest scan declares its probe policy (the escalation factor, the flat rescan of rows the partition search did not reach, the uncapped fail-closed rescan when Lance's counters are missing, and the flat start when the eligible set fits within the fetch); every pre-pass declares what an empty eligible set decides and whether the recorded full-text coverage of the bm25 scans it feeds admits it. The scan's ladder and both gates read these from the plan instead of their own constants, and the rrf gate's coverage guard is the recorded fact, no longer a dataset read at run time. Acceptance checks that every ladder terminates and keeps its fallbacks, and that every pre-pass draws its eligible set from required first hops and feeds only what its kind may prefilter. The execution report records each gate's verdict with the counts it read and each probe attempt per rung, and a replay must record the same decisions. --- crates/omnigraph-planner/src/lib.rs | 6 +- crates/omnigraph-planner/src/mirror.rs | 15 +- crates/omnigraph-planner/src/optimizer.rs | 38 +- crates/omnigraph-planner/src/physical.rs | 56 ++- .../src/validate/invariants.rs | 172 ++++++++- .../omnigraph-planner/src/validate/tests.rs | 95 ++++- crates/omnigraph-planner/tests/lower_walk.rs | 3 + .../tests/query_plan/explain.rs | 6 + crates/omnigraph/src/engine/lower.rs | 1 + crates/omnigraph/src/engine/mod.rs | 72 ++-- crates/omnigraph/src/engine/report.rs | 63 ++++ crates/omnigraph/src/engine/scan.rs | 24 +- crates/omnigraph/src/engine/search.rs | 348 +++++++++++++----- crates/omnigraph/src/table_store.rs | 39 +- .../omnigraph/tests/engine_v2_plan_replay.rs | 62 +++- 15 files changed, 826 insertions(+), 174 deletions(-) diff --git a/crates/omnigraph-planner/src/lib.rs b/crates/omnigraph-planner/src/lib.rs index b1a3e2df2..9af0d549d 100644 --- a/crates/omnigraph-planner/src/lib.rs +++ b/crates/omnigraph-planner/src/lib.rs @@ -58,9 +58,9 @@ pub use lower::{ pub use operation::{Operation, PageBudgetSpec, ScopeSpec, Side, TableRef}; pub use optimizer::{Bounds, physical_plan, rewrite}; pub use physical::{ - Assumptions, DatasetPin, Eligibility, Estimate, GatePolicy, Hop, NodeId, OrderKey, - OverfetchRung, PhysicalNode, PhysicalPlan, Prefilter, PrefilterMode, Properties, RankArm, - RankKind, RankScope, RankedAccess, ScanInput, StatisticSource, + Assumptions, DatasetPin, Eligibility, EmptyEligible, Estimate, GatePolicy, Hop, NearestPolicy, + NodeId, OrderKey, OverfetchRung, PhysicalNode, PhysicalPlan, Prefilter, PrefilterMode, + Properties, RankArm, RankKind, RankScope, RankedAccess, ScanInput, StatisticSource, }; pub use registry::{Coverage, Entry, Route, Shape}; pub use route::RouteOverride; diff --git a/crates/omnigraph-planner/src/mirror.rs b/crates/omnigraph-planner/src/mirror.rs index 8ddacf33d..dbaa8fa37 100644 --- a/crates/omnigraph-planner/src/mirror.rs +++ b/crates/omnigraph-planner/src/mirror.rs @@ -23,9 +23,9 @@ use crate::error::PlanError; use crate::logical::{ColumnRef, GqFilter, KeyJoinKind, Predicate, RuntimeFilterSpec, ScanSpec}; use crate::operation::TableRef; use crate::physical::{ - Assumptions, Eligibility, Estimate, Hop, NodeId, OrderKey, OverfetchRung, PhysicalNode, - PhysicalPlan, Prefilter, Properties, RankArm, RankKind, RankScope, RankedAccess, ScanInput, - StatisticSource, + Assumptions, Eligibility, EmptyEligible, Estimate, Hop, NearestPolicy, NodeId, OrderKey, + OverfetchRung, PhysicalNode, PhysicalPlan, Prefilter, Properties, RankArm, RankKind, RankScope, + RankedAccess, ScanInput, StatisticSource, }; use crate::source::SideId; @@ -740,6 +740,7 @@ pub struct RankedMirror { pub overfetch: Vec, pub prefilter: Option, pub eligibility: Eligibility, + pub policy: Option, } impl From<&RankedAccess> for RankedMirror { @@ -754,6 +755,7 @@ impl From<&RankedAccess> for RankedMirror { overfetch: ranked.overfetch.clone(), prefilter: ranked.prefilter.as_ref().map(PrefilterMirror::from), eligibility: ranked.eligibility, + policy: ranked.policy, } } } @@ -770,6 +772,7 @@ impl From for RankedAccess { overfetch: mirror.overfetch, prefilter: mirror.prefilter.map(Prefilter::from), eligibility: mirror.eligibility, + policy: mirror.policy, } } } @@ -785,6 +788,8 @@ pub struct PrefilterMirror { pub ranked_type: String, pub hops: Vec, pub feeds: Vec, + pub on_empty: EmptyEligible, + pub coverage_admits: bool, } impl From<&Prefilter> for PrefilterMirror { @@ -800,6 +805,8 @@ impl From<&Prefilter> for PrefilterMirror { }) .collect(), feeds: prefilter.feeds.clone(), + on_empty: prefilter.on_empty, + coverage_admits: prefilter.coverage_admits, } } } @@ -817,6 +824,8 @@ impl From for Prefilter { }) .collect(), feeds: mirror.feeds, + on_empty: mirror.on_empty, + coverage_admits: mirror.coverage_admits, } } } diff --git a/crates/omnigraph-planner/src/optimizer.rs b/crates/omnigraph-planner/src/optimizer.rs index 227a24f32..e6b869208 100644 --- a/crates/omnigraph-planner/src/optimizer.rs +++ b/crates/omnigraph-planner/src/optimizer.rs @@ -27,9 +27,9 @@ use crate::logical::{ use crate::lower::ContainsJoinFields; use crate::operation::{Operation, Side}; use crate::physical::{ - Assumptions, Eligibility, Estimate, Hop, NodeId, OrderKey, OverfetchRung, PhysicalNode, - PhysicalPlan, Prefilter, Properties, RankArm, RankKind, RankScope, RankedAccess, ScanInput, - StatisticSource, TextContains, + Assumptions, Eligibility, EmptyEligible, Estimate, Hop, NearestPolicy, NodeId, OrderKey, + OverfetchRung, PhysicalNode, PhysicalPlan, Prefilter, Properties, RankArm, RankKind, RankScope, + RankedAccess, ScanInput, StatisticSource, TextContains, }; use crate::source::{FullTextCoverage, NodeTypeSpec, PlanSource, SideId}; use crate::validate::subset::{Role, Rule, Tracer}; @@ -2182,7 +2182,20 @@ impl Lowering<'_> { binding: &str, scan: NodeId, feeds: Vec, + on_empty: EmptyEligible, ) -> Result { + let coverage_admits = feeds.iter().all(|feed| match self.physical.node(*feed) { + Some(PhysicalNode::Scan { + spec, + ranked: Some(ranked), + .. + }) if ranked.kind == RankKind::Bm25 => { + self.source + .full_text_coverage(&spec.table.type_key, &ranked.property) + == FullTextCoverage::Full + } + _ => true, + }); let ranked_type = match self.physical.node(scan) { Some(PhysicalNode::Scan { spec, .. }) => spec.table.node_type_name(), _ => None, @@ -2198,6 +2211,8 @@ impl Lowering<'_> { ranked_type, hops: Vec::new(), feeds, + on_empty, + coverage_admits, }); } let mut introduced_by_scan = false; @@ -2240,6 +2255,8 @@ impl Lowering<'_> { ranked_type, hops, feeds, + on_empty, + coverage_admits, }) } @@ -2562,6 +2579,7 @@ impl Lowering<'_> { overfetch: fetch.map(OverfetchRung::ladder).unwrap_or_default(), prefilter: None, eligibility: Eligibility::BeforeScoring, + policy: Some(NearestPolicy::DEFAULT), }; let (score, scan) = self.rank(lowered, binding, access)?; let top = top_level(&self.physical, lowered); @@ -2569,7 +2587,13 @@ impl Lowering<'_> { matches!(self.physical.node(*id), Some(PhysicalNode::Expand { src, .. }) if src == binding) }); if expanded_from { - let prefilter = self.prefilter(&top, binding, scan, vec![scan])?; + let prefilter = self.prefilter( + &top, + binding, + scan, + vec![scan], + EmptyEligible::ProvenEmpty, + )?; if let Some(PhysicalNode::Scan { ranked: Some(ranked), .. @@ -2599,6 +2623,7 @@ impl Lowering<'_> { overfetch: Vec::new(), prefilter: None, eligibility: Eligibility::BeforeScoring, + policy: None, }; let (score, scan) = self.rank(lowered, binding, access)?; if let Some(PhysicalNode::Scan { @@ -2653,6 +2678,10 @@ impl Lowering<'_> { overfetch: Vec::new(), prefilter: None, eligibility: Eligibility::BeforeScoring, + policy: match arm.kind { + RankKind::Nearest => Some(NearestPolicy::DEFAULT), + RankKind::Bm25 => None, + }, }; let (_, scan) = self.rank(root, &arm.binding, access)?; if arm.kind == RankKind::Bm25 { @@ -2677,6 +2706,7 @@ impl Lowering<'_> { &primary_arm.binding, primary_scan, feeds, + EmptyEligible::Postfilter, )?; let [primary_arm, secondary_arm] = <[RankArm; 2]>::try_from(lowered_arms) .map_err(|_| PlanError::Internal("an rrf has two arms".to_string()))?; diff --git a/crates/omnigraph-planner/src/physical.rs b/crates/omnigraph-planner/src/physical.rs index 3e516e9e0..063fdcd7d 100644 --- a/crates/omnigraph-planner/src/physical.rs +++ b/crates/omnigraph-planner/src/physical.rs @@ -192,12 +192,59 @@ pub struct Hop { /// ids of `ranked_type` that have every hop in `hops` and ANDs them into the /// scans `feeds` names as `id IN (...)`, when the gate policy admits the set. /// Empty `hops` admits nothing: the run records the shape fallback and the -/// scans run as planned. +/// scans run as planned. Every hop is a required first hop of the query, so +/// the eligible set holds every row that can survive the traversal. #[derive(Debug, Clone, PartialEq, Eq)] pub struct Prefilter { pub ranked_type: String, pub hops: Vec, pub feeds: Vec, + /// What an empty eligible set decides: the empty answer for a + /// standalone `nearest`, the unfiltered plan for a fusion. + pub on_empty: EmptyEligible, + /// Whether the recorded full-text coverage of every `bm25` scan in + /// `feeds` is full, the guard under which prefiltering keeps BM25 + /// scores; true when it feeds no `bm25` scan. + pub coverage_admits: bool, +} + +/// What a pre-pass's empty eligible set decides. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum EmptyEligible { + /// No row survives the traversal: the answer is empty and no scan runs. + ProvenEmpty, + /// The scans run unfiltered. + Postfilter, +} + +/// The adaptive policy of a `nearest` scan, declared before it runs: what +/// the scan may do after an attempt leaves it short of `fetch`. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +pub struct NearestPolicy { + /// A short scan under a probe cap rescans under the cap times this, + /// and uncapped once that reaches the ranked partitions. + pub probe_factor: usize, + /// Rows the prefilter admitted at `_distance = +inf` (the partition + /// search did not reach them) are rescanned flat and exact. + pub flat_rescan_on_unreached: bool, + /// A short capped scan without Lance's partition counters rescans + /// uncapped (fail closed). + pub uncapped_on_missing_counters: bool, + /// A scan whose eligible set is no larger than `fetch` scores flat from + /// its first attempt. + pub flat_when_eligible_within_fetch: bool, +} + +impl NearestPolicy { + /// The policy the planner declares: probe caps grow fourfold (20 → 80 → + /// 320 → none), and every fallback is on. + pub const DEFAULT: Self = Self { + probe_factor: 4, + flat_rescan_on_unreached: true, + uncapped_on_missing_counters: true, + flat_when_eligible_within_fetch: true, + }; } impl Prefilter { @@ -282,6 +329,8 @@ pub struct RankedAccess { /// `nearest` scan, whose candidate window is drawn from eligible rows; /// on a `bm25` scan before scoring only under recorded full coverage. pub eligibility: Eligibility, + /// The adaptive policy of a `nearest` scan; `None` on a `bm25` scan. + pub policy: Option, } impl RankedAccess { @@ -306,7 +355,10 @@ impl RankedAccess { "scope": self.scope, }); match self.kind { - RankKind::Nearest => value["nprobes"] = json!(self.nprobes), + RankKind::Nearest => { + value["nprobes"] = json!(self.nprobes); + value["policy"] = json!(self.policy); + } RankKind::Bm25 => value["eligibility"] = json!(self.eligibility), } value diff --git a/crates/omnigraph-planner/src/validate/invariants.rs b/crates/omnigraph-planner/src/validate/invariants.rs index 441516692..010d134ce 100644 --- a/crates/omnigraph-planner/src/validate/invariants.rs +++ b/crates/omnigraph-planner/src/validate/invariants.rs @@ -25,8 +25,8 @@ use crate::logical::{ColumnRef, EDGE_TYPE_MEMBER, IDENTITY_MEMBER}; use crate::lower::ContainsJoinFields; use crate::optimizer::{RRF_NEAREST_ARM_K, derived_order}; use crate::physical::{ - Assumptions, Eligibility, NodeId, OverfetchRung, PhysicalNode, PhysicalPlan, RankKind, - RankScope, RankedAccess, + Assumptions, Eligibility, EmptyEligible, NodeId, OverfetchRung, PhysicalNode, PhysicalPlan, + RankKind, RankScope, RankedAccess, }; use crate::source::FullTextCoverage; @@ -140,6 +140,7 @@ impl Requirements { self.check_search(plan, &matcher, budget)?; self.check_returns(plan, &top, &matcher, budget)?; self.check_order_and_cut(plan, &top, &matcher, budget)?; + check_policies(plan, budget)?; Ok(()) } @@ -870,6 +871,173 @@ impl Requirements { const SYSTEM_ID: &str = "@id"; +/// The adaptive search policies the plan declares (RFC 0047, "Execution and +/// replay use the same acceptance path"): every `nearest` scan's probe +/// ladder terminates and keeps its correctness fallbacks, and every +/// pre-pass draws its eligible set from required first hops, feeds only the +/// scans its kind may prefilter, decides an empty set as its kind must, and +/// guards BM25 scans by their recorded coverage. The gate policy's +/// thresholds are finite. +fn check_policies(plan: &PhysicalPlan, budget: &mut Budget) -> Result<(), ValidationError> { + let policy = plan.assumptions().gate_policy; + if !policy.ratio.is_finite() || policy.ratio < 0.0 { + return Err(ValidationError::violated( + "declared policy", + format!( + "the gate admits by ratio {}, which is no finite share", + policy.ratio + ), + )); + } + for (id, node) in plan.live() { + budget.visit(1)?; + match node { + PhysicalNode::Scan { + ranked: Some(access), + spec, + .. + } => { + match (access.kind, access.policy) { + (RankKind::Nearest, Some(policy)) => { + if policy.probe_factor < 2 + || !policy.flat_rescan_on_unreached + || !policy.uncapped_on_missing_counters + { + return Err(ValidationError::violated( + "declared policy", + format!( + "the nearest scan's policy {policy:?} lets its probe ladder stall or drops a fallback that keeps its rows in order" + ), + )); + } + } + (RankKind::Bm25, None) => {} + (kind, policy) => { + return Err(ValidationError::violated( + "declared policy", + format!("a {kind:?} scan declares the nearest policy {policy:?}"), + )); + } + } + if let Some(prefilter) = &access.prefilter { + if access.kind != RankKind::Nearest + || access.scope != RankScope::Order + || prefilter.feeds != [id] + || prefilter.on_empty != EmptyEligible::ProvenEmpty + { + return Err(ValidationError::violated( + "declared policy", + "only a standalone nearest scan carries its own pre-pass, feeding itself and proving an empty eligible set empty", + )); + } + let binding = spec.binding.as_deref().unwrap_or_default(); + check_prefilter(plan, plan.root(), binding, prefilter, budget)?; + } + } + PhysicalNode::RankFuse { + arms, prefilter, .. + } => { + if prefilter.on_empty != EmptyEligible::Postfilter { + return Err(ValidationError::violated( + "declared policy", + "a fusion's empty eligible set proves nothing; its arms run unfiltered", + )); + } + let mut admits = true; + for feed in &prefilter.feeds { + match plan.node(*feed) { + Some(PhysicalNode::Scan { + spec, + ranked: Some(ranked), + .. + }) if ranked.kind == RankKind::Bm25 => { + admits &= + plan.assumptions() + .full_text + .get(&Assumptions::full_text_key( + &spec.table.type_key, + &ranked.property, + )) + == Some(&FullTextCoverage::Full); + } + _ => { + return Err(ValidationError::violated( + "declared policy", + "a fusion's pre-pass feeds only its bm25 arms", + )); + } + } + } + if admits != prefilter.coverage_admits { + return Err(ValidationError::violated( + "prerequisite", + format!( + "the fusion's pre-pass claims coverage admits {}; the recorded coverage of its bm25 arms admits {admits}", + prefilter.coverage_admits + ), + )); + } + check_prefilter(plan, arms[0].input, &arms[0].binding, prefilter, budget)?; + } + _ => {} + } + } + Ok(()) +} + +/// Every hop of a pre-pass is a required first hop of `binding` in the +/// pipeline under `root`: a top-level traversal leaving the scanned binding +/// over that one edge type in that direction, at least one hop long. Such +/// an eligible set holds every row that can survive the traversal. +fn check_prefilter( + plan: &PhysicalPlan, + root: NodeId, + binding: &str, + prefilter: &crate::physical::Prefilter, + budget: &mut Budget, +) -> Result<(), ValidationError> { + if prefilter.hops.is_empty() { + return Ok(()); + } + let nodes = pipeline(plan, root, budget)?; + let scanned = nodes.iter().any(|id| { + matches!(plan.node(*id), Some(PhysicalNode::Scan { spec, .. }) + if spec.binding.as_deref() == Some(binding) + && spec.table.node_type_name() == Some(prefilter.ranked_type.as_str())) + }); + let introduced = nodes.iter().any( + |id| matches!(plan.node(*id), Some(PhysicalNode::Expand { dst, .. }) if dst == binding), + ); + if !scanned || introduced { + return Err(ValidationError::violated( + "declared policy", + format!( + "the pre-pass of `${binding}` filters a binding no scan of its type introduces" + ), + )); + } + for hop in &prefilter.hops { + let required = nodes.iter().any(|id| { + matches!(plan.node(*id), Some(PhysicalNode::Expand { src, edges, min_hops, .. }) + if src == binding + && *min_hops > 0 + && edges.named().is_some_and(|member| { + member.edge_type == hop.edge_type && member.direction == hop.direction + })) + }); + if !required { + return Err(ValidationError::violated( + "declared policy", + format!( + "the pre-pass of `${binding}` requires a {} {:?} hop no traversal of the query requires", + hop.edge_type, hop.direction + ), + )); + } + } + Ok(()) +} + /// Every node of `top` that projects the query's `return`. fn projections_of<'p>(plan: &'p PhysicalPlan, top: &[NodeId]) -> Vec<&'p Vec> { top.iter() diff --git a/crates/omnigraph-planner/src/validate/tests.rs b/crates/omnigraph-planner/src/validate/tests.rs index bc0fe1d1a..22cfe5d42 100644 --- a/crates/omnigraph-planner/src/validate/tests.rs +++ b/crates/omnigraph-planner/src/validate/tests.rs @@ -24,7 +24,9 @@ node Doc { title: String year: I64 open: Bool + embedding: Vector(2) } +edge Cites: Doc -> Doc "#; /// The bound-literal evaluator: literals and bound parameters through the @@ -67,7 +69,7 @@ impl Fixture { let checked = CheckedQuery::check(&catalog, &decl).unwrap(); let ir = omnigraph_compiler::lower_query(&catalog, checked.decl(), checked.types()).unwrap(); - let mut source = MemorySource::default(); + let mut source = MemorySource::default().with_edge_version("Cites", 1); for (name, node_type) in &catalog.node_types { source = source.with_node_type( name, @@ -477,3 +479,94 @@ fn an_exhausted_budget_is_a_resource_outcome() { } ); } + +const NEAREST: &str = r#"query q($v: Vector(2)) { + match { $d: Doc $d cites $e } + return { $d.slug } + order { nearest($d.embedding, $v) } + limit 3 +}"#; + +fn nearest() -> Fixture { + Fixture::new( + NEAREST, + &[( + "v", + Literal::List(vec![Literal::Float(0.0), Literal::Float(1.0)]), + )], + ) +} + +/// The ranked scan of `plan` and its access, mutably. +fn ranked_access(plan: &mut PhysicalPlan) -> &mut crate::physical::RankedAccess { + let id = node_ids(plan, |node| node.ranked().is_some())[0]; + match plan.node_mut(id) { + Some(PhysicalNode::Scan { + ranked: Some(ranked), + .. + }) => ranked, + _ => unreachable!("the id names a ranked scan"), + } +} + +#[test] +fn a_nearest_plan_declares_its_policy_and_prepass() { + let fixture = nearest(); + let mut plan = fixture.plan(); + let access = ranked_access(&mut plan).clone(); + assert_eq!(access.policy, Some(crate::physical::NearestPolicy::DEFAULT)); + let prefilter = access + .prefilter + .expect("the edge from `$d` declares a pre-pass"); + assert_eq!( + prefilter.on_empty, + crate::physical::EmptyEligible::ProvenEmpty + ); + assert!(prefilter.coverage_admits); + assert_eq!(prefilter.hops.len(), 1); + fixture + .accept(plan) + .expect("the declared policy is accepted"); +} + +#[test] +fn a_stalling_or_unguarded_ladder_fails_the_declared_policy() { + let fixture = nearest(); + let breaks: [fn(&mut crate::physical::RankedAccess); 4] = [ + |access| access.policy.as_mut().unwrap().probe_factor = 1, + |access| access.policy.as_mut().unwrap().flat_rescan_on_unreached = false, + |access| access.policy.as_mut().unwrap().uncapped_on_missing_counters = false, + |access| access.policy = None, + ]; + for edit in breaks { + let mut plan = fixture.plan(); + edit(ranked_access(&mut plan)); + let (check, detail) = fixture.refused(plan); + assert_eq!(check, "declared policy", "{detail}"); + } +} + +#[test] +fn a_prepass_off_the_required_hops_fails_the_declared_policy() { + let fixture = nearest(); + let breaks: [fn(&mut crate::physical::Prefilter); 3] = [ + |prefilter| prefilter.hops[0].edge_type = "Other".to_string(), + |prefilter| prefilter.on_empty = crate::physical::EmptyEligible::Postfilter, + |prefilter| prefilter.feeds.push(usize::MAX), + ]; + for edit in breaks { + let mut plan = fixture.plan(); + edit(ranked_access(&mut plan).prefilter.as_mut().unwrap()); + let (check, detail) = fixture.refused(plan); + assert_eq!(check, "declared policy", "{detail}"); + } +} + +#[test] +fn a_bm25_scan_with_a_nearest_policy_fails_the_declared_policy() { + let fixture = ranked(); + let mut plan = fixture.plan(); + ranked_access(&mut plan).policy = Some(crate::physical::NearestPolicy::DEFAULT); + let (check, detail) = fixture.refused(plan); + assert_eq!(check, "declared policy", "{detail}"); +} diff --git a/crates/omnigraph-planner/tests/lower_walk.rs b/crates/omnigraph-planner/tests/lower_walk.rs index 6220f04bc..c3593156c 100644 --- a/crates/omnigraph-planner/tests/lower_walk.rs +++ b/crates/omnigraph-planner/tests/lower_walk.rs @@ -2,6 +2,7 @@ //! claim is the call sequence an engine sees, which no query result shows. use omnigraph_compiler::ir::{IRExpr, IROrdering, IRProjection}; +use omnigraph_planner::EmptyEligible; use omnigraph_planner::{ ColumnRef, ContainsJoinFields, ExpandFields, HashJoinFields, Lower, NodeId, PhysicalNode, PhysicalPlan, PlanError, Prefilter, RankArm, RankFuseFields, RankKind, RankedAccess, ScanInput, @@ -13,6 +14,8 @@ fn no_prefilter() -> Prefilter { ranked_type: "Doc".to_string(), hops: Vec::new(), feeds: Vec::new(), + on_empty: EmptyEligible::Postfilter, + coverage_admits: true, } } diff --git a/crates/omnigraph-planner/tests/query_plan/explain.rs b/crates/omnigraph-planner/tests/query_plan/explain.rs index bf4cb99f7..23be53b47 100644 --- a/crates/omnigraph-planner/tests/query_plan/explain.rs +++ b/crates/omnigraph-planner/tests/query_plan/explain.rs @@ -295,6 +295,12 @@ fn the_physical_document_prints_gq_orderings_and_no_query_schema() { "fetch": 10, "nprobes": null, "scope": "order", + "policy": { + "probe_factor": 4, + "flat_rescan_on_unreached": true, + "uncapped_on_missing_counters": true, + "flat_when_eligible_within_fetch": true, + }, }) ); assert_eq!( diff --git a/crates/omnigraph/src/engine/lower.rs b/crates/omnigraph/src/engine/lower.rs index 0782621b9..1f4ff590f 100644 --- a/crates/omnigraph/src/engine/lower.rs +++ b/crates/omnigraph/src/engine/lower.rs @@ -161,6 +161,7 @@ impl<'a> Lowering<'a> { None => ranked.nprobes, }; mode.nearest_exact = rung.is_some_and(|rung| rung.exact); + mode.nearest_policy = ranked.policy; } RankKind::Bm25 => { let text = resolve_to_string(&ranked.query, self.params()).ok_or_else(|| { diff --git a/crates/omnigraph/src/engine/mod.rs b/crates/omnigraph/src/engine/mod.rs index d605d6b40..922715d9f 100644 --- a/crates/omnigraph/src/engine/mod.rs +++ b/crates/omnigraph/src/engine/mod.rs @@ -178,10 +178,11 @@ async fn run_once( let lowered = lowering.lower_query(pass)?; lowered.record_in_memory_filters(); let batch = run_plan(&lowered, lowering.plan, ctx).await?; - let report = *lowered + let report = lowered .report .lock() - .unwrap_or_else(std::sync::PoisonError::into_inner); + .unwrap_or_else(std::sync::PoisonError::into_inner) + .clone(); let rows = pass_rows(&lowered, lowering.plan, rung)?; Ok((batch, report, rows)) } @@ -194,10 +195,10 @@ struct NearestScan<'p> { overfetch: &'p [OverfetchRung], } -/// The two arms of an `rrf()` in arm order, and the pre-pass their -/// `RankFuse` declares. +/// An `rrf()` with both its arms: the `RankFuse` node and the pre-pass it +/// declares. struct Fusion<'p> { - arms: [ArmTarget<'p>; 2], + id: NodeId, prefilter: &'p Prefilter, } @@ -209,11 +210,11 @@ struct RankedScans<'p> { fn ranked_scans(plan: &PhysicalPlan) -> RankedScans<'_> { let mut nearest = None; - let mut arms: [Option>; 2] = [None, None]; + let mut arms = [false, false]; let mut fuse = None; for (id, node) in plan.live() { if let PhysicalNode::RankFuse { prefilter, .. } = node { - fuse = Some(prefilter); + fuse = Some((id, prefilter)); continue; } let PhysicalNode::Scan { @@ -223,10 +224,6 @@ fn ranked_scans(plan: &PhysicalPlan) -> RankedScans<'_> { else { continue; }; - let target = ArmTarget { - kind: ranked.kind, - property: &ranked.property, - }; match ranked.scope { RankScope::Order => { if ranked.kind == RankKind::Nearest && ranked.fetch.is_some() { @@ -237,17 +234,14 @@ fn ranked_scans(plan: &PhysicalPlan) -> RankedScans<'_> { }); } } - RankScope::Primary => arms[0] = Some(target), - RankScope::Secondary => arms[1] = Some(target), + RankScope::Primary => arms[0] = true, + RankScope::Secondary => arms[1] = true, } } RankedScans { nearest, fusion: match (arms, fuse) { - ([Some(primary), Some(secondary)], Some(prefilter)) => Some(Fusion { - arms: [primary, secondary], - prefilter, - }), + ([true, true], Some((id, prefilter))) => Some(Fusion { id, prefilter }), _ => None, }, } @@ -427,8 +421,14 @@ pub(crate) async fn execute( let policy = bound.plan.assumptions().gate_policy; let lowering = Lowering::new(&bound, context); let RankedScans { nearest, fusion } = ranked_scans(&bound.plan); - if let Some(Fusion { arms, prefilter }) = fusion { - let eligible = Box::pin(rrf_prefilter_gate(context, arms, prefilter, policy)).await; + if let Some(Fusion { id, prefilter }) = fusion { + let (eligible, verdict) = Box::pin(rrf_prefilter_gate(context, prefilter, policy)).await; + let plan = if eligible.is_some() { + "prefilter" + } else { + "postfilter" + }; + executed.decide(id, 0, report::Taken::gate(plan, &verdict)); let mut pass = Pass::default(); if let Some(ids) = eligible { for id in &prefilter.feeds { @@ -454,15 +454,29 @@ pub(crate) async fn execute( .. }) = &nearest { - pass = match Box::pin(nearest_prefilter_gate(context, prefilter, policy)).await { - NearestGatePlan::Prefilter(ids) => pass.prefiltered(*id, ids), - NearestGatePlan::Postfilter => pass, - NearestGatePlan::ProvenEmpty => pass.proven_empty(*id), + let (plan, verdict) = Box::pin(nearest_prefilter_gate(context, prefilter, policy)).await; + let (taken, next) = match plan { + NearestGatePlan::Prefilter(ids) => ("prefilter", pass.prefiltered(*id, ids)), + NearestGatePlan::Postfilter => ("postfilter", pass), + NearestGatePlan::ProvenEmpty => ("proven_empty", pass.proven_empty(*id)), }; + executed.decide(*id, 0, report::Taken::gate(taken, &verdict)); + pass = next; } let (result_batch, report, rows) = Box::pin(run_once(&lowering, &ctx, &pass, 0)).await?; executed.record(rows); + if let Some(NearestScan { id, .. }) = &nearest + && !report.probes.is_empty() + { + executed.decide( + *id, + 0, + report::Taken::Probes { + attempts: report.probes.clone(), + }, + ); + } let mut result_batch = result_batch; let mut report = report; let aggregate = bound @@ -528,6 +542,15 @@ pub(crate) async fn execute( let (retried, retried_report, rows) = Box::pin(run_once(&lowering, &ctx, &wider, rung)).await?; executed.record(rows); + if !retried_report.probes.is_empty() { + executed.decide( + id, + rung, + report::Taken::Probes { + attempts: retried_report.probes.clone(), + }, + ); + } result_batch = retried; report = retried_report; if let PassStep::Exact { k } = step { @@ -712,6 +735,8 @@ mod traversal_admission_tests { direction: Direction::Out, }], feeds, + on_empty: omnigraph_planner::EmptyEligible::ProvenEmpty, + coverage_admits: true, }; let mut plan = selected_plan(); let input = 0; @@ -748,6 +773,7 @@ mod traversal_admission_tests { overfetch: vec![], prefilter: None, eligibility: omnigraph_planner::Eligibility::BeforeScoring, + policy: Some(omnigraph_planner::NearestPolicy::DEFAULT), }), }; validate_traversal_admission(&plan).unwrap(); diff --git a/crates/omnigraph/src/engine/report.rs b/crates/omnigraph/src/engine/report.rs index 0b19171df..a6113b854 100644 --- a/crates/omnigraph/src/engine/report.rs +++ b/crates/omnigraph/src/engine/report.rs @@ -89,9 +89,67 @@ impl ReportRow { } } +/// One adaptive search decision the run took inside the policy its plan +/// declares: a pre-pass gate's verdict with the facts it read, or the probe +/// attempts of a `nearest` scan, in the overfetch pass `rung`. +#[derive(Debug, Clone, PartialEq, Eq, Serialize)] +pub(crate) struct SearchDecision { + /// The ranked scan or fusion the decision belongs to. + pub(crate) id: NodeId, + pub(crate) rung: usize, + #[serde(flatten)] + pub(crate) taken: Taken, +} + +/// What a [`SearchDecision`] took. +#[derive(Debug, Clone, PartialEq, Eq, Serialize)] +#[serde(tag = "decision", rename_all = "snake_case")] +pub(crate) enum Taken { + /// The gate's plan (`prefilter`, `postfilter` or `proven_empty`), the + /// fallback that decided a postfilter, and the counts it read. + Gate { + plan: &'static str, + fallback: Option<&'static str>, + forced: bool, + eligible: Option, + corpus: Option, + }, + /// Every attempt of the scan's probe ladder, in order. + Probes { + attempts: Vec, + }, +} + +impl Taken { + /// The report form of a gate verdict under the plan the gate chose. + pub(super) fn gate( + plan: &'static str, + verdict: &crate::instrumentation::RrfGateVerdict, + ) -> Self { + use crate::instrumentation::RrfGateFallback; + Self::Gate { + plan, + fallback: verdict.fallback.map(|fallback| match fallback { + RrfGateFallback::Threshold => "threshold", + RrfGateFallback::Shape => "shape", + RrfGateFallback::Coverage => "coverage", + RrfGateFallback::BuildErr => "build_error", + RrfGateFallback::EmptyEligible => "empty_eligible", + RrfGateFallback::Forced => "forced", + }), + forced: verdict.forced, + eligible: verdict.eligible, + corpus: verdict.corpus, + } + } +} + #[derive(Debug, Default, Clone, PartialEq, Eq, Serialize)] pub struct ExecutionReport { rows: Vec, + /// The adaptive search decisions of the run, in the order taken. + #[serde(skip_serializing_if = "Vec::is_empty")] + search: Vec, } impl ExecutionReport { @@ -105,6 +163,11 @@ impl ExecutionReport { self.rows.iter().find(|row| row.id == id) } + /// Record one adaptive search decision. + pub(super) fn decide(&mut self, id: NodeId, rung: usize, taken: Taken) { + self.search.push(SearchDecision { id, rung, taken }); + } + /// Fold one pass in: a node met before gains the pass's attempt, and is /// `executed` once any pass polled its operator. pub(super) fn record(&mut self, pass: Vec) { diff --git a/crates/omnigraph/src/engine/scan.rs b/crates/omnigraph/src/engine/scan.rs index 7a1b302cc..b3c0bd24f 100644 --- a/crates/omnigraph/src/engine/scan.rs +++ b/crates/omnigraph/src/engine/scan.rs @@ -235,11 +235,20 @@ pub(super) async fn execute_node_scan( ), _ => None, }; - let mut use_index = match (nearest_target.as_ref(), known_matches) { - (Some(_), _) if search_mode.nearest_exact => false, - (Some((_, _, k)), Some(matches)) => matches > *k, + let policy = match nearest_target { + Some(_) => Some(search_mode.nearest_policy.ok_or_else(|| { + OmniError::manifest_internal("a nearest scan carries no declared policy") + })?), + None => None, + }; + let mut use_index = match (nearest_target.as_ref(), known_matches, policy) { + (Some(_), _, _) if search_mode.nearest_exact => false, + (Some((_, _, k)), Some(matches), Some(policy)) => { + !(policy.flat_when_eligible_within_fetch && matches <= *k) + } _ => true, }; + scan_report.probes.clear(); let mut last_rung: Option<(usize, u64)> = None; let mut final_summary: Option<(Option, Option)> = None; let (batches, attempt_memory) = loop { @@ -319,10 +328,19 @@ pub(super) async fn execute_node_scan( exhausted: !use_index || known_matches.is_some_and(|matches| rows >= matches), dataset_rows: type_rows, }); + scan_report.probes.push(ProbeAttempt { + maximum_nprobes: probe_budget, + flat: !use_index, + rows, + }); if !use_index { break (batches, attempt_memory); } + let Some(policy) = policy else { + break (batches, attempt_memory); + }; match ladder_step( + policy, rows, *k, known_matches, diff --git a/crates/omnigraph/src/engine/search.rs b/crates/omnigraph/src/engine/search.rs index 1db1cf4de..c751f91fe 100644 --- a/crates/omnigraph/src/engine/search.rs +++ b/crates/omnigraph/src/engine/search.rs @@ -3,9 +3,7 @@ //! values; the per-pass `Pass` the overfetch ladder and the prefilter gates //! write; the gates and probe ladders themselves. -use omnigraph_planner::{ - GatePolicy, Hop, NodeId, OverfetchRung, Prefilter, PrefilterMode, RankKind, -}; +use omnigraph_planner::{GatePolicy, Hop, NodeId, OverfetchRung, Prefilter, PrefilterMode}; use super::*; @@ -22,6 +20,8 @@ pub(super) struct SearchMode { /// The nearest scan runs flat (`use_index(false)`, every row scored): the /// overfetch loop's exact pass. pub(super) nearest_exact: bool, + /// The adaptive policy the plan declares for the nearest scan. + pub(super) nearest_policy: Option, /// The nearest prefilter gate proved the answer empty: no node of the /// ranked type satisfies an Expand's first hop. Read off the `Pass` by /// `Lowering::search_mode`; an RRF arm's pass never carries it. @@ -133,14 +133,6 @@ impl Pass { } } -/// What the `rrf` gate reads of one arm of an `rrf()`: the index it ranks -/// with and the ranked property, whose FTS coverage the gate checks. -#[derive(Debug, Clone, Copy)] -pub(super) struct ArmTarget<'a> { - pub(super) kind: RankKind, - pub(super) property: &'a str, -} - /// Shared eligible-id set, `Debug`-opaque so a logged `SearchMode` prints the /// cardinality instead of up to `GatePolicy::max_ids` id strings. #[derive(Clone)] @@ -155,9 +147,23 @@ impl std::fmt::Debug for EligibleIds { /// What the nearest scan reported back to the query level: the LAST scan of /// the ranked variable in a pass. `rows == k` means the scan was full, so a /// shortfall above it can only be recovered by asking for more candidates. -#[derive(Debug, Default, Clone, Copy)] +#[derive(Debug, Default, Clone)] pub(super) struct ScanReport { pub(super) nearest_scan: Option, + /// Every Lance attempt of that nearest scan in this pass, in order: the + /// probe cap it ran under and whether it scored flat. + pub(super) probes: Vec, +} + +/// One attempt of a nearest scan's probe ladder. +#[derive(Debug, Clone, Copy, PartialEq, Eq, serde::Serialize)] +pub(crate) struct ProbeAttempt { + /// The probe cap; `None` is uncapped. + pub(crate) maximum_nprobes: Option, + /// The attempt scored every admitted row (`use_index(false)`). + pub(crate) flat: bool, + /// Rows the attempt returned. + pub(crate) rows: usize, } /// One nearest scan as `execute_node_scan`'s ladder left it. @@ -401,7 +407,8 @@ pub(crate) fn check_param_date_literals( /// - the eligible set MUST over-approximate the traversal's survivors (a /// superset only costs speedup; a subset changes answers) — every /// shape the planner's `prefilter` admits is an instance; -/// - full FTS fragment coverage (uncovered fragments are scored +/// - full FTS fragment coverage, the plan's recorded fact +/// (`Prefilter::coverage_admits`; uncovered fragments are scored /// filter-dependently, so a mask would change their scores); /// - `nearest` arms are never prefiltered (their constitutive `k` makes a /// prefiltered run answer-different) — the caller's threading rule; @@ -415,10 +422,9 @@ pub(crate) fn check_param_date_literals( /// `rrf_gate_verdicts` probe entry. pub(super) async fn rrf_prefilter_gate( context: &EngineContext<'_>, - arms: [ArmTarget<'_>; 2], prefilter: &Prefilter, policy: GatePolicy, -) -> Option { +) -> (Option, RrfGateVerdict) { let fall_back = |fallback: RrfGateFallback, forced: bool, eligible: Option, corpus: Option| { tracing::debug!( @@ -426,53 +432,33 @@ pub(super) async fn rrf_prefilter_gate( forced, "rrf prefilter gate fell back to the postfilter plan" ); - record_rrf_gate_verdict(RrfGateVerdict { + let verdict = RrfGateVerdict { plan: RrfGatePlan::Postfilter, fallback: Some(fallback), forced, eligible, corpus, - }); + }; + record_rrf_gate_verdict(verdict.clone()); + (None, verdict) }; if policy.mode == PrefilterMode::ForcePostfilter { - fall_back(RrfGateFallback::Forced, true, None, None); - return None; + return fall_back(RrfGateFallback::Forced, true, None, None); } let forced = policy.mode == PrefilterMode::ForcePrefilter; if !prefilter.admits() { - fall_back(RrfGateFallback::Shape, forced, None, None); - return None; + return fall_back(RrfGateFallback::Shape, forced, None, None); } - let bm25_props: Vec<&str> = arms - .iter() - .filter(|arm| arm.kind == RankKind::Bm25) - .map(|arm| arm.property) - .collect(); let ranked_type = prefilter.ranked_type.as_str(); let node_key = format!("node:{ranked_type}"); let snapshot = context.snapshot; let Some(node_entry) = snapshot.dataset(&node_key) else { - fall_back(RrfGateFallback::Shape, forced, None, None); - return None; + return fall_back(RrfGateFallback::Shape, forced, None, None); }; let corpus = node_entry.entity_count; - match snapshot.open_lance_dataset(&node_key).await { - Ok(ds) => { - for prop in &bm25_props { - match crate::table_store::TableStore::fts_covers_all_fragments(&ds, prop).await { - Ok(true) => {} - Ok(false) | Err(_) => { - fall_back(RrfGateFallback::Coverage, forced, None, Some(corpus)); - return None; - } - } - } - } - Err(_) => { - fall_back(RrfGateFallback::Coverage, forced, None, Some(corpus)); - return None; - } + if !prefilter.coverage_admits { + return fall_back(RrfGateFallback::Coverage, forced, None, Some(corpus)); } #[cfg_attr(not(debug_assertions), allow(unused_mut))] @@ -489,22 +475,22 @@ pub(super) async fn rrf_prefilter_gate( { EligibleOutcome::Ids { ids, eligible } => (ids, eligible), EligibleOutcome::FallBack { fallback, eligible } => { - fall_back(fallback, forced, eligible, Some(corpus)); - return None; + return fall_back(fallback, forced, eligible, Some(corpus)); } }; #[cfg(debug_assertions)] if let Some(dropped) = crate::instrumentation::rrf_gate_subset_drop() { ids.retain(|id| *id != dropped); } - record_rrf_gate_verdict(RrfGateVerdict { + let verdict = RrfGateVerdict { plan: RrfGatePlan::Prefilter, fallback: None, forced, eligible: Some(eligible_count), corpus: Some(corpus), - }); - Some(EligibleIds(Arc::new(ids))) + }; + record_rrf_gate_verdict(verdict.clone()); + (Some(EligibleIds(Arc::new(ids))), verdict) } /// Outcome of the adjacency eligible-id computation shared by the rrf and @@ -622,7 +608,7 @@ pub(super) async fn nearest_prefilter_gate( context: &EngineContext<'_>, prefilter: &Prefilter, policy: GatePolicy, -) -> NearestGatePlan { +) -> (NearestGatePlan, RrfGateVerdict) { let forced = policy.mode != PrefilterMode::Auto; let fall_back = |fallback: RrfGateFallback, eligible: Option, corpus: Option| { tracing::debug!( @@ -630,27 +616,29 @@ pub(super) async fn nearest_prefilter_gate( forced, "nearest prefilter gate fell back to the unfiltered scan" ); - record_ann_prefilter_verdict(RrfGateVerdict { + let verdict = RrfGateVerdict { plan: RrfGatePlan::Postfilter, fallback: Some(fallback), forced, eligible, corpus, - }); + }; + record_ann_prefilter_verdict(verdict.clone()); + verdict }; if policy.mode == PrefilterMode::ForcePostfilter { - fall_back(RrfGateFallback::Forced, None, None); - return NearestGatePlan::Postfilter; + let verdict = fall_back(RrfGateFallback::Forced, None, None); + return (NearestGatePlan::Postfilter, verdict); } if !prefilter.admits() { - fall_back(RrfGateFallback::Shape, None, None); - return NearestGatePlan::Postfilter; + let verdict = fall_back(RrfGateFallback::Shape, None, None); + return (NearestGatePlan::Postfilter, verdict); } let ranked_type = prefilter.ranked_type.as_str(); let node_key = format!("node:{ranked_type}"); let Some(node_entry) = context.snapshot.dataset(&node_key) else { - fall_back(RrfGateFallback::Shape, None, None); - return NearestGatePlan::Postfilter; + let verdict = fall_back(RrfGateFallback::Shape, None, None); + return (NearestGatePlan::Postfilter, verdict); }; let corpus = node_entry.entity_count; match adjacency_eligible_ids( @@ -665,22 +653,28 @@ pub(super) async fn nearest_prefilter_gate( .await { EligibleOutcome::Ids { ids, eligible } => { - record_ann_prefilter_verdict(RrfGateVerdict { + let verdict = RrfGateVerdict { plan: RrfGatePlan::Prefilter, fallback: None, forced, eligible: Some(eligible), corpus: Some(corpus), - }); - NearestGatePlan::Prefilter(EligibleIds(Arc::new(ids))) + }; + record_ann_prefilter_verdict(verdict.clone()); + ( + NearestGatePlan::Prefilter(EligibleIds(Arc::new(ids))), + verdict, + ) } EligibleOutcome::FallBack { fallback, eligible } => { - fall_back(fallback, eligible, Some(corpus)); - if fallback == RrfGateFallback::EmptyEligible { - NearestGatePlan::ProvenEmpty - } else { - NearestGatePlan::Postfilter - } + let verdict = fall_back(fallback, eligible, Some(corpus)); + let plan = match (fallback, prefilter.on_empty) { + (RrfGateFallback::EmptyEligible, omnigraph_planner::EmptyEligible::ProvenEmpty) => { + NearestGatePlan::ProvenEmpty + } + _ => NearestGatePlan::Postfilter, + }; + (plan, verdict) } } } @@ -793,13 +787,15 @@ pub(super) fn collect_referenced_edge_names( } } -/// Per-rung multiplier of the probe ladder (20 → 80 → 320 → none). -pub(super) const ANN_PROBE_ESCALATION_FACTOR: usize = 4; - -/// The next rung of the probe ladder after `current` starved a scan: ×4, or -/// no cap once the next rung would cover the ranked partitions anyway. -pub(super) fn next_probe_budget(current: usize, partitions_ranked: usize) -> Option { - let next = current.saturating_mul(ANN_PROBE_ESCALATION_FACTOR); +/// The next rung of the probe ladder after `current` starved a scan: times +/// the plan's declared `factor`, or no cap once the next rung would cover +/// the ranked partitions anyway. +pub(super) fn next_probe_budget( + current: usize, + partitions_ranked: usize, + factor: usize, +) -> Option { + let next = current.saturating_mul(factor); (next < partitions_ranked).then_some(next) } @@ -823,10 +819,12 @@ pub(super) enum LadderStep { } /// The ladder's decision after one nearest scan under `maximum` (`None` = -/// uncapped) returned `rows` of the `k` asked for. `summary` is Lance's -/// `(partitions_searched, partitions_ranked)`, each absent on a flat plan. +/// uncapped) returned `rows` of the `k` asked for, within the plan's +/// declared `policy`. `summary` is Lance's `(partitions_searched, +/// partitions_ranked)`, each absent on a flat plan. #[allow(clippy::too_many_arguments)] pub(super) fn ladder_step( + policy: omnigraph_planner::NearestPolicy, rows: usize, k: usize, known_matches: Option, @@ -836,7 +834,7 @@ pub(super) fn ladder_step( maximum: Option, last_rung: Option<(usize, u64)>, ) -> LadderStep { - if has_infinite_distance { + if has_infinite_distance && policy.flat_rescan_on_unreached { return LadderStep::FlatRescan; } let Some(maximum) = maximum else { @@ -855,8 +853,12 @@ pub(super) fn ladder_step( Some((None, None)) => return LadderStep::Stop, Some((Some(searched), Some(ranked))) => (searched, ranked), Some((Some(_), None)) | Some((None, Some(_))) | None => { - return LadderStep::RescanUncapped { - summary_missing: true, + return if policy.uncapped_on_missing_counters { + LadderStep::RescanUncapped { + summary_missing: true, + } + } else { + LadderStep::Stop }; } }; @@ -866,7 +868,11 @@ pub(super) fn ladder_step( if last_rung == Some((rows, searched)) { return LadderStep::Stop; } - match next_probe_budget(maximum, usize::try_from(ranked).unwrap_or(usize::MAX)) { + match next_probe_budget( + maximum, + usize::try_from(ranked).unwrap_or(usize::MAX), + policy.probe_factor, + ) { Some(next) => LadderStep::Rescan(next), None => LadderStep::RescanUncapped { summary_missing: false, @@ -894,6 +900,34 @@ mod ann_probe_budget_tests { }; const IVF_SHORT: Option<(Option, Option)> = Some((Some(1), Some(1_000))); + const POLICY: omnigraph_planner::NearestPolicy = omnigraph_planner::NearestPolicy::DEFAULT; + + /// The ladder takes its factor and its fallbacks from the plan's + /// declared policy, not from a constant of its own. + #[test] + fn the_ladder_runs_the_declared_policy() { + let wider = omnigraph_planner::NearestPolicy { + probe_factor: 2, + ..POLICY + }; + assert_eq!( + ladder_step(wider, 3, 10, None, None, false, IVF_SHORT, Some(20), None), + LadderStep::Rescan(40) + ); + let no_rescan = omnigraph_planner::NearestPolicy { + flat_rescan_on_unreached: false, + uncapped_on_missing_counters: false, + ..POLICY + }; + assert_eq!( + ladder_step(no_rescan, 3, 10, None, None, true, IVF_SHORT, None, None), + LadderStep::Stop + ); + assert_eq!( + ladder_step(no_rescan, 3, 10, None, None, false, None, Some(20), None), + LadderStep::Stop + ); + } /// Rust test: no `.gqt` fixture trains an IVF index, so no case carries a probe cap. #[test] @@ -997,22 +1031,42 @@ mod ann_probe_budget_tests { /// Rust test: no `.gqt` fixture trains an IVF index, so no case ranks partitions. #[test] fn probe_ladder_multiplies_then_uncaps() { - assert_eq!(next_probe_budget(20, 1_000), Some(80)); - assert_eq!(next_probe_budget(80, 1_000), Some(320)); - assert_eq!(next_probe_budget(320, 1_000), None); - assert_eq!(next_probe_budget(20, 60), None); - assert_eq!(next_probe_budget(1, 100), Some(4)); + assert_eq!(next_probe_budget(20, 1_000, 4), Some(80)); + assert_eq!(next_probe_budget(80, 1_000, 4), Some(320)); + assert_eq!(next_probe_budget(320, 1_000, 4), None); + assert_eq!(next_probe_budget(20, 60, 4), None); + assert_eq!(next_probe_budget(1, 100, 4), Some(4)); } /// Rust test: an infinite `_distance` needs a trained IVF index; no `.gqt` fixture has one. #[test] fn ladder_rescans_flat_on_an_infinite_distance_before_every_other_stop() { assert_eq!( - ladder_step(10, 10, Some(10), Some(10), true, IVF_SHORT, None, None), + ladder_step( + POLICY, + 10, + 10, + Some(10), + Some(10), + true, + IVF_SHORT, + None, + None + ), LadderStep::FlatRescan ); assert_eq!( - ladder_step(3, 10, None, None, true, Some((None, None)), Some(20), None), + ladder_step( + POLICY, + 3, + 10, + None, + None, + true, + Some((None, None)), + Some(20), + None + ), LadderStep::FlatRescan ); } @@ -1021,7 +1075,7 @@ mod ann_probe_budget_tests { #[test] fn ladder_stops_without_a_cap() { assert_eq!( - ladder_step(3, 10, None, None, false, IVF_SHORT, None, None), + ladder_step(POLICY, 3, 10, None, None, false, IVF_SHORT, None, None), LadderStep::Stop ); } @@ -1030,7 +1084,7 @@ mod ann_probe_budget_tests { #[test] fn ladder_stops_when_the_scan_is_full() { assert_eq!( - ladder_step(10, 10, None, None, false, IVF_SHORT, Some(20), None), + ladder_step(POLICY, 10, 10, None, None, false, IVF_SHORT, Some(20), None), LadderStep::Stop ); } @@ -1039,11 +1093,31 @@ mod ann_probe_budget_tests { #[test] fn ladder_stops_when_every_known_match_is_here() { assert_eq!( - ladder_step(5, 10, Some(5), None, false, IVF_SHORT, Some(20), None), + ladder_step( + POLICY, + 5, + 10, + Some(5), + None, + false, + IVF_SHORT, + Some(20), + None + ), LadderStep::Stop ); assert_eq!( - ladder_step(4, 10, Some(5), None, false, IVF_SHORT, Some(20), None), + ladder_step( + POLICY, + 4, + 10, + Some(5), + None, + false, + IVF_SHORT, + Some(20), + None + ), LadderStep::Rescan(80) ); } @@ -1052,11 +1126,31 @@ mod ann_probe_budget_tests { #[test] fn ladder_stops_when_the_scan_holds_the_whole_dataset() { assert_eq!( - ladder_step(7, 10, None, Some(7), false, IVF_SHORT, Some(20), None), + ladder_step( + POLICY, + 7, + 10, + None, + Some(7), + false, + IVF_SHORT, + Some(20), + None + ), LadderStep::Stop ); assert_eq!( - ladder_step(7, 10, None, Some(8), false, IVF_SHORT, Some(20), None), + ladder_step( + POLICY, + 7, + 10, + None, + Some(8), + false, + IVF_SHORT, + Some(20), + None + ), LadderStep::Rescan(80) ); } @@ -1065,7 +1159,17 @@ mod ann_probe_budget_tests { #[test] fn ladder_treats_a_summary_with_neither_counter_as_a_flat_scan() { assert_eq!( - ladder_step(3, 10, None, None, false, Some((None, None)), Some(20), None), + ladder_step( + POLICY, + 3, + 10, + None, + None, + false, + Some((None, None)), + Some(20), + None + ), LadderStep::Stop ); } @@ -1075,7 +1179,7 @@ mod ann_probe_budget_tests { fn ladder_fails_closed_without_both_counters() { for summary in [None, Some((Some(1), None)), Some((None, Some(8)))] { assert_eq!( - ladder_step(3, 10, None, None, false, summary, Some(20), None), + ladder_step(POLICY, 3, 10, None, None, false, summary, Some(20), None), LadderStep::RescanUncapped { summary_missing: true }, @@ -1089,7 +1193,7 @@ mod ann_probe_budget_tests { fn ladder_stops_when_every_ranked_partition_was_searched() { for summary in [Some((Some(8), Some(8))), Some((Some(9), Some(8)))] { assert_eq!( - ladder_step(3, 10, None, None, false, summary, Some(20), None), + ladder_step(POLICY, 3, 10, None, None, false, summary, Some(20), None), LadderStep::Stop, "summary {summary:?}" ); @@ -1100,15 +1204,45 @@ mod ann_probe_budget_tests { #[test] fn ladder_stops_when_widening_changed_nothing() { assert_eq!( - ladder_step(3, 10, None, None, false, IVF_SHORT, Some(80), Some((3, 1))), + ladder_step( + POLICY, + 3, + 10, + None, + None, + false, + IVF_SHORT, + Some(80), + Some((3, 1)) + ), LadderStep::Stop ); assert_eq!( - ladder_step(3, 10, None, None, false, IVF_SHORT, Some(80), Some((3, 0))), + ladder_step( + POLICY, + 3, + 10, + None, + None, + false, + IVF_SHORT, + Some(80), + Some((3, 0)) + ), LadderStep::Rescan(320) ); assert_eq!( - ladder_step(3, 10, None, None, false, IVF_SHORT, Some(80), Some((2, 1))), + ladder_step( + POLICY, + 3, + 10, + None, + None, + false, + IVF_SHORT, + Some(80), + Some((2, 1)) + ), LadderStep::Rescan(320) ); } @@ -1117,18 +1251,28 @@ mod ann_probe_budget_tests { #[test] fn ladder_climbs_then_uncaps() { assert_eq!( - ladder_step(3, 10, None, None, false, IVF_SHORT, Some(20), None), + ladder_step(POLICY, 3, 10, None, None, false, IVF_SHORT, Some(20), None), LadderStep::Rescan(80) ); assert_eq!( - ladder_step(3, 10, None, None, false, IVF_SHORT, Some(320), None), + ladder_step(POLICY, 3, 10, None, None, false, IVF_SHORT, Some(320), None), LadderStep::RescanUncapped { summary_missing: false } ); let narrow_index = Some((Some(1), Some(60))); assert_eq!( - ladder_step(3, 10, None, None, false, narrow_index, Some(20), None), + ladder_step( + POLICY, + 3, + 10, + None, + None, + false, + narrow_index, + Some(20), + None + ), LadderStep::RescanUncapped { summary_missing: false } diff --git a/crates/omnigraph/src/table_store.rs b/crates/omnigraph/src/table_store.rs index 66c0c40fd..f6c574e9e 100644 --- a/crates/omnigraph/src/table_store.rs +++ b/crates/omnigraph/src/table_store.rs @@ -4733,29 +4733,22 @@ impl TableStore { has_fts_index_on(ds, column).await } - /// Metadata-only check (no data IO) of whether the FTS (inverted) index - /// entries on `column` together cover EVERY current fragment of `ds`. - /// The FTS twin of `dataset_index::key_column_index_coverage`, consumed as a - /// correctness fence by the rrf prefilter gate: rows in fragments no - /// entry covers are scored by a filter-dependent batch-derived scorer - /// instead of the index-global BM25 statistics (lance - /// `inverted/index.rs`), so a prefilter mask would change their scores - /// and the gate's two plans would stop being answer-identical. - /// Coverage is the UNION across same-column entries: index optimization - /// creates delta entries each covering a disjoint fragment subset, and - /// demanding one all-covering entry would permanently disable the - /// prefilter plan on exactly the append + optimize maintenance schedule - /// production tables follow. `false` is always safe — the gate then - /// runs the postfilter plan — so every unprovable case (no FTS index, - /// an entry without a fragment bitmap) contributes nothing. - pub(crate) async fn fts_covers_all_fragments(ds: &Dataset, column: &str) -> Result { - Ok(Self::fts_coverage(ds, column).await? == omnigraph_planner::FullTextCoverage::Full) - } - - /// How far `column`'s full-text index covers `ds`'s fragments: every - /// fragment (the union of the entries' fragment bitmaps holds them all), - /// some (an entry exists, but the bitmaps do not prove every fragment), - /// or no full-text entry at all. + /// Metadata-only read (no data IO) of how far the FTS (inverted) index + /// entries on `column` together cover `ds`'s current fragments: every + /// fragment, some (an entry exists, but the bitmaps do not prove every + /// fragment), or no full-text entry at all. The FTS twin of + /// `dataset_index::key_column_index_coverage`, read while planning: rows + /// in fragments no entry covers are scored by a filter-dependent + /// batch-derived scorer instead of the index-global BM25 statistics + /// (lance `inverted/index.rs`), so a filter applied before scoring would + /// change their scores. Coverage is the UNION across same-column + /// entries: index optimization creates delta entries each covering a + /// disjoint fragment subset, and demanding one all-covering entry would + /// permanently rule out filtering before scoring on exactly the append + + /// optimize maintenance schedule production tables follow. Anything + /// short of full is always safe — eligibility then applies after + /// scoring and the gates run their postfilter plans — so every + /// unprovable case (an entry without a fragment bitmap) is partial. pub(crate) async fn fts_coverage( ds: &Dataset, column: &str, diff --git a/crates/omnigraph/tests/engine_v2_plan_replay.rs b/crates/omnigraph/tests/engine_v2_plan_replay.rs index 75118ba66..8b000a660 100644 --- a/crates/omnigraph/tests/engine_v2_plan_replay.rs +++ b/crates/omnigraph/tests/engine_v2_plan_replay.rs @@ -645,6 +645,15 @@ fn report_rows(report: &impl serde::Serialize) -> Vec { .clone() } +/// The adaptive search decisions a run recorded: each gate's verdict with +/// the counts it read, and each nearest scan's probe attempts per rung. +fn search_decisions(report: &impl serde::Serialize) -> Vec { + serde_json::to_value(report).unwrap()["search"] + .as_array() + .cloned() + .unwrap_or_default() +} + /// The rule a replay's trace is held to: `id`, `operator`, `status`, `rung` /// and `ran` repeat always; `actual_rows` repeats where both attempts were /// drained; `drained` itself is a scheduling fact and is not compared. @@ -681,16 +690,19 @@ fn sides(rows: &[Value]) -> Vec { } /// The first run and its replays: the result rows and the bound plan of the -/// inspected run, and the report rows of the run and of the second replay. +/// inspected run, its report rows and the search decisions it recorded. struct Replayed { result: Vec, plan: omnigraph_planner::BoundPlan, rows: Vec, + search: Vec, } /// One inspected run and two replays of its plan through the door (the plan -/// read back through its mirrors); each returns the run's rows and trace, the -/// second one proving the door's caches carry no state into a row. +/// read back through its mirrors); each returns the run's rows, trace and +/// search decisions (a gate re-establishes its verdict from the pins, and +/// the ladder takes the same probe rungs), the second one proving the +/// door's caches carry no state into a row. async fn replayed(db: &Session, source: &str, name: &str, params: &ParamMap) -> Replayed { let run = db .query_inspected(ReadTarget::branch("main"), source, name, params) @@ -701,6 +713,7 @@ async fn replayed(db: &Session, source: &str, name: &str, params: &ParamMap) -> assert_eq!(read_back.plan, run.plan, "the bound plan reads back equal"); let result = rows_of(&run.result); let rows = report_rows(&run.report); + let search = search_decisions(&run.report); for _ in 0..2 { let replay = db .replay_bound_plan(ReadTarget::branch("main"), &envelope) @@ -708,11 +721,13 @@ async fn replayed(db: &Session, source: &str, name: &str, params: &ParamMap) -> .unwrap(); assert_eq!(rows_of(&replay.result), result); assert_same_trace(&rows, &report_rows(&replay.report)); + assert_eq!(search_decisions(&replay.report), search); } Replayed { result, plan: run.plan, rows, + search, } } @@ -744,8 +759,9 @@ async fn a_hash_join_traversal_replays_with_its_switches() { async fn a_contains_join_with_a_residual_replays_through_its_marked_scan() { let dir = tempfile::tempdir().unwrap(); let db = citations(&dir).await; - let Replayed { result, plan, rows } = - replayed(&db, CITATION_QUERIES, "cited", &ParamMap::new()).await; + let Replayed { + result, plan, rows, .. + } = replayed(&db, CITATION_QUERIES, "cited", &ParamMap::new()).await; assert_eq!( result, [ @@ -899,9 +915,31 @@ async fn a_nearest_ladder_replays_the_same_rungs() { let dir = tempfile::tempdir().unwrap(); let db = docs(&dir).await; let params = ParamMap::from([("q".to_string(), Literal::List(vec![Literal::Float(0.0); 4]))]); - let Replayed { result, plan, rows } = - replayed(&db, DOC_QUERIES, "nearest_with_edge", ¶ms).await; + let Replayed { + result, + plan, + rows, + search, + } = replayed(&db, DOC_QUERIES, "nearest_with_edge", ¶ms).await; assert_eq!(result.len(), 3); + let decided: Vec<(&str, u64)> = search + .iter() + .map(|decision| { + ( + decision["decision"].as_str().unwrap(), + decision["rung"].as_u64().unwrap(), + ) + }) + .collect(); + assert_eq!( + decided.first(), + Some(&("gate", 0)), + "the edge's pre-pass gate decides first: {search:#?}" + ); + assert!( + decided.iter().filter(|(kind, _)| *kind == "probes").count() == 2, + "every rung records its probe attempts: {search:#?}" + ); let rungs: Vec = rows[0]["attempts"] .as_array() .unwrap() @@ -925,8 +963,16 @@ async fn a_fusion_replays() { let db = docs(&dir).await; let mut params = ParamMap::from([("t".to_string(), Literal::String("needle".to_string()))]); params.insert("q".to_string(), Literal::List(vec![Literal::Float(0.0); 4])); - let Replayed { result, .. } = replayed(&db, DOC_QUERIES, "fused", ¶ms).await; + let Replayed { result, search, .. } = replayed(&db, DOC_QUERIES, "fused", ¶ms).await; assert_eq!(result.len(), 3); + assert_eq!( + search + .iter() + .filter(|decision| decision["decision"] == "gate") + .count(), + 1, + "the fusion's gate records one verdict: {search:#?}" + ); } #[tokio::test] From 03d675bc245d829661eea0f816aae0fba4ab86dd Mon Sep 17 00:00:00 2001 From: Ragnor Comerford Date: Thu, 1 Oct 2026 20:11:26 +0200 Subject: [PATCH 08/29] planner: test replay decoding limits and version refusals Bytes over the evidence limit exhaust before anything is decoded, every envelope version is read before the body, and the replay door answers an envelope of another version as a conflict that asks for the query again. --- .../omnigraph-planner/src/validate/replay.rs | 87 +++++++++++++++++++ .../omnigraph/tests/engine_v2_plan_replay.rs | 38 ++++++++ 2 files changed, 125 insertions(+) diff --git a/crates/omnigraph-planner/src/validate/replay.rs b/crates/omnigraph-planner/src/validate/replay.rs index bd0ea6f94..5c748d9b7 100644 --- a/crates/omnigraph-planner/src/validate/replay.rs +++ b/crates/omnigraph-planner/src/validate/replay.rs @@ -223,3 +223,90 @@ pub fn catalog_digest(catalog: &Catalog) -> Option { CatalogIdentity::SourceUnbound => None, } } + +#[cfg(test)] +mod tests { + use super::*; + use crate::bound::ValueTable; + use crate::physical::PhysicalPlan; + + fn envelope() -> ReplayEnvelope { + ReplayEnvelope { + replay_version: REPLAY_VERSION, + rules_version: RULES_VERSION, + semantics_version: SEMANTICS_VERSION, + query: ReplayQuery { + source: "query q() { match { $d: Doc } return { $d.slug } }".to_string(), + name: "q".to_string(), + }, + scope: ValidationScope::ExactSubset, + catalog: None, + plan: BoundPlan { + plan: PhysicalPlan::new(), + values: ValueTable::default(), + }, + derivation: Some(Derivation::default()), + } + } + + #[test] + fn an_envelope_reads_back_equal() { + let bytes = envelope().to_bytes(); + let decoded = decode_replay(&bytes, ValidationLimits::DEFAULT).unwrap(); + assert_eq!(decoded.query, envelope().query); + assert_eq!(decoded.scope, ValidationScope::ExactSubset); + assert_eq!(decoded.derivation, Some(Derivation::default())); + } + + /// The byte limit refuses before anything is decoded, so even bytes that + /// are no JSON at all exhaust it rather than fail to parse. + #[test] + fn bytes_over_the_limit_exhaust_before_decoding() { + let limits = ValidationLimits { + evidence_bytes: 8, + ..ValidationLimits::DEFAULT + }; + assert_eq!( + decode_replay(b"not json at all", limits).unwrap_err(), + ReplayRefusal::Exhausted { + limit: "evidence_bytes", + value: 8 + } + ); + } + + /// A version is read before the rest: an unsupported one asks for the + /// query again even when the body would not decode, and a missing header + /// is no envelope. + #[test] + fn versions_are_checked_before_the_body() { + for field in ["replay_version", "rules_version", "semantics_version"] { + let mut value = serde_json::to_value(envelope()).unwrap(); + value[field] = serde_json::json!(u32::MAX); + value["plan"] = serde_json::json!("not a plan"); + let refusal = decode_replay( + &serde_json::to_vec(&value).unwrap(), + ValidationLimits::DEFAULT, + ) + .unwrap_err(); + assert!( + matches!(&refusal, ReplayRefusal::ReplanRequired { reason } if reason.contains(field)), + "{refusal:?}" + ); + } + assert!(matches!( + decode_replay(br#"{"plan": {}}"#, ValidationLimits::DEFAULT).unwrap_err(), + ReplayRefusal::ReplanRequired { .. } + )); + let mut value = serde_json::to_value(envelope()).unwrap(); + value["plan"] = serde_json::json!("not a plan"); + assert!(matches!( + decode_replay( + &serde_json::to_vec(&value).unwrap(), + ValidationLimits::DEFAULT + ) + .unwrap_err(), + ReplayRefusal::InvalidEvidence { .. } + )); + } +} diff --git a/crates/omnigraph/tests/engine_v2_plan_replay.rs b/crates/omnigraph/tests/engine_v2_plan_replay.rs index 8b000a660..fd466370e 100644 --- a/crates/omnigraph/tests/engine_v2_plan_replay.rs +++ b/crates/omnigraph/tests/engine_v2_plan_replay.rs @@ -1412,3 +1412,41 @@ async fn an_exact_subset_member_replays_only_with_its_derivation() { ); } } + +/// An envelope of another format, rule catalogue or semantics version is +/// refused before its plan is read, as a conflict that asks for the query +/// again: a version change never reinterprets an old envelope. +#[tokio::test] +async fn an_envelope_of_another_version_asks_for_the_query_again() { + let dir = tempfile::tempdir().unwrap(); + let db = people(&dir).await; + let run = db + .query_inspected( + ReadTarget::branch("main"), + PEOPLE_QUERIES, + "liked", + &ParamMap::new(), + ) + .await + .unwrap(); + let saved: Value = + serde_json::from_slice(&run.replay_envelope(PEOPLE_QUERIES, "liked")).unwrap(); + for field in ["replay_version", "rules_version", "semantics_version"] { + let mut altered = saved.clone(); + altered[field] = serde_json::json!(saved[field].as_u64().unwrap() + 1); + let error = db + .replay_bound_plan( + ReadTarget::branch("main"), + &serde_json::to_vec(&altered).unwrap(), + ) + .await + .err() + .unwrap_or_else(|| panic!("{field}: the replay must be refused")); + assert!( + matches!(&error, OmniError::Manifest(manifest) + if manifest.kind == omnigraph::error::ManifestErrorKind::Conflict) + && error.to_string().contains(field), + "{field}: {error}" + ); + } +} From 2039356b321ea3ad4a7f2740dae049747499a459 Mon Sep 17 00:00:00 2001 From: Ragnor Comerford Date: Thu, 1 Oct 2026 20:14:56 +0200 Subject: [PATCH 09/29] planner: membership, entry parity and evidence-cost tests; linear retention A table of admitted and excluded forms pins exact-fragment membership; the plain and explained acceptance entries agree on plan, scope and an exhausted limit; an instrument prints derivation bytes, steps, nodes, visits and time as a member query grows. Predicate retention resumes after its last match, so in-order placement checks in linear work, and a derivation charges the conjuncts each absorption copies. --- .../omnigraph-planner/src/validate/budget.rs | 5 + .../src/validate/invariants.rs | 10 +- .../omnigraph-planner/src/validate/subset.rs | 1 + .../omnigraph-planner/src/validate/tests.rs | 171 ++++++++++++++++++ 4 files changed, 185 insertions(+), 2 deletions(-) diff --git a/crates/omnigraph-planner/src/validate/budget.rs b/crates/omnigraph-planner/src/validate/budget.rs index 16d710bbc..759ac28f8 100644 --- a/crates/omnigraph-planner/src/validate/budget.rs +++ b/crates/omnigraph-planner/src/validate/budget.rs @@ -62,6 +62,11 @@ impl Budget { self.limits } + /// What the checks charged so far: `(nodes, steps, work)`. + pub fn used(&self) -> (u64, u64, u64) { + (self.nodes, self.steps, self.work) + } + /// Charge `count` visits before making them. pub fn visit(&mut self, count: u64) -> Result<(), ValidationError> { charge(&mut self.work, count, self.limits.work, "work") diff --git a/crates/omnigraph-planner/src/validate/invariants.rs b/crates/omnigraph-planner/src/validate/invariants.rs index 010d134ce..57bcac149 100644 --- a/crates/omnigraph-planner/src/validate/invariants.rs +++ b/crates/omnigraph-planner/src/validate/invariants.rs @@ -220,14 +220,20 @@ impl Requirements { for clause in &self.eligibility { matcher.conjuncts(clause, &mut written_conjuncts); } + // Placement keeps written order within each node, so the search + // resumes after the last match: in-order placement costs one match + // per conjunct, and any other order still finds its conjunct. + let mut cursor = 0; for written in written_conjuncts { if matcher.constant(written) && matcher.value(written) == Some(Literal::Bool(true)) { continue; } let mut found = false; - for conjunct in &placed { - if matcher.matches(written, conjunct, Position::Filter, budget)? { + for offset in 0..placed.len() { + let index = (cursor + offset) % placed.len(); + if matcher.matches(written, &placed[index], Position::Filter, budget)? { found = true; + cursor = index + 1; break; } } diff --git a/crates/omnigraph-planner/src/validate/subset.rs b/crates/omnigraph-planner/src/validate/subset.rs index ae120efea..4317bfe8f 100644 --- a/crates/omnigraph-planner/src/validate/subset.rs +++ b/crates/omnigraph-planner/src/validate/subset.rs @@ -891,6 +891,7 @@ fn apply( "a filter is absorbed only between logical nodes".into(), )); } + budget.visit(u64::try_from(conjuncts.len() + filter.len()).unwrap_or(u64::MAX))?; let Some(position) = conjuncts.iter().position(|held| held == conjunct) else { return Err(invalid(format!( "the filter holds no conjunct `{}` to absorb", diff --git a/crates/omnigraph-planner/src/validate/tests.rs b/crates/omnigraph-planner/src/validate/tests.rs index 22cfe5d42..4ca0fa320 100644 --- a/crates/omnigraph-planner/src/validate/tests.rs +++ b/crates/omnigraph-planner/src/validate/tests.rs @@ -24,6 +24,10 @@ node Doc { title: String year: I64 open: Bool + rank: I32? + score: F64 + born: Date + tags: [String] embedding: Vector(2) } edge Cites: Doc -> Doc @@ -570,3 +574,170 @@ fn a_bm25_scan_with_a_nearest_policy_fails_the_declared_policy() { let (check, detail) = fixture.refused(plan); assert_eq!(check, "declared policy", "{detail}"); } + +/// The scope one query is accepted under. +fn scope_of(query: &str, params: &[(&str, Literal)]) -> ValidationScope { + let fixture = Fixture::new(query, params); + fixture + .accept(fixture.plan()) + .unwrap_or_else(|error| panic!("`{query}` is not accepted: {error:?}")) + .scope() +} + +/// Every admitted form of the exact fragment is a member, and every +/// excluded family is not: membership reads the checked declaration only. +#[test] +fn membership_admits_the_fragment_and_nothing_else() { + let members = [ + "query q() { match { $d: Doc } return { $d.slug } }", + "query q() { match { $d: Doc { open: true } } return { $d.slug } }", + "query q($y: I64) { match { $d: Doc $d.year >= $y } return { $d.slug } }", + "query q() { match { $d: Doc $d.year != 3 and not ($d.title < \"m\" or $d.open) } return { $d.slug } }", + "query q() { match { $d: Doc $d.rank is null or $d.rank > 2 } return { $d.slug, $d.rank } }", + "query q() { match { $d: Doc $d.open } return { $d.title as t } order { $d.year desc, $d.title } limit 4 }", + "query q($t: String) { match { $d: Doc $d.title = $t } return { $d.slug } limit 0 }", + "query q($q: String) { match { $d: Doc } return { bm25($d.title, $q) as s, $d.slug } order { bm25($d.title, $q), $d.slug desc } limit 2 }", + ]; + for query in members { + let params = [ + ("y", Literal::Integer(1)), + ("t", Literal::String("x".into())), + ("q", Literal::String("graph".into())), + ]; + let used: Vec<(&str, Literal)> = params + .iter() + .filter(|(name, _)| query.contains(&format!("${name}:"))) + .cloned() + .collect(); + assert_eq!( + scope_of(query, &used), + ValidationScope::ExactSubset, + "{query}" + ); + } + let outside = [ + "query q() { match { $d: Doc $d.title contains \"g\" } return { $d.slug } }", + "query q() { match { $d: Doc $d.score > 1.5 } return { $d.slug } }", + "query q() { match { $d: Doc $d.born > date(\"2020-01-01\") } return { $d.slug } }", + "query q() { match { $d: Doc { tags: \"a\" } } return { $d.slug } }", + "query q() { match { $d: Doc $d cites $e } return { $d.slug } }", + "query q() { match { $d: Doc $e: Doc } return { $d.slug } }", + "query q() { match { $d: Doc not { $d cites $e } } return { $d.slug } }", + "query q() { match { $d: Doc } return { count($d) as n } }", + "query q() { match { $d: Doc } return { $d } }", + "query q() { match { $d: Doc } return { $d.@id } }", + "query q() { match { $d: Doc } return { $d.slug } order { $d.score } }", + "query q() { match { $d: Doc search($d.title, \"g\") } return { $d.slug } }", + "query q($v: Vector(2)) { match { $d: Doc } return { $d.slug } order { nearest($d.embedding, $v) } limit 2 }", + "query q($q: String) { match { $d: Doc } return { $d.slug } order { rrf(bm25($d.title, $q), bm25($d.title, $q)) } limit 2 }", + ]; + for query in outside { + let params = [ + ( + "v", + Literal::List(vec![Literal::Float(0.0), Literal::Float(1.0)]), + ), + ("q", Literal::String("graph".into())), + ]; + let used: Vec<(&str, Literal)> = params + .iter() + .filter(|(name, _)| query.contains(&format!("${name}:"))) + .cloned() + .collect(); + assert_eq!( + scope_of(query, &used), + ValidationScope::InvariantsOnly, + "{query}" + ); + } +} + +/// The plain and the explained acceptance share one path: the same plan and +/// scope, and the same outcome when a validation limit runs out. +#[test] +fn both_acceptance_entries_agree_even_when_a_limit_runs_out() { + for limits in [ + ValidationLimits::DEFAULT, + ValidationLimits { + work: 5, + ..ValidationLimits::DEFAULT + }, + ValidationLimits { + steps: 2, + ..ValidationLimits::DEFAULT + }, + ] { + let fixture = ranked(); + let constants = Bound(&fixture.params); + let input = AcceptInput { + checked: &fixture.checked, + catalog: &fixture.catalog, + ir: &fixture.ir, + params: &fixture.params, + constants: &constants, + limits, + }; + let plain = crate::gate::accept_query(&input, &fixture.source, &BOUNDS); + let explained = crate::gate::accept_query_explained(&input, &fixture.source, &BOUNDS); + match (plain, explained) { + (Ok(plain), Ok((explained, explain))) => { + assert_eq!(plain.plan(), explained.plan()); + assert_eq!(plain.scope(), explained.scope()); + assert_eq!(explain.validation, Some(explained.summary())); + } + (Err(plain), Err(explained)) => { + assert_eq!(plain, explained); + assert!( + matches!(plain, crate::gate::Unrouted::ValidationExhausted { .. }), + "{plain:?}" + ); + } + (plain, explained) => panic!("the entries disagree: {plain:?} / {explained:?}"), + } + } +} + +/// Evidence size and checking work as a member query grows: serialized +/// derivation bytes, rule applications, retained nodes and visits, and the +/// checking time. A decision instrument, not a gate. +#[test] +#[ignore = "instrument: prints derivation bytes, steps, nodes, visits and time per conjunct count"] +fn derivation_cost_grows_with_the_query() { + for conjuncts in [1usize, 4, 16, 64, 256] { + let filter: Vec = (0..conjuncts) + .map(|index| format!("$d.year != {index}")) + .collect(); + let query = format!( + "query q() {{ match {{ $d: Doc {} }} return {{ $d.slug }} order {{ $d.year }} limit 10 }}", + filter.join(" ") + ); + let fixture = Fixture::new(&query, &[]); + let traced = fixture.traced(); + let derivation = traced.derivation.clone().unwrap(); + let bytes = serde_json::to_vec(&derivation).unwrap().len(); + let constants = Bound(&fixture.params); + let input = AcceptInput { + checked: &fixture.checked, + catalog: &fixture.catalog, + ir: &fixture.ir, + params: &fixture.params, + constants: &constants, + limits: ValidationLimits::DEFAULT, + }; + let mut budget = Budget::new(ValidationLimits::DEFAULT); + let started = std::time::Instant::now(); + let (scope, _) = check( + &traced.optimized.physical, + &input, + Some(derivation), + &mut budget, + ) + .unwrap(); + let elapsed = started.elapsed(); + let (nodes, steps, work) = budget.used(); + println!( + "conjuncts={conjuncts} scope={} bytes={bytes} steps={steps} nodes={nodes} work={work} elapsed={elapsed:?}", + scope.as_str() + ); + } +} From 199558de92f7cbf0ed549be65b315cffb6577d10 Mon Sep 17 00:00:00 2001 From: Ragnor Comerford Date: Thu, 1 Oct 2026 20:16:07 +0200 Subject: [PATCH 10/29] lance guards: pin the full-text prefilter and postfilter scoring facts A prefilter changes the BM25 score of a row no full-text segment covers, and a postfilter keeps the whole corpus's score; the eligibility placement of a bm25 scan rests on both. --- .../omnigraph/tests/lance_surface_guards.rs | 140 ++++++++++++++++++ 1 file changed, 140 insertions(+) diff --git a/crates/omnigraph/tests/lance_surface_guards.rs b/crates/omnigraph/tests/lance_surface_guards.rs index 0d95158b6..192530e03 100644 --- a/crates/omnigraph/tests/lance_surface_guards.rs +++ b/crates/omnigraph/tests/lance_surface_guards.rs @@ -2660,6 +2660,146 @@ async fn fts_scan_accepts_score_in_a_projection_set_before_full_text_search() { ); } +/// The two Lance 11 facts BM25 eligibility placement rests on. Lance scores +/// the fragments no full-text segment covers flat, counting documents and +/// tokens over the rows its prefilter admits, so a prefilter changes an +/// uncovered row's score; with `prefilter(false)` the full-text search +/// scores the whole corpus and the filter refines its output, so the score is +/// the unfiltered one. OmniGraph filters a bm25 scan before scoring only under +/// full coverage (`Eligibility::BeforeScoring`) and runs Lance's postfilter +/// otherwise. If the first assertion goes red (the flat tail scores from +/// filter-independent statistics), the placement rule can be dropped. +#[tokio::test] +async fn fts_prefilter_changes_unindexed_scores_and_postfilter_keeps_them() { + use arrow_array::types::Float32Type; + + let dir = tempfile::tempdir().unwrap(); + let uri = dir.path().join("tail_scores.lance"); + let uri = uri.to_str().unwrap(); + let schema = Arc::new(Schema::new(vec![ + Field::new("id", DataType::Utf8, false), + Field::new("text", DataType::Utf8, false), + Field::new("year", DataType::Int64, false), + ])); + let batch = |rows: &[(&str, &str, i64)]| { + RecordBatch::try_new( + schema.clone(), + vec![ + Arc::new(StringArray::from( + rows.iter().map(|row| row.0).collect::>(), + )), + Arc::new(StringArray::from( + rows.iter().map(|row| row.1).collect::>(), + )), + Arc::new(arrow_array::Int64Array::from( + rows.iter().map(|row| row.2).collect::>(), + )), + ], + ) + .unwrap() + }; + let indexed = batch(&[ + ("d1", "graph engines", 2020), + ("d2", "graph search", 2021), + ("d3", "databases", 2019), + ]); + let mut dataset = Dataset::write( + RecordBatchIterator::new(vec![Ok(indexed)], schema.clone()), + uri, + Some(WriteParams { + mode: WriteMode::Create, + enable_stable_row_ids: true, + data_storage_version: Some(LanceFileVersion::V2_2), + ..Default::default() + }), + ) + .await + .unwrap(); + dataset + .create_index( + &["text"], + IndexType::Inverted, + None, + &InvertedIndexParams::default(), + true, + ) + .await + .unwrap(); + let tail = batch(&[ + ("t1", "graph", 2022), + ( + "t2", + "graph graph graph graph graph graph graph graph", + 1990, + ), + ]); + dataset + .append( + RecordBatchIterator::new(vec![Ok(tail)], schema.clone()), + Some(WriteParams { + mode: WriteMode::Append, + data_storage_version: Some(LanceFileVersion::V2_2), + ..Default::default() + }), + ) + .await + .unwrap(); + + let score_of = |batch: &RecordBatch, id: &str| { + let ids = batch.column_by_name("id").unwrap().as_string::(); + let scores = batch + .column_by_name("_score") + .unwrap() + .as_primitive::(); + (0..batch.num_rows()) + .find(|row| ids.value(*row) == id) + .map(|row| scores.value(row)) + }; + let search = |filter: Option<(&'static str, bool)>| { + let dataset = dataset.clone(); + async move { + let mut scanner = dataset.scan(); + scanner.project(&["id", "_score"]).unwrap(); + if let Some((filter, prefilter)) = filter { + scanner.filter(filter).unwrap(); + scanner.prefilter(prefilter); + } + scanner + .full_text_search( + FullTextSearchQuery::new("graph".to_string()) + .with_column("text".to_string()) + .unwrap(), + ) + .unwrap(); + scanner.try_into_batch().await.unwrap() + } + }; + let unfiltered = search(None).await; + let prefiltered = search(Some(("year >= 2000", true))).await; + let postfiltered = search(Some(("year >= 2000", false))).await; + let whole = score_of(&unfiltered, "t1").expect("t1 matches"); + assert_ne!( + score_of(&prefiltered, "t1"), + Some(whole), + "a prefilter changes the score of a row no full-text segment covers" + ); + assert_eq!( + score_of(&postfiltered, "t1"), + Some(whole), + "a postfilter keeps the whole corpus's score" + ); + assert_eq!( + score_of(&postfiltered, "t2"), + None, + "the postfilter still refines" + ); + assert_eq!( + score_of(&prefiltered, "d1"), + score_of(&unfiltered, "d1"), + "an indexed row scores from the index's statistics either way" + ); +} + // --- Lance 10 compatibility: fence late-hydrated KNN ordering -------------- // // lance#7868 makes execute_plan preserve order when the plan still advertises From 9812e7309ee56409c31cee63edd49be04d78dca8 Mon Sep 17 00:00:00 2001 From: Ragnor Comerford Date: Thu, 1 Oct 2026 20:20:08 +0200 Subject: [PATCH 11/29] docs: plan acceptance, declared search policy and RFC 0047 step 2a as built The execution guide describes plan acceptance, the exact fragment and the replay envelope; the explain guide documents validation, a bm25 scan's eligibility placement, a nearest scan's policy, the recorded full-text coverage and the profile's search decision rows, which the profile now carries; the testing map names the validator's owners; the Lance guide adds the eligibility placement fence; the GQT README adds the scope and eligibility claims. RFC 0047 records where the built design differs from its text, and three release notes describe the user-visible changes. --- .../bm25-scores-ignore-filters.fixed.md | 6 + changelog.d/plan-acceptance.added.md | 8 ++ ...tie-break-declared-bindings-first.fixed.md | 6 + crates/omnigraph-gqt/README.md | 16 ++- crates/omnigraph/src/engine/report.rs | 18 ++- .../tests/engine_v2_scrubbed_replay.rs | 48 +++++++- docs/dev/execution.md | 99 ++++++++++++--- docs/dev/lance.md | 3 +- docs/dev/testing.md | 5 +- docs/rfcs/0047-search-plan-truth.md | 116 ++++++++++++++++-- docs/user/queries/explain.md | 42 +++++-- 11 files changed, 320 insertions(+), 47 deletions(-) create mode 100644 changelog.d/bm25-scores-ignore-filters.fixed.md create mode 100644 changelog.d/plan-acceptance.added.md create mode 100644 changelog.d/tie-break-declared-bindings-first.fixed.md diff --git a/changelog.d/bm25-scores-ignore-filters.fixed.md b/changelog.d/bm25-scores-ignore-filters.fixed.md new file mode 100644 index 000000000..a0b73228d --- /dev/null +++ b/changelog.d/bm25-scores-ignore-filters.fixed.md @@ -0,0 +1,6 @@ +- A `bm25()` score no longer depends on the query's filters when rows were + written after the property's full-text index was built: such a scan now + applies its filter after scoring. See the [explain + guide][bm25-eligibility-explain]. + +[bm25-eligibility-explain]: ../docs/user/queries/explain.md diff --git a/changelog.d/plan-acceptance.added.md b/changelog.d/plan-acceptance.added.md new file mode 100644 index 000000000..6005026fa --- /dev/null +++ b/changelog.d/plan-acceptance.added.md @@ -0,0 +1,8 @@ +- Every read plan is checked against the query before it runs, and `explain` + reports what the check covered in a `validation` field: `exact_subset` for a + single-binding query with property filters, an optional leading `bm25()`, + property sort keys and a limit, whose every rewrite was checked, and + `invariants_only` for every other query. See the [explain + guide][plan-acceptance-explain]. + +[plan-acceptance-explain]: ../docs/user/queries/explain.md diff --git a/changelog.d/tie-break-declared-bindings-first.fixed.md b/changelog.d/tie-break-declared-bindings-first.fixed.md new file mode 100644 index 000000000..3a6c3830d --- /dev/null +++ b/changelog.d/tie-break-declared-bindings-first.fixed.md @@ -0,0 +1,6 @@ +- Rows that tie on every `order` key are now ordered by the identities of the + bindings the query declares before those of anonymous traversal endpoints, + so a `limit` cuts them as the declared bindings' order says. See the + [explain guide][tie-break-explain]. + +[tie-break-explain]: ../docs/user/queries/explain.md diff --git a/crates/omnigraph-gqt/README.md b/crates/omnigraph-gqt/README.md index 2041e8372..595049ef7 100644 --- a/crates/omnigraph-gqt/README.md +++ b/crates/omnigraph-gqt/README.md @@ -223,6 +223,7 @@ filter reads [d.rank, e.rank] sort tiebreak [$d, $e] pass projection_pushdown not pass aggregate_pushdown +validation scope exact_subset ``` A `scan [ as $var]:` line selects the scans of that type (or the one @@ -267,11 +268,16 @@ csr`): the side of the node's declared switch the run took, read from the execution report of the same run (the last attempt of the node's row, joined to the explain row by the node's `id`); it fails when the run recorded no side on that node or another side. A `scan [ as $var]: ranked -[ fetch ][ nprobes ]` line claims the ranked access path -of the scan, the candidates it asks the index for and, on a `nearest` scan, -the probe cap the plan carries (`0` spells no cap, as the `ann_nprobes` -setting does). Nothing is compared as rendered text, so a planner that -reaches the same facts by another route keeps the case green. +[ fetch ][ nprobes ][ eligibility ]` line +claims the ranked access path of the scan, the candidates it asks the index +for, on a `nearest` scan the probe cap the plan carries (`0` spells no cap, +as the `ann_nprobes` setting does), and on a `bm25` scan where it applies its +eligibility (`before_scoring` under full full-text coverage, else +`after_scoring`). `validation scope ` claims +what acceptance checked of the plan, as the document's `validation` reports +it; a failed planning step has no document and keeps its `expect error:`. +Nothing is compared as rendered text, so a planner that reaches the same +facts by another route keeps the case green. ## Reference comparison diff --git a/crates/omnigraph/src/engine/report.rs b/crates/omnigraph/src/engine/report.rs index a6113b854..ae587dfa4 100644 --- a/crates/omnigraph/src/engine/report.rs +++ b/crates/omnigraph/src/engine/report.rs @@ -225,8 +225,9 @@ impl Executed { /// The report as actuals rows in the explain row schema: `tree` /// `profile`, no `depth`, `node` the plan node's kind, `detail` the row's - /// own fields (`id`, `operator`, `status`, `attempts`). The third public - /// surface beside the rows and explain. + /// own fields (`id`, `operator`, `status`, `attempts`), then one row per + /// search decision (`node` `gate` or `probes`). The third public surface + /// beside the rows and explain. pub fn profile(&self) -> Result { let mut rows = ExplainRows::default(); for row in &self.report.rows { @@ -243,6 +244,19 @@ impl Executed { })?; rows.push(PROFILE_TREE, None, node, detail.to_string()); } + for decision in &self.report.search { + let detail = serde_json::to_value(decision).map_err(|error| { + OmniError::manifest_internal(format!( + "search decision of node {} cannot serialize: {error}", + decision.id + )) + })?; + let node = match decision.taken { + Taken::Gate { .. } => "gate", + Taken::Probes { .. } => "probes", + }; + rows.push(PROFILE_TREE, None, node, detail.to_string()); + } rows.into_result() } } diff --git a/crates/omnigraph/tests/engine_v2_scrubbed_replay.rs b/crates/omnigraph/tests/engine_v2_scrubbed_replay.rs index e9bcb1bd8..7cd90a495 100644 --- a/crates/omnigraph/tests/engine_v2_scrubbed_replay.rs +++ b/crates/omnigraph/tests/engine_v2_scrubbed_replay.rs @@ -333,7 +333,8 @@ async fn ann_nprobes_is_a_field_of_the_ranked_scan_and_the_rows_do_not_move() { } /// The profile surface: one row per report row in the explain row schema, -/// its `node` the plan node's kind and its `detail` the row's own fields. +/// its `node` the plan node's kind and its `detail` the row's own fields, +/// then one row per search decision the run took. #[tokio::test] async fn profile_rows_carry_the_report_in_the_explain_row_schema() { let dir = tempfile::tempdir().unwrap(); @@ -384,4 +385,49 @@ async fn profile_rows_carry_the_report_in_the_explain_row_schema() { ); } } + + let docs = tempfile::tempdir().unwrap(); + let db = session( + Omnigraph::init(docs.path().to_str().unwrap(), DOC_SCHEMA) + .await + .unwrap(), + ); + db.load_jsonl( + r#"{"type":"Doc","data":{"slug":"d0","embedding":[0.0,0.0,0.0,0.0]}} +{"type":"Doc","data":{"slug":"d1","embedding":[1.0,0.0,0.0,0.0]}}"#, + LoadMode::Overwrite, + ) + .await + .unwrap(); + let params = ParamMap::from([("q".to_string(), Literal::List(vec![Literal::Float(0.0); 4]))]); + let run = with_setting(&db, "engine", "v2") + .query_inspected(ReadTarget::branch("main"), NEAREST, "by_vector", ¶ms) + .await + .unwrap(); + let batch = run.profile().unwrap().concat_batches().unwrap(); + let nodes = batch + .column(2) + .as_any() + .downcast_ref::() + .unwrap(); + let details = batch + .column(3) + .as_any() + .downcast_ref::() + .unwrap(); + let probes: Vec = (0..batch.num_rows()) + .filter(|row| nodes.value(*row) == "probes") + .map(|row| serde_json::from_str(details.value(row)).unwrap()) + .collect(); + assert_eq!( + probes.len(), + 1, + "the nearest scan's one pass records its probes" + ); + assert_eq!(probes[0]["decision"], "probes"); + assert_eq!(probes[0]["rung"], 0); + assert!( + !probes[0]["attempts"].as_array().unwrap().is_empty(), + "{probes:?}" + ); } diff --git a/docs/dev/execution.md b/docs/dev/execution.md index 08b213090..7a2142af0 100644 --- a/docs/dev/execution.md +++ b/docs/dev/execution.md @@ -10,11 +10,14 @@ A query runs against one resolved `ReadTarget`: 1. Resolve the branch or graph snapshot once. 2. Parse and type-check `.gq` source against that snapshot's accepted catalog. -3. Lower the checked query to typed IR. -4. Select any search mode and the edge types required by traversal or an - anti-join. -5. Execute the IR against the same snapshot. -6. Serialize result batches at the calling boundary. +3. Lower the checked query to typed IR, keeping the checked declaration and + its type context beside it. +4. Gather the planning inputs once: bound parameters (`now()` among them), + constants folded to their values, statistics, settings and the full-text + coverage of every ranked property. +5. Plan the query and accept the plan (see [Plan acceptance](#plan-acceptance)). +6. Bind the accepted plan's values and execute it against the same snapshot. +7. Serialize result batches at the calling boundary. The executor never refreshes a mutable branch head midway through a query. Historical reads build from the requested immutable table versions; current @@ -26,6 +29,7 @@ Stable code owners: |---|---| | Parser, type checker, lowering | `crates/omnigraph-compiler/src/query/`, `src/ir/` | | Read doors | `crates/omnigraph/src/exec/query_doors.rs` | +| Plan acceptance | `crates/omnigraph-planner/src/validate/` | | Plan lowering, operators and context (engine v2) | `crates/omnigraph/src/engine/` | | Frozen engine v1, the GQT test reference | `crates/omnigraph-reference-engine/` | | Lance scan boundary | `crates/omnigraph/src/table_store.rs` | @@ -587,6 +591,53 @@ truth: the engine's own recursion with its catch-all arm was a second list of handled variants beside the `PhysicalNode` enum, and it is gone. That is why no RFC accompanies it. +## Plan acceptance + +Every read reaches execution through one door: `accept_query` plans the query +and the planner's validator checks the plan against what the checked +declaration requires; only that check constructs an `AcceptedPlan`, whose +fields are private, and `bind` and `execute` take nothing else. The ordinary +run, `Session::explain_query`, the `explain` statement and the inspected run +share the path, so explain validates exactly what a run validates and reports +the outcome as `validation.scope`. + +The requirements come from the declaration, its type context and the +catalog, not from the IR: binding identity, search identity and its declared +approximation, every eligibility conjunct (checked per `rrf` arm, since each +arm copies the match), correlated blocks, the projection and the origin of +every projected score, order and cut, and the policies the plan declares +for its search. A matcher relates each written expression to the IR through +the compiler's lowering patterns (property leaves on physical columns, `in` +as a swapped `contains`, a bare search call as `= true`, a folded constant as +the value the engine's evaluator gives it). The order check recomputes the +root's order from the operators (`optimizer::derived_order`, typed +`OrderKey`s) and accepts the planner's tie-break omissions that are +comparator equivalences: a key already in the comparator, every key when the +query returns only order keys, and identities of bindings the query does not +name (anonymous endpoints, cycle temps) after every named one. + +A query of one binding, Boolean/integer/String eligibility, an optional +leading `bm25()` over a String property, direct property sort keys, a limit +and a projection of properties and the selected score is a member of the +exact fragment (`validate/subset.rs`). For a member the optimizer records the +rules it applies (`AbsorbScanFilter`, `PruneScanColumns`, `Lower`, +`RankBm25Scan`) over arena references; the validator checks that the IR is +the declaration's lowering, rebuilds the canonical chain +`Limit(Sort(Project([Search](Filter(Scan)))))`, re-applies every rule +against its precondition and requires the result to equal the plan. A member +is accepted as `exact_subset`; every other plan as `invariants_only`. + +A fresh plan that fails a check is a planner defect (an internal error). A +check that exhausts a `ValidationLimits` budget (evidence bytes, derivation +nodes, rule applications, visits) is `ResourceLimitExceeded`; a refusal by +design keeps its diagnostic and is a bad request. The replay door +(`Session::replay_bound_plan`) takes a serialized `ReplayEnvelope`: the query +source and name, the scope, the schema digest, the bound plan and a member's +derivation. It checks the byte limit and the envelope's versions before +decoding the plan, refuses another schema and a dataset the plan did not pin +as conflicts, recompiles the query, and accepts the plan again; a plan that +fails a check there is invalid evidence, a bad request. + ## Search and rank `nearest`, text search/BM25, and reciprocal-rank fusion are first-class @@ -598,16 +649,27 @@ their ordered results. BM25 scans have no candidate cap. The final ordering applies the score, secondary keys and every binding's identity before the query limit; a full candidate window cannot prove that it contains the leading tied identities. +A BM25 scan declares where it applies its eligibility (its pushed filter, a +gate's eligible set and the members of its search predicates): +`before_scoring`, Lance's prefilter, only when the property's full-text index +covers every fragment at the pinned version, a fact the plan records in +`Assumptions.full_text`; `after_scoring`, Lance's postfilter, otherwise. +Lance scores uncovered fragments flat from statistics over the rows its +prefilter admits, so filtering before scoring there would change scores. A `nearest` scan carries a probe cap per index delta (the `ann_nprobes` session setting, `request` scope, default 20, `0` is no cap; the process -default is `OMNIGRAPH_ANN_NPROBES`). `execute_node_scan` runs a probe ladder: a capped scan short of -`k` with partitions unread reruns at four times the cap, then uncapped. The -stop rules (`ladder_step`) end the ladder on every other cause of a short -scan, read from Lance's execution summary (`partitions_searched` / -`partitions_ranked`) and from the `_distance = +inf` marker Lance emits when a -prefilter admits fewer rows than `k`, which triggers one flat exact rescan; a -missing summary fails closed into one uncapped rescan. Above the scan, +default is `OMNIGRAPH_ANN_NPROBES`) and the adaptive policy the plan declares +(`RankedAccess.policy`, `NearestPolicy`). `execute_node_scan` runs a probe +ladder within it: a capped scan short of `k` with partitions unread reruns at +`probe_factor` (4) times the cap, then uncapped. The stop rules +(`ladder_step`) end the ladder on every other cause of a short scan, read +from Lance's execution summary (`partitions_searched` / `partitions_ranked`) +and from the `_distance = +inf` marker Lance emits when a prefilter admits +fewer rows than `k`, which triggers one flat exact rescan; a missing summary +fails closed into one uncapped rescan; a scan whose eligible set fits within +`k` scores flat from its first attempt. Acceptance refuses a policy whose +ladder cannot terminate or that drops either fallback. Above the scan, `execute` runs the overfetch ladder the plan declares (`RankedAccess.overfetch`): a full scan whose result is short of `limit` after later operators reruns with `k` times 4, then times 16, seeded with the @@ -616,10 +678,15 @@ type's row count, no probe cap), so a short answer is never served while survivors exist; aggregate returns never overfetch. Before either, `nearest_prefilter_gate` runs the pre-pass the plan declares (`RankedAccess.prefilter`) for a standalone `nearest` constrained by a traversal and pushes the eligible ids into the scan as an `id IN (...)` prefilter; an empty -eligible set proves the answer empty and runs no scan. The `rrf` gate is -answer-preserving (its set over-approximates the survivors); the nearest gate -is answer-changing by design (it ranks the eligible entities exactly instead -of the global window's survivors). +eligible set proves the answer empty and runs no scan (`Prefilter.on_empty`). +The `rrf` gate is answer-preserving (its set over-approximates the +survivors) and prefilters its bm25 arms only under the recorded full +coverage (`Prefilter.coverage_admits`); the nearest gate is answer-changing +by design (it ranks the eligible entities exactly instead of the global +window's survivors). Acceptance checks that every hop of a pre-pass is a +required first hop of the query. The execution report records each gate's +verdict with the counts it read and each probe attempt per overfetch rung +(`search` decisions), and a replay must record the same. Lance 11 still loses final KNN ordering metadata in one late payload-hydration shape, so OmniGraph requests one output partition for the affected nearest diff --git a/docs/dev/lance.md b/docs/dev/lance.md index 5a7e9f050..1bccd993d 100644 --- a/docs/dev/lance.md +++ b/docs/dev/lance.md @@ -166,7 +166,8 @@ history of dependency bumps. | Graph keys | Schema-v9 node/edge tables carry the exact non-null `__id` unenforced primary key (edges also `__src`/`__dst`); v8 tables keep the `id`/`src`/`dst` spellings introduced in v6, and the explicit system-column upgrade (RFC 0040) renames them with one rename-only `Operation::Project` commit per table (`TableStore::renamed_schema`), staged detached and published as a pin (RFC 0067, detached-only), which must keep every fragment, field id, the primary-key marker and index attachments. Strict insert/upsert uses the sealed filter-bearing adapter; raw keyed Append is forbidden. | `lance_surface_guards.rs`, staged-table tests, `forbidden_apis.rs`, `system_column_upgrade.rs` | | Stable row IDs | Graph tables use stable row IDs; delete/update/index maintenance must retain their mapping. Overwrite allocates fresh IDs and restore retains the allocation high-water marks. Staged-view IDs are provisional, not committed identity. | `lance_surface_guards.rs`, staged-table tests, `writes.rs` | | KNN result order | A late payload-hydration plan can lose global ordering metadata, so nearest requests one final output partition. Internal reads remain parallel. | `lance_surface_guards.rs`, `search.rs` | -| KNN probe budget | A `nearest` scan sets `maximum_nprobes` per index delta and widens it from Lance's execution summary (`partitions_searched` / `partitions_ranked`, read through `scan_stats_callback`); a prefilter admitting fewer rows than `k` makes Lance emit the unreached rows at `_distance = +inf`, which the engine resolves with a flat exact rescan (`use_index(false)`); `count_rows(None)` is the live row count (deletions excluded), read for the ladder's exhaustion stop and as the overfetch loop's exact-pass `k`. A renamed counter or marker turns the ladder fail-closed. | `lance_surface_guards.rs`, `search.rs` | +| KNN probe budget | A `nearest` scan sets `maximum_nprobes` per index delta and widens it from Lance's execution summary (`partitions_searched` / `partitions_ranked`, read through `scan_stats_callback`); a prefilter admitting fewer rows than `k` makes Lance emit the unreached rows at `_distance = +inf`, which the engine resolves with a flat exact rescan (`use_index(false)`); `count_rows(None)` is the live row count (deletions excluded), read for the ladder's exhaustion stop and as the overfetch loop's exact-pass `k`. A renamed counter or marker turns the ladder fail-closed. The factor and both fallbacks are the plan's declared `NearestPolicy`. | `lance_surface_guards.rs`, `search.rs` | +| Full-text eligibility placement | Lance scores fragments no full-text segment covers flat, counting documents and tokens over the rows its prefilter admits (`plan_flat_match_query`), so a prefilter changes those rows' BM25 scores; `prefilter(false)` scores the whole corpus and refines afterwards. A bm25 scan therefore filters before scoring only under the plan's recorded full coverage (`Eligibility::BeforeScoring`) and runs the postfilter otherwise. Drop the placement rule at the bump whose flat tail scores from filter-independent statistics. | `lance_surface_guards.rs::fts_prefilter_changes_unindexed_scores_and_postfilter_keeps_them`, `cases/v2/bm25_score_ignores_a_filter_over_an_unindexed_fragment.gqt` | | Full-text analyzer | Every selected FTS segment needs artifact-scoped compatibility proof. Explicit branch rebuilds replace all segments from rows; old snapshots may refuse FTS. | staged-table tests, `search.rs`, `maintenance.rs` | | Blob v2 | Null, valid empty, non-empty, selector cardinality, neighboring bytes, and 3→1 compaction are pinned on Lance 11. | `lance_surface_guards.rs`, `maintenance.rs` | | Index coverage | Indexes are derived. Rewrites and compaction may leave an uncovered tail; reads must combine indexed and scan paths until explicit reconciliation. | `scalar_indexes.rs`, `search.rs`, `maintenance.rs` | diff --git a/docs/dev/testing.md b/docs/dev/testing.md index 3da165469..2801646bd 100644 --- a/docs/dev/testing.md +++ b/docs/dev/testing.md @@ -48,7 +48,8 @@ The engine integration suite is grouped by behavior, not implementation module: | V2 execution and memory | `engine_v2.rs`, `engine_v2_memory.rs`; `repro_issue_703.rs` and `repro_issue_723.rs` own ignored scale symptoms | | Frozen engine v1, the reference | `crates/omnigraph-reference-engine/tests/frozen.rs` pins every source file's bytes; `forbidden_apis.rs` owns that `omnigraph-gqt` is the crate's only dependent and that its source names neither the engine nor the planner; the crate has no other tests, and a `.gqt` step's `--- expect same as v1` compares v2's rows with its answer | | V2 plan nodes' operators and the execution report | In-source `engine/report/tests.rs` owns which operator each node builds and what its row says (`ran`, `actual_rows`, `drained`); `crates/omnigraph-planner/tests/lower_walk.rs` owns the walk's order and refusals; the `ran` lines of `--- expect plan` read the report of the case's own run | -| V2 plan sufficiency: the bound plan alone reproduces a run | `crates/omnigraph/tests/engine_v2_plan_replay.rs` (see [Plan replay](#plan-replay)): one query per node kind with a switch or a ladder, replayed twice through `Session::replay_bound_plan` from the serialized `BoundPlan`, equal rows and the same trace required; `crates/omnigraph-planner/tests/bound_plan.rs` owns the serialized form; in-source `engine/search.rs` tests own the declared ladder's stepping; `engine/plan_source.rs` tests own that the assumed memory limit sizes the plan, the lowering and the pool | +| V2 plan sufficiency: the bound plan alone reproduces a run | `crates/omnigraph/tests/engine_v2_plan_replay.rs` (see [Plan replay](#plan-replay)): one query per node kind with a switch or a ladder, replayed twice through `Session::replay_bound_plan` from its serialized replay envelope, equal rows, the same trace and the same search decisions required; `crates/omnigraph-planner/tests/bound_plan.rs` owns the serialized form; in-source `engine/search.rs` tests own the declared ladder's stepping; `engine/plan_source.rs` tests own that the assumed memory limit sizes the plan, the lowering and the pool | +| V2 plan acceptance: every read plan checked against its checked query | In-source `crates/omnigraph-planner/src/validate/tests.rs` owns one broken plan per check (binding and search identity, predicate retention, row cut, order and tie-break, projection and score origin, declared policies, full-text prerequisites), exact-fragment membership over admitted and excluded forms, the derivation's rules and rejections, entry parity under an exhausted limit, and the `instrument:` derivation cost curve; `validate/replay.rs` and `validate/budget.rs` tests own decoding limits, version refusals and budget charges; `engine_v2.rs::every_read_door_shares_one_acceptance` owns that the run, both explain doors and the inspected run accept one plan and refuse alike; GQT cases claim the scope with `validation scope ` in `--- expect plan` | | V2 scrubbed replay: no input beside the bound plan and the snapshot reaches execution | `engine_v2_scrubbed_replay.rs` owns the mechanism (the `std::env` grep over `crates/omnigraph/src/engine/`, the replay under an ambient memory limit and an ambient `OMNIGRAPH_EXPAND_INDEXED_MAX_FRONTIER` the plan did not capture, `ann_nprobes` as a plan field, the `profile` rows); `engine_v2_plan_replay.rs` owns the snapshot pins (an edge write or an insert after planning refuses the replay); the inventory cases `cases/v2/planner/input_*.gqt` own one input class each (`engine`, `ann_nprobes`, index build state), and the clock class is owned by `engine_v2_plan_replay.rs`; `engine_v2_memory.rs` owns `ran id_lookup` under a refused build | | V2 declared switches: the report names the side a run took | `cases/v2/planner/switch_ran_on_the_report.gqt` (the `ran` line of `--- expect plan`); in-source `engine/report/tests.rs` owns the switch's row and a scan whose plan declares no fallback | | Search and physical indexes | `search.rs`, `scalar_indexes.rs`, `lance_surface_guards.rs`, `rrf_prefilter_gate.rs` (the rrf plan gate's differential oracle and fences), `repro_issue_563.rs` (`#[ignore]`d overflow-scale symptom tier) | @@ -237,7 +238,7 @@ S3-backed tests skip unless `OMNIGRAPH_S3_TEST_BUCKET` and the corresponding AWS Query behavior has two test tiers. A `.gqt` case owns what is visible in rows, counts, result column types, or errors. A Rust test owns what the format cannot express: mechanism, scale, process environment, concurrency. The claim that a v2 run is a function of its bound plan and the snapshot is of the second kind: no case reaches the replay door, and the GQT runner applies no replay check of its own to a step (its own checks are the report-shape parse of the `ran` lines' input, `schema_drift` and `check_pin`). The runner once replayed every v2 step against its report (two per-case replay invariants); that fence went when the plan started carrying everything execution reads, because a per-case replay in the same process found nothing the tests below do not find once, and cost every case a second run. The mapping from plan nodes to operators needs no check here: its rule is in [execution.md](execution.md#v2-plan-lowering-one-node-one-operator). -`crates/omnigraph/tests/engine_v2_plan_replay.rs` owns the replay. Each test gathers one query through `Session::query_inspected`, serializes the `BoundPlan` it emitted through the planner's mirrors, reads it back (it must read back equal), and executes it twice through `Session::replay_bound_plan`, a door that takes the bound plan and the engine context and nothing else about the query. Each replay must return the run's rows and repeat its trace: `id`, `operator` and `status` per row, `rung` and `ran` per attempt, and `actual_rows` wherever both attempts are `drained`. `drained` says the consumer pulled the operator's stream to its end; below an operator that stopped early (a `Limit`, a `Sort` with a fetch), a producer's count is a lower bound that depends on how far ahead its channel ran, and is not compared. One query per node kind that has a switch or a ladder: a `HashJoin` over an `Expand` (both switches on the report), a `nearest` ladder with an edge (two rungs), an `rrf` fusion, a `Limit` of zero (everything below skipped), a bulk `AntiJoin`, an `Aggregate`. Three more tests own the pins: after an edge write the replay of a traversal is refused, after an insert the replay of an unfiltered count is refused, and a write to a table the plan never read leaves the replay accepted, because `engine::plan_pins_snapshot` compares the plan's `Assumptions.datasets` (path, Lance branch and version per table key the planner read, or its absence) against the snapshot. +`crates/omnigraph/tests/engine_v2_plan_replay.rs` owns the replay. Each test gathers one query through `Session::query_inspected`, serializes the run's replay envelope (`Executed::replay_envelope`: the query's source and name, the accepted scope, the schema digest, the `BoundPlan` through the planner's mirrors and an exact-fragment member's derivation), reads it back (the plan must read back equal), and executes it twice through `Session::replay_bound_plan`, a door that recompiles the named query against the target's catalog, binds it to the plan's own values, accepts the plan again and takes no session setting. Each replay must return the run's rows, record the run's search decisions (gate verdicts and probe attempts) and repeat its trace: `id`, `operator` and `status` per row, `rung` and `ran` per attempt, and `actual_rows` wherever both attempts are `drained`. `drained` says the consumer pulled the operator's stream to its end; below an operator that stopped early (a `Limit`, a `Sort` with a fetch), a producer's count is a lower bound that depends on how far ahead its channel ran, and is not compared. One query per node kind that has a switch or a ladder: a `HashJoin` over an `Expand` (both switches on the report), a `nearest` ladder with an edge (two rungs), an `rrf` fusion, a `Limit` of zero (everything below skipped), a bulk `AntiJoin`, an `Aggregate`. Three more tests own the pins: after an edge write the replay of a traversal is refused, after an insert the replay of an unfiltered count is refused, and a write to a table the plan never read leaves the replay accepted, because `engine::plan_pins_snapshot` compares the plan's `Assumptions.datasets` (path, Lance branch and version per table key the planner read, or its absence) against the snapshot; a pin mismatch and a changed schema are conflicts. The envelope's own outcomes are tested there too: another envelope version asks for the query again, an exact-fragment member replays only with its unedited derivation, and an edited plan that drops a predicate is invalid evidence. The scrubbed side is `crates/omnigraph/tests/engine_v2_scrubbed_replay.rs` (process environment, which no case can express): a grep that fails on any `std::env` read under `crates/omnigraph/src/engine/` outside `plan_source.rs`, the one permitted reader of configuration, before planning; two replays that set an ambient value the plan did not capture (the task-local memory limit, `OMNIGRAPH_EXPAND_INDEXED_MAX_FRONTIER`) and require the captured one to win; and the replay of a plan gathered under one `ann_nprobes` in a session set to another. `Session::replay_bound_plan` takes no settings argument and sizes the pool from the plan's memory limit. The inventory sweep is one case per input class that a case can set two ways, `cases/v2/planner/input_*.gqt` (`engine`, `ann_nprobes`, index build state), each requiring equal rows or a plan line that differs, and the clock class is one of the replay plans of `engine_v2_plan_replay.rs`; the `ran` line and the `nprobes` claim of `--- expect plan` are what a differing plan line is spelled with. diff --git a/docs/rfcs/0047-search-plan-truth.md b/docs/rfcs/0047-search-plan-truth.md index 35a8ea311..805ed4ad2 100644 --- a/docs/rfcs/0047-search-plan-truth.md +++ b/docs/rfcs/0047-search-plan-truth.md @@ -7,7 +7,7 @@ implementation: in-progress authors: - Ragnor Comerford (@ragnorc) created: 2026-09-01 -updated: 2026-09-30 +updated: 2026-10-01 discussion: "https://github.com/ModernRelay/omnigraph/pull/791" supersedes: [] superseded_by: [] @@ -399,6 +399,11 @@ check the earlier translation. | Row cut | limit and the query stage it applies to | final truncation follows eligibility and the required order; keep nearest candidate caps and RRF arm caps distinct from this final cut; aggregate limits cut groups, ordinary limits cut rows | | Approximation | the retrieval kind's declared contract | preserve the exact/approximate classification; an exact execution of `nearest` does not change its declared contract | +A correlated block (`not`, `exists`, `count`, `sum`) is required as a block: +its aggregate, comparison and right-hand side. Its inner predicates are +checked through the block's plan subtree, not derived from the declaration's +inner clauses, whose variables the lowering may rename. + For a search-ordered aggregate, derive population requirements before grouping and order/cut requirements after grouping. With no remaining order key, group order stays unspecified as defined above. Full-text index readiness comes @@ -416,7 +421,14 @@ fact](#full-text-index-presence-as-a-planning-fact), not from query syntax. order, a Sort establishes its comparator's order, and a row limit preserves that order. These rules apply at the relevant input/output, including hidden score - columns; partition-local order does not establish a global order. + columns; partition-local order does not establish a global order. The + validator recomputes the order from the operators + (`optimizer::derived_order` over typed `OrderKey`s) rather than reading a + declared property. Three planner omissions in the identity tie-break are + comparator equivalences and accepted: an identity already in the + comparator, every identity when the query returns only order keys (rows + that tie are indistinguishable), and the identities of bindings the query + does not name (anonymous endpoints, cycle temps), after every named one. 3. Check required properties and recorded prerequisites before accepting the plan. Cost estimates select among accepted candidates and cannot discharge a missing requirement. @@ -444,6 +456,10 @@ belong to the following closed list; anything else uses invariant checks only. No implicit cast, arithmetic, division, function call other than the selected BM25, clock/random read, subquery, list operation, traversal, join, optional match, aggregation, distinct, offset, nearest or RRF belongs to this fragment. +As built, the admitted scalar types are Bool, I32, I64, U32, U64 and String, +nullable or not, never a list; an integer literal compares with any integer +property and a parameter only with its own declared type; a meta-field +(`@id`) is no admitted operand, sort key or projection item. There is no inference from a general expression's claimed determinism. A parameter is resolved once; rewriting cannot reread the clock or substitute a new value. The fragment extends only through a versioned rule addition. @@ -459,8 +475,11 @@ BM25. The planner records the facts forming `C`; replay verifies them against the accepted view. A changed eligible-id mask is not the same `C`. First reconstruct the canonical logical form from those checked inputs: -`Project(V, Limit(n, Sort(K, Filter(p, S))))`, where `p=true` when absent and -`S` is `Scan(C)` or `BM25(a, Scan(C))`. Match the compiler IR and initial +`Limit(n, Sort(K, fetch=n, T, Project(V, [BM25(a)] Filter(p, Scan(C)))))`, +the order the planner resolves a query in, where the projection carries +every sort key as a hidden column, `Filter` is absent when `p=true`, `Sort` +is absent for an unordered query, `T` is the identity tie-break and `fetch` +is the limit. Match the compiler IR and initial logical plan to this form using the rules below, including typed column/alias maps. This also checks any pre-planner constant substitution against the retained bound values. A declaration, expression or hidden score lost during @@ -469,8 +488,10 @@ well typed. A ***rewrite trace*** is a checked derivation from that form to the selected physical plan. It records rule ids, subtree references and typed substitutions. -The checker reconstructs each successor itself and compares its typed -structure to the proposed successor. Referenced nodes must already exist in +The checker reconstructs each successor itself. As built, the trace carries no +proposed successor: a successor is a function of its predecessor and the +typed substitution, so a serialized copy would duplicate what the checker +derives, and the final comparison with the candidate closes the derivation. Referenced nodes must already exist in the checked derivation. Node identity is established from validated contents; an optimizer-supplied id or hash is insufficient. The final reconstruction must equal the candidate, including all operator arguments, hidden-column @@ -489,6 +510,17 @@ The initial rule catalogue is closed: | Use ordered top-k | `Limit(n, Sort(K, fetch=f, X))` becomes `Sort(K, fetch=min(n,f), X)`, taking `min(n,None)=n`. The same checked child `X`, typed comparator `K` and hidden-column map are retained. The Sort specification emits that ordered prefix. | | Remove a redundant Sort | `Sort(K, fetch=None, X)` becomes `X` only when independently derived global ordering has the full typed `K` as a prefix. With `fetch=f`, the successor is `Limit(f,X)`. Direction, null placement, comparison semantics, tie keys and output column mapping must all match. | +As built in step 2a, the catalogue is the four rules the planner applies to +the fragment: `lower` (any of the lowering rows above; the sort lowers after +its ranking and leads with the score key), `absorb_scan_filter` (the +unranked absorption row, `T` being the fragment's predicate grammar), +`prune_scan_columns` (the scan reads a column set holding every column the +nodes above it read and only columns of its table; it replaces the +projection-placement row, since the planner prunes the scan rather than +placing a projection) and `rank_bm25_scan` (the BM25 lowering row, with the +eligibility placement below). The canonical sort already carries +`fetch=n`, so no top-k rule applies, and the planner removes no Sort. + `T` is a finite, versioned table from the admitted predicate's typed operators to typed scan-filter operators. Literal/parameter and property leaves retain their exact type, value and resolved identity; a comparison keeps its opcode @@ -499,8 +531,17 @@ entry names source and target implementations of the same specified semantics and has null/boundary conformance tests. If an entry is absent, the absorption rule is unavailable and the standalone Filter remains. This is an enumerated implementation assumption, not a runtime test of arbitrary predicate -equivalence. For BM25 the initial rules always keep eligibility above search; -they never absorb it into the scoring scan. +equivalence. As built, a BM25 scan absorbs eligibility like an unranked scan and +declares where it applies it: before scoring (Lance's prefilter) only under +the recorded full full-text coverage of the property, where Lance scores +every row from the index's statistics and the filter changes no score; +after scoring (Lance's postfilter, `prefilter(false)`) otherwise, which is +eligibility above search. Lance scores fragments no full-text segment covers +flat from statistics over the rows its prefilter admits, so filtering +before scoring there changed BM25 scores; the step 2a regression +`cases/v2/bm25_score_ignores_a_filter_over_an_unindexed_fragment.gqt` and the +Lance guard `fts_prefilter_changes_unindexed_scores_and_postfilter_keeps_them` +pin both facts. Projection carriers are internal columns, not returned fields. The outer visible projection may be implemented by a checked output-schema mapping, @@ -544,6 +585,18 @@ same accept, invalid or budget-exhausted result; costs do not decide validity. Wall-clock timeout and cancellation remain separate resource outcomes. Exhaustion is not evidence that a query or plan is invalid. +As built, `ValidationLimits::DEFAULT` allows 1 MiB of envelope bytes, +4,096 derivation nodes, 1,024 rule applications and 2^20 visits; a visit is +charged per expression and plan node checked and per conjunct an absorption +copies. A member's derivation grows linearly with its conjuncts (the +`instrument:` test `derivation_cost_grows_with_the_query` prints bytes, steps, +nodes, visits and time; 256 conjuncts take about 58 KB, 261 steps and 11 ms). +No planning-time memory pool exists, so evidence memory is bounded by the byte +and node limits instead of being charged to the query pool. The planner +always records a member's derivation; no optional rewrite is skipped for +budget, and exhaustion on a fresh plan is a resource outcome +(`ResourceLimitExceeded`). + If optional rewriting exceeds its evidence budget, use the already validated predecessor and its evidence. If baseline validation or replay exceeds the budget, return a typed resource-limit outcome naming the exhausted limit; @@ -579,6 +632,19 @@ trusted because it was serialized; any recorded branch observation must agree with the re-established evidence. This preserves the existing replay scope without asserting general equivalence between approximate search policies. +As built, a `nearest` scan declares a `NearestPolicy` (`probe_factor`, +`flat_rescan_on_unreached`, `uncapped_on_missing_counters`, +`flat_when_eligible_within_fetch`), and every pre-pass declares `on_empty` +(`proven_empty` for a standalone `nearest`, `postfilter` for a fusion) and +`coverage_admits`, the recorded full coverage of the bm25 scans it feeds. +The ladder and both gates read these from the plan. Acceptance requires a +probe factor of at least two and both fallbacks, every pre-pass hop to be a +required first hop of the query, feeds of the right kind, and the coverage +guard to match the recorded facts. The execution report records each gate's +verdict with its counts and each probe attempt per rung (`search` +decisions, also profile rows), and a replay reruns the gates against the +pinned snapshot and must record the same decisions. + Replay carries original query inputs and the evidence for its accepted scope, including the exact-subset derivation when applicable. Its envelope identifies the evidence format, rule catalogue and compiler/operator semantics versions. @@ -596,6 +662,14 @@ schema identities and snapshot facts remain fixed through execution. | The planner produces invalid evidence, or a supported baseline lacks its required check | Report an internal compiler/planner defect. Do not turn the validator gap into a new language refusal. | | A configured validation limit, timeout or cancellation ends checking | Return the resource outcome defined above; no accepted wrapper is constructed. | +As built, the replay envelope carries the query source and name, the scope, +the digest of the accepted schema, the bound plan (whose values are the +captured parameters) and a member's derivation, under `replay_version`, +`rules_version` and `semantics_version` 1. A version mismatch, a different +schema digest and a dataset the plan did not pin are conflicts (409) that ask +for the query again; malformed evidence or a plan failing a check is a bad +request (400); exhaustion is `ResourceLimitExceeded` (413). + Existing query-legality refusals and unavailable index prerequisites retain their distinct outcomes. Cross-version executable-plan portability is not promised; compatible query inputs can always be submitted for fresh planning. @@ -640,8 +714,9 @@ compiler with its code catalogue. `CompilerError::Query` carries it; its display is the legacy one-line form (`parse error: …`, `type error: T33: …`), so existing assertions and logic-test error needles keep holding. The planner builds the same type for its refusals, with `stage: plan` and the expression -it refused; `plan_source::plan_query` maps `Unrouted::UnsupportedQuery` to -that diagnostic instead of `no_plan`, which stays for genuine defects. The +it refused; the engine's planning door (`plan_source::unaccepted`) maps +`Unrouted::UnsupportedQuery` to that diagnostic instead of `no_plan`, which +stays for genuine defects. The server's `ErrorOutput` gains the optional `diagnostic` detail, so every existing error body is byte-identical. A declaration without its parameter list needs a grammar recognizer (`missing_param_list`), because the parser @@ -651,7 +726,10 @@ attempt it can name. ### Full-text index presence as a planning fact `PlanSource` gains one fact: whether a property's full-text index has built -segments at the pinned dataset version. The planner reads it while resolving +segments at the pinned dataset version. Step 2a introduced it as +`full_text_coverage` (`full`, `partial` or `absent`), recorded per ranked +property in `Assumptions.full_text` for the eligibility placement; step 3 +reads `absent` for the refusal. The planner reads it while resolving a ranked scan or a search predicate and refuses when it is absent. The read goes through the recording wrapper, so the fact is part of the plan's `Assumptions` and a replay against another snapshot is refused, as for every @@ -885,7 +963,7 @@ and regressions. Step 7 can ship independently. |---|---|---| | 1 | Diagnostics contract for parse and type refusals (PR #759, merged 2026-09-30) | None | | 2 | Planner refusals carry diagnostics; a refusal by design is a bad request | #786 | -| 2a | Shared query/explain validation and acceptance; exact-subset rules, adaptive-policy checks, bounded/versioned replay evidence, GQT scope assertions | supports steps 3-6 | +| 2a | Shared query/explain validation and acceptance; exact-subset rules, adaptive-policy checks, bounded/versioned replay evidence, GQT scope assertions. The order checks of a fusion and of a search-ordered aggregate land with step 5, which implements their order | supports steps 3-6 | | 3 | `T27` and `FullTextIndexRequired` | #747 | | 4 | The ranked binding roots its component; the `rrf()` arms type error | #789 | | 5 | One total order: fused score column, `Sort` over fusion and search-ordered aggregates, `T37` retired | #787, #788 | @@ -931,3 +1009,17 @@ None. machine-applicable only after the edited text parses. - 2026-09-30: retitled to "Search plan validation and result guarantees" to reflect shared validation and the explicit scope of search result guarantees. +- 2026-10-01: step 2a implemented, with these amendments where the + built planner differs from the text above and the behavior is sound: the + canonical form is the planner's resolve order, with the projection below + the sort; the derivation stores rule applications and substitutions, not + successors; the rule catalogue is `lower`, `absorb_scan_filter`, + `prune_scan_columns` and `rank_bm25_scan`; a BM25 scan filters before + scoring only under recorded full full-text coverage and after scoring + otherwise (which fixed filter-dependent scores on partially indexed + data); three tie-break omissions are comparator equivalences, and the + planner now orders declared bindings' identities before made-up ones (the + validator caught the old order); correlated blocks are required as + blocks; evidence memory is bounded by byte and node limits because no + planning pool exists; the fusion and aggregate order checks land with + step 5. diff --git a/docs/user/queries/explain.md b/docs/user/queries/explain.md index 138f7465a..9abed8876 100644 --- a/docs/user/queries/explain.md +++ b/docs/user/queries/explain.md @@ -67,7 +67,14 @@ value), `fetch` (the candidates the index is asked for: the query's limit for `nearest`, `null` for `bm25`, which is uncapped), `nprobes` on a `nearest` scan (the `ann_nprobes` setting when the plan was built, the IVF partitions probed per index delta; `null` is no cap) and `scope` (`order` -for the query's own order, `primary` or `secondary` for an arm of `rrf()`). A +for the query's own order, `primary` or `secondary` for an arm of `rrf()`). +A `nearest` scan's `policy` declares what the run may do when an attempt +leaves it short: `probe_factor` (each rescan multiplies the probe cap), +`flat_rescan_on_unreached`, `uncapped_on_missing_counters` and +`flat_when_eligible_within_fetch`. A `bm25` scan's `eligibility` says where it +applies its filter: `before_scoring` when the property's full-text index +covers every fragment of the pinned version, `after_scoring` otherwise, so a +filter never changes a BM25 score. A physical `RankFuse` row has two inputs, one subtree per arm, each with its own ranked `Scan`; it carries `arms` (binding and kind per arm), `k`, `limit` and @@ -86,14 +93,23 @@ carries no `runtime_filter` key unless a `ContainsJoin` marked it. Then one `plan` row per optimizer pass that fired (`node` `pass`), and one per remaining field of the explain document: `route` (the engine route the plan describes), `logical_hash` (the structural hash of the logical plan), -`explain_version`, `operation`, and the statistics used by planning. The +`explain_version`, `operation`, and the statistics used by planning; then +`validation`, what plan acceptance checked: `{"scope": "exact_subset"}` when +the query is a member of the exact fragment (one binding, Boolean, integer and +String filters, an optional leading `bm25()`, property sort keys, a limit and a +projection of properties and the score) and every rewrite from its canonical +form was checked, `{"scope": "invariants_only"}` when the plan was checked +against the query's requirements alone. A plan that fails acceptance is never +explained: the query fails as it would fail to run. The last `plan` row is `assumptions`, what the planner read to build the plan, as one JSON object: `params`, the names of the parameters it read (never a value); `settings`, each setting it read with its spelling (`traversal`, `ann_nprobes`); `env`, each environment variable it read with the value it resolved (`OMNIGRAPH_EXPAND_INDEXED_MAX_FRONTIER`, `OMNIGRAPH_EXPAND_INDEXED_MAX_HOPS`, when the query traverses); `gate_policy`, -the `rrf_plan` mode and the prefilter gate's admission thresholds; and +the `rrf_plan` mode and the prefilter gate's admission thresholds; +`full_text`, the full-text coverage (`full`, `partial` or `absent`) of every +property a `bm25` scan ranks, keyed `
.`; and `memory_limit`, the query pool in bytes. Selector statements also capture `traversal_work_limit`, shared across the execution, and `has_wildcard_traversal`, retained through rewrites so historical replay can enforce the target rule. Query @@ -105,8 +121,11 @@ ordering names `$p._distance asc` or `$p._score desc`, and the physical `Sort` above a search order leads its `keys` with that score key, followed by the query's plain keys; a fusion's ordering is `rrf($a, $b) desc` and it plans no `Sort`. A `Sort` row's `tiebreak` lists the metadata keys it appends after -`keys` (`$p.@id`, and `$e.@type` plus `$e.@id` for selected bound edges), empty -where equal rows are indistinguishable. +`keys` (`$p.@id`, and `$e.@type` plus `$e.@id` for selected bound edges, the +bindings the query declares in name order, then any binding the query does not +name), empty where equal rows are indistinguishable; its `ordering` property +is its whole comparator, those keys included as `$p.@id asc`, and a `Filter` +keeps its input's ordering. The `datafusion` tree is the plan the query executes on the `v2` route: the physical tree lowered to operators, every read operator omnigraph's own @@ -143,12 +162,19 @@ the node built: `ran` is `true` or `false` on an operator without a choice, and on the operator of a declared switch it is the side that ran: `hash_join` or `id_lookup` on the `HashJoinExec` of a `HashJoin`, `csr` or `indexed_scan` -on the `ExpandExec` of an `Expand`, the mode the traversal ended on. Explain itself runs nothing and carries no `profile` row; the +on the `ExpandExec` of an `Expand`, the mode the traversal ended on. After +the node rows, one row per search decision the run took within its plan's +declared policy: `node` `gate` (a pre-pass gate's `plan`, `prefilter`, +`postfilter` or `proven_empty`, with its `fallback`, `forced`, `eligible` and +`corpus`) or `probes` (a `nearest` scan's `attempts`, each with its +`maximum_nprobes`, `flat` and `rows`), each with the node's `id` and the +overfetch `rung`. Explain itself runs nothing and carries no `profile` row; the profile is returned beside the rows by the run that produced them (`Session::query_inspected`, the v2 inspection door, through `Executed::profile`). The row schema is `explain_version` 4. Saved physical -plans use a `bound_plan_version: 1` envelope whose `body` contains `plan` and -`values`. Older unversioned plans and unsupported versions are refused with a +plans use a `bound_plan_version: 2` envelope whose `body` contains `plan` and +`values`; version 2 types each node's ordering and declares ranked scans' +eligibility and policy. Older unversioned plans and unsupported versions are refused with a regeneration instruction, including plans without traversal nodes. An `explain` statement is served by `omnigraph query` and `POST /query`. It From 27bbf2b17ec7959ee4f4d55648abbb8565352361 Mon Sep 17 00:00:00 2001 From: Ragnor Comerford Date: Thu, 1 Oct 2026 20:25:48 +0200 Subject: [PATCH 12/29] planner: test derivation references to unreached and superseded nodes --- .../omnigraph-planner/src/validate/tests.rs | 23 +++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/crates/omnigraph-planner/src/validate/tests.rs b/crates/omnigraph-planner/src/validate/tests.rs index 4ca0fa320..ca578f5df 100644 --- a/crates/omnigraph-planner/src/validate/tests.rs +++ b/crates/omnigraph-planner/src/validate/tests.rs @@ -741,3 +741,26 @@ fn derivation_cost_grows_with_the_query() { ); } } + +/// A rule may name only a node the derivation has reached and not yet +/// replaced: a forward reference and a superseded node are both refused. +#[test] +fn a_reference_to_an_unreached_or_superseded_node_is_refused() { + let fixture = filtered(); + let plan = fixture.plan(); + let derivation = fixture.derivation().unwrap(); + let mut forward = derivation.clone(); + forward.steps[0].at = vec![usize::MAX, 0]; + let mut superseded = derivation.clone(); + let first = superseded.steps[0].at.clone(); + superseded.steps[1].at = first; + for broken in [forward, superseded] { + match fixture.accept_with(plan.clone(), Some(broken)) { + Err(ValidationError::Violated { check, detail }) => { + assert_eq!(check, "exact subset"); + assert!(detail.contains("is not"), "{detail}"); + } + other => panic!("{other:?}"), + } + } +} From 6d036ad455c6a07795dd4f60566a0a4a606493bc Mon Sep 17 00:00:00 2001 From: Ragnor Comerford Date: Thu, 1 Oct 2026 20:33:33 +0200 Subject: [PATCH 13/29] forbidden apis: register the three-way full-text coverage read --- crates/omnigraph/tests/forbidden_apis.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/omnigraph/tests/forbidden_apis.rs b/crates/omnigraph/tests/forbidden_apis.rs index 683f2019e..81ce1f7e9 100644 --- a/crates/omnigraph/tests/forbidden_apis.rs +++ b/crates/omnigraph/tests/forbidden_apis.rs @@ -730,7 +730,7 @@ gateway_surfaces! { "scan_proven_insert_delta_bounded", "include_proven_insert_blob_selection", "materialize_blob_batch", "scan_stream", "scan_stream_bounded", "scan_stream_with", "scan_plan_with", "ordered_scan_error", "scan", "scan_with", - "fts_covers_all_fragments", + "fts_coverage", "count_rows", "dataset_version", "table_state", "scan_with_staged", "scan_with_pending", "scan_with_pending_materialized_blobs", "count_rows_with_staged", From 7d4963bd171d1110b9b3763f70d1bbfff6378396 Mon Sep 17 00:00:00 2001 From: Ragnor Comerford Date: Thu, 1 Oct 2026 20:52:36 +0200 Subject: [PATCH 14/29] planner, engine: lowering-aware key binding; replay re-establishes coverage An order key the planner binds to a return item by equal lowering is accepted when the written key lowers to that item, though the two are written differently (`"a" in $d.tags` and `$d.tags contains "a"`), and a tie-break omission whose returned items all lower to order keys is accepted as an equivalence. Before this a legal query failed as a planner defect. The replay door re-establishes every full-text coverage fact a plan records from the pinned snapshot before acceptance: an envelope claiming full coverage of a partially indexed property, with its bm25 scan and derivation switched to filter before scoring, was accepted and would have changed BM25 scores. Both found in a Codex review. --- ...ound_to_an_equally_lowered_return_item.gqt | 51 ++++++++++++ .../src/validate/invariants.rs | 28 ++++++- crates/omnigraph/src/engine/mod.rs | 2 +- crates/omnigraph/src/engine/plan_source.rs | 37 +++++++++ crates/omnigraph/src/exec/query_doors.rs | 4 +- .../omnigraph/tests/engine_v2_plan_replay.rs | 77 +++++++++++++++++++ docs/dev/execution.md | 6 +- docs/dev/testing.md | 2 +- docs/rfcs/0047-search-plan-truth.md | 5 +- 9 files changed, 203 insertions(+), 9 deletions(-) create mode 100644 crates/omnigraph-gqt/cases/v2/order_key_bound_to_an_equally_lowered_return_item.gqt diff --git a/crates/omnigraph-gqt/cases/v2/order_key_bound_to_an_equally_lowered_return_item.gqt b/crates/omnigraph-gqt/cases/v2/order_key_bound_to_an_equally_lowered_return_item.gqt new file mode 100644 index 000000000..86a301fb9 --- /dev/null +++ b/crates/omnigraph-gqt/cases/v2/order_key_bound_to_an_equally_lowered_return_item.gqt @@ -0,0 +1,51 @@ +# issue: none +# notes: Two return items written differently can lower to one expression: `"a" in $d.tags` and +# notes: `$d.tags contains "a"` are the same list membership. An order key binds to the first +# notes: return item it lowers equal to, and ordering by either spelling, or by the alias of the +# notes: first, is accepted and orders the rows by that membership. + +--- runner +timeout_ms: 10000 +environments: + - target: omnigraph-engine + storage: local-filesystem + +--- schema +node Doc { + slug: String @key + tags: [String] +} + +--- seed +{"type":"Doc","data":{"slug":"d1","tags":["a"]}} +{"type":"Doc","data":{"slug":"d2","tags":["b"]}} + +--- query +query by_contains() { + match { $d: Doc } + return { "a" in $d.tags as a, $d.tags contains "a" as b } + order { $d.tags contains "a" } +} + +--- expect ordered +{"a": false, "b": false} +{"a": true, "b": true} + +--- expect shape +a: Bool +b: Bool + +--- query +query by_alias() { + match { $d: Doc } + return { "a" in $d.tags as a, $d.tags contains "a" as b } + order { a desc } +} + +--- expect ordered +{"a": true, "b": true} +{"a": false, "b": false} + +--- expect shape +a: Bool +b: Bool diff --git a/crates/omnigraph-planner/src/validate/invariants.rs b/crates/omnigraph-planner/src/validate/invariants.rs index 57bcac149..dce237ead 100644 --- a/crates/omnigraph-planner/src/validate/invariants.rs +++ b/crates/omnigraph-planner/src/validate/invariants.rs @@ -740,7 +740,9 @@ impl Requirements { } /// Whether a planned sort key is the written key: as written, or the - /// `return` item the planner bound it to by its result column. In a + /// `return` item the planner bound it to by its result column, which the + /// written key must lower to (two differently written expressions may + /// lower to one, and the planner binds the first such item). In a /// grouped query a written meta-field key binds by name to the unaliased /// return item whose column it names, as the compiler lowers it. fn same_key( @@ -768,7 +770,9 @@ impl Requirements { && item.alias.is_none() && meta_field_result_key(written).as_deref() == Some(column) && meta_field_result_key(&item.expr).as_deref() == Some(column); - return Ok(item.expr == *written || by_name); + return Ok(by_name + || item.expr == *written + || matcher.matches(written, &returns[index].expr, Position::Return, budget)?); } matcher.matches(written, &planned.expr, Position::Plain, budget) } @@ -825,7 +829,25 @@ impl Requirements { _ => false, } }); - if covered { + // The same claim over the lowered expressions: an expression a key + // lowers to, compared with what each returned item lowers to, since + // two differently written expressions may lower to one. + fn lowered_key<'k>(key: &'k IROrdering, returns: &'k [IRProjection]) -> &'k IRExpr { + match &key.expr { + IRExpr::AliasRef(alias) => returns + .iter() + .find(|projection| result_column(projection).as_deref() == Some(alias)) + .map_or(&key.expr, |projection| &projection.expr), + expr => expr, + } + } + let covered_lowered = !returns.is_empty() + && returns.iter().all(|projection| { + order_by + .iter() + .any(|key| *lowered_key(key, returns) == projection.expr) + }); + if covered || covered_lowered { return Ok(()); } let keyed = |column: &ColumnRef| { diff --git a/crates/omnigraph/src/engine/mod.rs b/crates/omnigraph/src/engine/mod.rs index 922715d9f..245e4ee07 100644 --- a/crates/omnigraph/src/engine/mod.rs +++ b/crates/omnigraph/src/engine/mod.rs @@ -73,7 +73,7 @@ pub(crate) use explain::{explain_document, explain_rows}; pub(crate) use graph::{EmbeddingResolver, GraphIndexHandle}; use lower::Lowering; use plan_source::{ExplainedQuery, QuerySource, accept_query, explain_query}; -pub(crate) use plan_source::{accept_replay, replay_refused}; +pub(crate) use plan_source::{accept_replay, replay_refused, replayed_coverage_holds}; pub(crate) use report::{Executed, PlanRun}; use report::{ExecutionReport, ReportRow}; use run::{pass_rows, run_plan}; diff --git a/crates/omnigraph/src/engine/plan_source.rs b/crates/omnigraph/src/engine/plan_source.rs index 8d734e4c9..4448ab8a8 100644 --- a/crates/omnigraph/src/engine/plan_source.rs +++ b/crates/omnigraph/src/engine/plan_source.rs @@ -476,6 +476,43 @@ pub(crate) fn accept_replay( omnigraph_planner::accept_replay(envelope, &input).map_err(replay_refused) } +/// Re-establish from the pinned snapshot every full-text coverage fact a +/// replayed plan records, since a serialized fact cannot vouch for itself: +/// a pinned dataset version fixes its index coverage, so a recorded value +/// that differs, or one for a table the plan pins no version of, is invalid +/// evidence. The dataset pins are checked first (`plan_pins_snapshot`). +pub(crate) async fn replayed_coverage_holds( + plan: &omnigraph_planner::PhysicalPlan, + snapshot: &Snapshot, +) -> Result<()> { + let refuse = |reason: String| replay_refused(ReplayRefusal::InvalidEvidence { reason }); + for (key, recorded) in &plan.assumptions().full_text { + let Some((type_key, property)) = key.rsplit_once('.') else { + return Err(refuse(format!( + "the plan records full-text coverage under `{key}`, which names no table property" + ))); + }; + if !plan.assumptions().datasets.contains_key(type_key) { + return Err(refuse(format!( + "the plan records full-text coverage of `{key}` without pinning `{type_key}`" + ))); + } + let actual = match snapshot.dataset(type_key) { + Some(_) => { + let dataset = snapshot.open_lance_dataset(type_key).await?; + crate::table_store::TableStore::fts_coverage(&dataset, property).await? + } + None => FullTextCoverage::Absent, + }; + if actual != *recorded { + return Err(refuse(format!( + "the plan records full-text coverage {recorded:?} of `{key}`; the pinned snapshot holds {actual:?}" + ))); + } + } + Ok(()) +} + /// A refused replay as the caller sees it: replanning is a conflict to /// resolve by resubmitting the query, invalid evidence is a bad request, an /// exhausted limit is a resource outcome. diff --git a/crates/omnigraph/src/exec/query_doors.rs b/crates/omnigraph/src/exec/query_doors.rs index 9700a8d19..e823acc2b 100644 --- a/crates/omnigraph/src/exec/query_doors.rs +++ b/crates/omnigraph/src/exec/query_doors.rs @@ -224,7 +224,8 @@ impl Session { /// `query_inspected` builds (the snapshot and catalog of `target`, a /// graph index scoped to the plan's `Expand`s), refuses a snapshot whose /// dataset versions are not the ones the plan's scans, counts and - /// traversals pinned, and calls the same `execute`. + /// traversals pinned, re-establishes every full-text coverage fact the + /// plan records from that snapshot, and calls the same `execute`. /// /// # Errors /// @@ -253,6 +254,7 @@ impl Session { has_wildcard, )?; engine::plan_pins_snapshot(plan, &resolved.snapshot)?; + engine::replayed_coverage_holds(plan, &resolved.snapshot).await?; let CompiledRead::Query(query) = self.compile_named_query(&catalog, &envelope.query.source, &envelope.query.name)? else { diff --git a/crates/omnigraph/tests/engine_v2_plan_replay.rs b/crates/omnigraph/tests/engine_v2_plan_replay.rs index fd466370e..ee0fc8f3c 100644 --- a/crates/omnigraph/tests/engine_v2_plan_replay.rs +++ b/crates/omnigraph/tests/engine_v2_plan_replay.rs @@ -1450,3 +1450,80 @@ async fn an_envelope_of_another_version_asks_for_the_query_again() { ); } } + +/// A replay re-establishes the full-text coverage its plan records from the +/// pinned snapshot instead of trusting the envelope: an envelope claiming +/// full coverage of a partially indexed property, with its bm25 scan and +/// derivation switched to filter before scoring, is refused, since that run +/// would change BM25 scores. +#[tokio::test] +async fn a_replay_rechecks_recorded_full_text_coverage() { + const SOURCE: &str = r#"query recent($t: String) { + match { $d: Doc $d.year >= 2000 } + return { $d.slug, bm25($d.text, $t) as score } + order { bm25($d.text, $t) } +}"#; + let dir = tempfile::tempdir().unwrap(); + let db = session( + Omnigraph::init( + dir.path().to_str().unwrap(), + "node Doc { slug: String @key text: String @index year: I64 }", + ) + .await + .unwrap(), + ); + db.load_jsonl( + r#"{"type":"Doc","data":{"slug":"d1","text":"graph engines","year":2020}} +{"type":"Doc","data":{"slug":"d2","text":"databases","year":2019}}"#, + LoadMode::Overwrite, + ) + .await + .unwrap(); + db.ensure_indices().await.unwrap(); + db.load_jsonl( + r#"{"type":"Doc","data":{"slug":"t1","text":"graph","year":2022}} +{"type":"Doc","data":{"slug":"t2","text":"graph graph graph graph","year":1990}}"#, + LoadMode::Append, + ) + .await + .unwrap(); + let params = ParamMap::from([("t".to_string(), Literal::String("graph".to_string()))]); + let run = db + .query_inspected(ReadTarget::branch("main"), SOURCE, "recent", ¶ms) + .await + .unwrap(); + let envelope = run.replay_envelope(SOURCE, "recent"); + let honest = db + .replay_bound_plan(ReadTarget::branch("main"), &envelope) + .await + .unwrap(); + assert_eq!(rows_of(&honest.result), rows_of(&run.result)); + let mut forged: Value = serde_json::from_slice(&envelope).unwrap(); + let mut placements = 0; + edit_objects(&mut forged, &mut |object| { + if object.get("eligibility") == Some(&Value::from("after_scoring")) { + object.insert("eligibility".to_string(), Value::from("before_scoring")); + placements += 1; + } + if let Some(Value::Object(coverage)) = object.get_mut("full_text") { + for recorded in coverage.values_mut() { + *recorded = Value::from("full"); + } + } + }); + assert_eq!(placements, 2, "the scan and its derivation step"); + let error = db + .replay_bound_plan( + ReadTarget::branch("main"), + &serde_json::to_vec(&forged).unwrap(), + ) + .await + .err() + .expect("a forged coverage claim must be refused"); + assert!( + matches!(&error, OmniError::Manifest(manifest) + if manifest.kind == omnigraph::error::ManifestErrorKind::BadRequest) + && error.to_string().contains("full-text coverage"), + "{error}" + ); +} diff --git a/docs/dev/execution.md b/docs/dev/execution.md index 7a2142af0..5a768f3ac 100644 --- a/docs/dev/execution.md +++ b/docs/dev/execution.md @@ -635,8 +635,10 @@ design keeps its diagnostic and is a bad request. The replay door source and name, the scope, the schema digest, the bound plan and a member's derivation. It checks the byte limit and the envelope's versions before decoding the plan, refuses another schema and a dataset the plan did not pin -as conflicts, recompiles the query, and accepts the plan again; a plan that -fails a check there is invalid evidence, a bad request. +as conflicts, re-establishes every full-text coverage fact the plan records +from the pinned snapshot, recompiles the query, and accepts the plan again; a +recorded fact the snapshot contradicts, or a plan that fails a check, is +invalid evidence, a bad request. ## Search and rank diff --git a/docs/dev/testing.md b/docs/dev/testing.md index 906b9a3ef..15a49f275 100644 --- a/docs/dev/testing.md +++ b/docs/dev/testing.md @@ -262,7 +262,7 @@ S3-backed tests skip unless `OMNIGRAPH_S3_TEST_BUCKET` and the corresponding AWS Query behavior has two test tiers. A `.gqt` case owns what is visible in rows, counts, result column types, or errors. A Rust test owns what the format cannot express: mechanism, scale, process environment, concurrency. The claim that a v2 run is a function of its bound plan and the snapshot is of the second kind: no case reaches the replay door, and the GQT runner applies no replay check of its own to a step (its own checks are the report-shape parse of the `ran` lines' input, `schema_drift` and `check_pin`). The runner once replayed every v2 step against its report (two per-case replay invariants); that fence went when the plan started carrying everything execution reads, because a per-case replay in the same process found nothing the tests below do not find once, and cost every case a second run. The mapping from plan nodes to operators needs no check here: its rule is in [execution.md](execution.md#v2-plan-lowering-one-node-one-operator). -`crates/omnigraph/tests/engine_v2_plan_replay.rs` owns the replay. Each test gathers one query through `Session::query_inspected`, serializes the run's replay envelope (`Executed::replay_envelope`: the query's source and name, the accepted scope, the schema digest, the `BoundPlan` through the planner's mirrors and an exact-fragment member's derivation), reads it back (the plan must read back equal), and executes it twice through `Session::replay_bound_plan`, a door that recompiles the named query against the target's catalog, binds it to the plan's own values, accepts the plan again and takes no session setting. Each replay must return the run's rows, record the run's search decisions (gate verdicts and probe attempts) and repeat its trace: `id`, `operator` and `status` per row, `rung` and `ran` per attempt, and `actual_rows` wherever both attempts are `drained`. `drained` says the consumer pulled the operator's stream to its end; below an operator that stopped early (a `Limit`, a `Sort` with a fetch), a producer's count is a lower bound that depends on how far ahead its channel ran, and is not compared. One query per node kind that has a switch or a ladder: a `HashJoin` over an `Expand` (both switches on the report), a `nearest` ladder with an edge (two rungs), an `rrf` fusion, a `Limit` of zero (everything below skipped), a bulk `AntiJoin`, an `Aggregate`. Three more tests own the pins: after an edge write the replay of a traversal is refused, after an insert the replay of an unfiltered count is refused, and a write to a table the plan never read leaves the replay accepted, because `engine::plan_pins_snapshot` compares the plan's `Assumptions.datasets` (path, Lance branch and version per table key the planner read, or its absence) against the snapshot; a pin mismatch and a changed schema are conflicts. The envelope's own outcomes are tested there too: another envelope version asks for the query again, an exact-fragment member replays only with its unedited derivation, and an edited plan that drops a predicate is invalid evidence. +`crates/omnigraph/tests/engine_v2_plan_replay.rs` owns the replay. Each test gathers one query through `Session::query_inspected`, serializes the run's replay envelope (`Executed::replay_envelope`: the query's source and name, the accepted scope, the schema digest, the `BoundPlan` through the planner's mirrors and an exact-fragment member's derivation), reads it back (the plan must read back equal), and executes it twice through `Session::replay_bound_plan`, a door that recompiles the named query against the target's catalog, binds it to the plan's own values, accepts the plan again and takes no session setting. Each replay must return the run's rows, record the run's search decisions (gate verdicts and probe attempts) and repeat its trace: `id`, `operator` and `status` per row, `rung` and `ran` per attempt, and `actual_rows` wherever both attempts are `drained`. `drained` says the consumer pulled the operator's stream to its end; below an operator that stopped early (a `Limit`, a `Sort` with a fetch), a producer's count is a lower bound that depends on how far ahead its channel ran, and is not compared. One query per node kind that has a switch or a ladder: a `HashJoin` over an `Expand` (both switches on the report), a `nearest` ladder with an edge (two rungs), an `rrf` fusion, a `Limit` of zero (everything below skipped), a bulk `AntiJoin`, an `Aggregate`. Three more tests own the pins: after an edge write the replay of a traversal is refused, after an insert the replay of an unfiltered count is refused, and a write to a table the plan never read leaves the replay accepted, because `engine::plan_pins_snapshot` compares the plan's `Assumptions.datasets` (path, Lance branch and version per table key the planner read, or its absence) against the snapshot; a pin mismatch and a changed schema are conflicts. The envelope's own outcomes are tested there too: another envelope version asks for the query again, an exact-fragment member replays only with its unedited derivation, an edited plan that drops a predicate is invalid evidence, and an envelope claiming full full-text coverage of a partially indexed property is refused against the pinned snapshot. The scrubbed side is `crates/omnigraph/tests/engine_v2_scrubbed_replay.rs` (process environment, which no case can express): a grep that fails on any `std::env` read under `crates/omnigraph/src/engine/` outside `plan_source.rs`, the one permitted reader of configuration, before planning; two replays that set an ambient value the plan did not capture (the task-local memory limit, `OMNIGRAPH_EXPAND_INDEXED_MAX_FRONTIER`) and require the captured one to win; and the replay of a plan gathered under one `ann_nprobes` in a session set to another. `Session::replay_bound_plan` takes no settings argument and sizes the pool from the plan's memory limit. The inventory sweep is one case per input class that a case can set two ways, `cases/v2/planner/input_*.gqt` (`engine`, `ann_nprobes`, index build state), each requiring equal rows or a plan line that differs, and the clock class is one of the replay plans of `engine_v2_plan_replay.rs`; the `ran` line and the `nprobes` claim of `--- expect plan` are what a differing plan line is spelled with. diff --git a/docs/rfcs/0047-search-plan-truth.md b/docs/rfcs/0047-search-plan-truth.md index 805ed4ad2..d1d7d2450 100644 --- a/docs/rfcs/0047-search-plan-truth.md +++ b/docs/rfcs/0047-search-plan-truth.md @@ -668,7 +668,10 @@ captured parameters) and a member's derivation, under `replay_version`, `rules_version` and `semantics_version` 1. A version mismatch, a different schema digest and a dataset the plan did not pin are conflicts (409) that ask for the query again; malformed evidence or a plan failing a check is a bad -request (400); exhaustion is `ResourceLimitExceeded` (413). +request (400); exhaustion is `ResourceLimitExceeded` (413). The recorded +full-text coverage is data-dependent evidence: replay reads it again from the +pinned datasets, and a recorded value the snapshot contradicts is invalid +evidence. Existing query-legality refusals and unavailable index prerequisites retain their distinct outcomes. Cross-version executable-plan portability is not From d69570923c6ff24d0052f3f56b841e9785b85348 Mon Sep 17 00:00:00 2001 From: Ragnor Comerford Date: Sun, 4 Oct 2026 17:34:33 +0200 Subject: [PATCH 15/29] planner, engine: constant aliased order keys; fusion arms report their probes Two defects a Codex review found in plan acceptance and the run report. An order key that is constant for every row (a parameter, `now()`, a single edge type's `@type`) orders nothing, and the order check discounts such keys on both sides. When `return` projects the constant under an alias, the planner binds the key to that alias, and the check compared an unresolved alias on one side with a discounted constant on the other, so `return { $x as x } order { $x }` failed acceptance as a planner defect. Both sides now resolve an alias to the `return` item it names before discounting constants. A fusion recorded its gate decision and returned without its nearest arms' probe attempts, so inspection and replay could not see them, and every nearest scan wrote one shared report, so two nearest arms would overwrite each other. Each nearest scan now reports under its own plan node; the standalone search reads its scan's entry and a fusion records each arm's attempts after its gate. Regressions: cases/v2/constant_order_key_bound_to_a_return_alias.gqt, validate/tests.rs::a_constant_key_bound_to_a_return_alias_is_accepted, and engine_v2_plan_replay.rs::a_fusion_replays, extended with a two-arm rrf(nearest, nearest). --- ...tant_order_key_bound_to_a_return_alias.gqt | 73 +++++++++++++++++++ .../src/validate/invariants.rs | 19 +++-- .../omnigraph-planner/src/validate/tests.rs | 23 ++++++ crates/omnigraph/src/engine/lower.rs | 10 ++- crates/omnigraph/src/engine/mod.rs | 31 +++++++- .../src/engine/operators/scan/mod.rs | 17 +++-- crates/omnigraph/src/engine/search.rs | 10 ++- .../omnigraph/tests/engine_v2_plan_replay.rs | 37 ++++++++-- 8 files changed, 193 insertions(+), 27 deletions(-) create mode 100644 crates/omnigraph-gqt/cases/v2/constant_order_key_bound_to_a_return_alias.gqt diff --git a/crates/omnigraph-gqt/cases/v2/constant_order_key_bound_to_a_return_alias.gqt b/crates/omnigraph-gqt/cases/v2/constant_order_key_bound_to_a_return_alias.gqt new file mode 100644 index 000000000..5d7666b03 --- /dev/null +++ b/crates/omnigraph-gqt/cases/v2/constant_order_key_bound_to_a_return_alias.gqt @@ -0,0 +1,73 @@ +# issue: none +# notes: An order key that is constant for every row (a parameter, or a single edge type's +# notes: `@type` in a grouped query) orders nothing. Written as the expression or as its +# notes: `return` alias, the planner binds it to that alias, and acceptance sees the key as the +# notes: constant it names: the rows fall to the identity tie-break, and the query is accepted. + +--- runner +timeout_ms: 10000 +environments: + - target: omnigraph-engine + storage: local-filesystem + +--- schema +node Person { + name: String @key +} +edge Knows: Person -> Person + +--- seed +{"type":"Person","data":{"name":"b"}} +{"type":"Person","data":{"name":"a"}} +{"edge":"Knows","from":"a","to":"b"} +{"edge":"Knows","from":"b","to":"a"} + +--- query +query by_parameter($x: I64) { + match { $p: Person } + return { $p.name, $x as x } + order { $x } +} + +--- params +{"x": 7} + +--- expect ordered +{"p.name": "a", "x": 7} +{"p.name": "b", "x": 7} + +--- expect shape +p.name: String +x: I64 + +--- query +query by_parameter_alias($x: I64) { + match { $p: Person } + return { $p.name, $x as x } + order { x desc } +} + +--- params +{"x": 7} + +--- expect ordered +{"p.name": "a", "x": 7} +{"p.name": "b", "x": 7} + +--- expect shape +p.name: String +x: I64 + +--- query +query by_single_edge_type() { + match { $from: Person $from $edge: knows $to } + return { $edge.@type as edge_type, count($to) as n } + order { $edge.@type } +} + +--- expect unordered +{"edge_type": "Knows", "n": 2} + +--- expect shape +edge_type: String +n: I64 diff --git a/crates/omnigraph-planner/src/validate/invariants.rs b/crates/omnigraph-planner/src/validate/invariants.rs index dce237ead..41c436b02 100644 --- a/crates/omnigraph-planner/src/validate/invariants.rs +++ b/crates/omnigraph-planner/src/validate/invariants.rs @@ -636,10 +636,12 @@ impl Requirements { if score.is_some() && self.aggregate { return Ok(()); } + // A key is constant by what it names: an alias by the `return` item it + // aliases, on the written side and on the planned side alike. let written: Vec<&omnigraph_compiler::query::ast::Ordering> = self .order .iter() - .filter(|key| !matcher.constant(&key.expr)) + .filter(|key| !matcher.constant(self.resolved_key(&key.expr))) .collect(); if score.is_none() && self.order.is_empty() { let sorted = top @@ -708,7 +710,17 @@ impl Requirements { )); } } - let rest: Vec<&IROrdering> = keys.filter(|key| !constant_ir(&key.expr)).collect(); + let returns = projection_of(plan, top) + .map(Vec::as_slice) + .unwrap_or_default(); + let planned_constant = |expr: &IRExpr| match expr { + IRExpr::AliasRef(column) => returns + .iter() + .find(|projection| result_column(projection).as_deref() == Some(column)) + .is_some_and(|projection| constant_ir(&projection.expr)), + other => constant_ir(other), + }; + let rest: Vec<&IROrdering> = keys.filter(|key| !planned_constant(&key.expr)).collect(); if rest.len() != written.len() { return Err(ValidationError::violated( "order", @@ -719,9 +731,6 @@ impl Requirements { ), )); } - let returns = projection_of(plan, top) - .map(Vec::as_slice) - .unwrap_or_default(); for (key, planned) in written.iter().zip(&rest) { if key.descending != planned.descending || !self.same_key(&key.expr, planned, returns, matcher, budget)? diff --git a/crates/omnigraph-planner/src/validate/tests.rs b/crates/omnigraph-planner/src/validate/tests.rs index ca578f5df..b22be112f 100644 --- a/crates/omnigraph-planner/src/validate/tests.rs +++ b/crates/omnigraph-planner/src/validate/tests.rs @@ -387,6 +387,29 @@ fn another_limit_fails_the_row_cut() { assert_eq!(check, "row cut"); } +/// An order key constant for every row (a parameter, `now()`) orders +/// nothing; when `return` projects it under an alias the planner binds the key +/// to that alias, and the order check treats the alias as the constant it +/// projects. +#[test] +fn a_constant_key_bound_to_a_return_alias_is_accepted() { + for (query, params) in [ + ( + "query q($x: I64) { match { $d: Doc } return { $d.slug, $x as x } order { $x } }", + vec![("x", Literal::Integer(7))], + ), + ( + "query q() { match { $d: Doc } return { $d.slug, now() as t } order { now() } }", + vec![], + ), + ] { + let fixture = Fixture::new(query, ¶ms); + fixture + .accept(fixture.plan()) + .unwrap_or_else(|error| panic!("{query}: {error:?}")); + } +} + #[test] fn a_dropped_or_reversed_key_fails_the_order() { let fixture = ranked(); diff --git a/crates/omnigraph/src/engine/lower.rs b/crates/omnigraph/src/engine/lower.rs index 1f4ff590f..732d5df79 100644 --- a/crates/omnigraph/src/engine/lower.rs +++ b/crates/omnigraph/src/engine/lower.rs @@ -61,8 +61,9 @@ pub(super) struct Lowered { pub(super) root: Plan, /// The one operator of every live node, the tree `root` heads. pub(super) operators: HashMap, - /// What the ranked scan reported in this pass, for the overfetch ladder. - pub(super) report: Arc>, + /// What each nearest scan reported in this pass, by plan node: the + /// overfetch ladder's input and the run's probe decisions. + pub(super) report: Arc>, /// The in-memory filter applications of the tree, one per conjunct. in_memory_filters: usize, } @@ -332,7 +333,7 @@ struct Walk<'l, 'a> { lowering: &'l Lowering<'a>, pass: &'l Pass, scope: Scope, - report: Arc>, + report: Arc>, outers: Vec, operators: HashMap, in_memory_filters: usize, @@ -358,7 +359,7 @@ impl<'l, 'a> Walk<'l, 'a> { }), ctx: Arc::new(ProjectionContext::for_plan(lowering.catalog, lowering.plan)), }, - report: Arc::new(Mutex::new(ScanReport::default())), + report: Arc::new(Mutex::new(ScanReports::new())), outers: Vec::new(), operators: HashMap::new(), in_memory_filters: 0, @@ -452,6 +453,7 @@ impl Lower for Walk<'_, '_> { ScanSource::Table { mode: Box::new(mode), report: Arc::clone(&self.report), + node: id, } } _ => { diff --git a/crates/omnigraph/src/engine/mod.rs b/crates/omnigraph/src/engine/mod.rs index 245e4ee07..b846b1984 100644 --- a/crates/omnigraph/src/engine/mod.rs +++ b/crates/omnigraph/src/engine/mod.rs @@ -174,7 +174,7 @@ async fn run_once( ctx: &QueryContext, pass: &Pass, rung: usize, -) -> Result<(RecordBatch, ScanReport, Vec)> { +) -> Result<(RecordBatch, ScanReports, Vec)> { let lowered = lowering.lower_query(pass)?; lowered.record_in_memory_filters(); let batch = run_plan(&lowered, lowering.plan, ctx).await?; @@ -439,6 +439,23 @@ pub(crate) async fn execute( lowered.record_in_memory_filters(); let fused = Box::pin(run_plan(&lowered, &bound.plan, &ctx)).await?; executed.record(pass_rows(&lowered, &bound.plan, 0)?); + // Each nearest arm's probe ladder, by its scan, after the fusion's gate. + let reports = lowered + .report + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .clone(); + for (scan, report) in reports { + if !report.probes.is_empty() { + executed.decide( + scan, + 0, + report::Taken::Probes { + attempts: report.probes, + }, + ); + } + } return Ok(PlanRun { result: QueryResult::new(fused.schema(), vec![fused]), plan: bound, @@ -464,8 +481,15 @@ pub(crate) async fn execute( pass = next; } - let (result_batch, report, rows) = Box::pin(run_once(&lowering, &ctx, &pass, 0)).await?; + let (result_batch, reports, rows) = Box::pin(run_once(&lowering, &ctx, &pass, 0)).await?; executed.record(rows); + let nearest_report = |mut reports: ScanReports| { + nearest + .as_ref() + .and_then(|scan| reports.remove(&scan.id)) + .unwrap_or_default() + }; + let report = nearest_report(reports); if let Some(NearestScan { id, .. }) = &nearest && !report.probes.is_empty() { @@ -539,9 +563,10 @@ pub(crate) async fn execute( pass.with_exact_nearest(id, k) } }; - let (retried, retried_report, rows) = + let (retried, retried_reports, rows) = Box::pin(run_once(&lowering, &ctx, &wider, rung)).await?; executed.record(rows); + let retried_report = nearest_report(retried_reports); if !retried_report.probes.is_empty() { executed.decide( id, diff --git a/crates/omnigraph/src/engine/operators/scan/mod.rs b/crates/omnigraph/src/engine/operators/scan/mod.rs index 7faa3ff08..9f7da6949 100644 --- a/crates/omnigraph/src/engine/operators/scan/mod.rs +++ b/crates/omnigraph/src/engine/operators/scan/mod.rs @@ -18,7 +18,7 @@ use omnigraph_compiler::ir::{IRExpr, ParamMap}; use super::{breaker_properties, breaker_stream, conform, external, polled, streaming_properties}; use crate::db::Snapshot; use crate::engine::scan::{execute_node_scan, prefix_batch, scan_output_schema}; -use crate::engine::search::{NeededColumns, ScanReport, SearchMode}; +use crate::engine::search::{NeededColumns, ScanReport, ScanReports, SearchMode}; use crate::error::Result; mod input; @@ -32,7 +32,9 @@ pub(super) use runtime_filter::{Filled, Needles}; pub(crate) enum ScanSource { Table { mode: Box, - report: Arc>, + /// Where a nearest scan reports its ladder, under `node`. + report: Arc>, + node: omnigraph_planner::NodeId, }, Dependent { input: Arc, @@ -207,13 +209,15 @@ impl ExecutionPlan for ScanExec { ctx: Arc, ) -> DfResult { assert_eq!(partition, 0, "ScanExec has one partition"); - let (mode, report) = match &self.source { + let (mode, report, node) = match &self.source { ScanSource::Dependent { input } => { return self .execute_input(input.execute(0, Arc::clone(&ctx))?, ctx) .map(|stream| polled(&self.metrics, stream)); } - ScanSource::Table { mode, report } => (mode.as_ref().clone(), Arc::clone(report)), + ScanSource::Table { mode, report, node } => { + (mode.as_ref().clone(), Arc::clone(report), *node) + } }; if let Some(slot) = self.runtime_filter.as_ref().filter(|_| self.pipelines()) { return self.execute_pipelined(mode, slot.take(), ctx); @@ -262,9 +266,10 @@ impl ExecutionPlan for ScanExec { reservation.metric("nearest_dataset_rows", nearest.dataset_rows as usize); } if scan_report.nearest_scan.is_some() { - *report + report .lock() - .unwrap_or_else(std::sync::PoisonError::into_inner) = scan_report; + .unwrap_or_else(std::sync::PoisonError::into_inner) + .insert(node, scan_report); } let prefixed = prefix_batch(&batch, &binding).map_err(external)?; conform(prefixed, &declared).map_err(external) diff --git a/crates/omnigraph/src/engine/search.rs b/crates/omnigraph/src/engine/search.rs index c751f91fe..e91fe691c 100644 --- a/crates/omnigraph/src/engine/search.rs +++ b/crates/omnigraph/src/engine/search.rs @@ -144,9 +144,13 @@ impl std::fmt::Debug for EligibleIds { } } -/// What the nearest scan reported back to the query level: the LAST scan of -/// the ranked variable in a pass. `rows == k` means the scan was full, so a -/// shortfall above it can only be recovered by asking for more candidates. +/// What each nearest table scan of one pass reported, by its plan node: a +/// standalone search order has one, an `rrf()` one per nearest arm. +pub(super) type ScanReports = std::collections::BTreeMap; + +/// What one nearest scan reported back to the query level in a pass. +/// `rows == k` means the scan was full, so a shortfall above it can only be +/// recovered by asking for more candidates. #[derive(Debug, Default, Clone)] pub(super) struct ScanReport { pub(super) nearest_scan: Option, diff --git a/crates/omnigraph/tests/engine_v2_plan_replay.rs b/crates/omnigraph/tests/engine_v2_plan_replay.rs index ee0fc8f3c..24774b5e5 100644 --- a/crates/omnigraph/tests/engine_v2_plan_replay.rs +++ b/crates/omnigraph/tests/engine_v2_plan_replay.rs @@ -548,6 +548,12 @@ query fused($t: String, $q: Vector(4)) { order { rrf(nearest($d.embedding, $q), bm25($d.text, $t)) } limit 3 } +query fused_vectors($q: Vector(4), $r: Vector(4)) { + match { $d: Doc } + return { $d.slug } + order { rrf(nearest($d.embedding, $q), nearest($d.embedding, $r)) } + limit 3 +} "#; const CITATION_SCHEMA: &str = r#" @@ -965,13 +971,32 @@ async fn a_fusion_replays() { params.insert("q".to_string(), Literal::List(vec![Literal::Float(0.0); 4])); let Replayed { result, search, .. } = replayed(&db, DOC_QUERIES, "fused", ¶ms).await; assert_eq!(result.len(), 3); + let kinds: Vec<&str> = search + .iter() + .map(|decision| decision["decision"].as_str().unwrap()) + .collect(); assert_eq!( - search - .iter() - .filter(|decision| decision["decision"] == "gate") - .count(), - 1, - "the fusion's gate records one verdict: {search:#?}" + kinds, + ["gate", "probes"], + "the fusion's gate records one verdict, then its nearest arm its probe attempts: {search:#?}" + ); + + // Two nearest arms record their attempts apart, one decision per scan. + let mut params = + ParamMap::from([("q".to_string(), Literal::List(vec![Literal::Float(0.0); 4]))]); + params.insert("r".to_string(), Literal::List(vec![Literal::Float(9.0); 4])); + let Replayed { result, search, .. } = + replayed(&db, DOC_QUERIES, "fused_vectors", ¶ms).await; + assert_eq!(result.len(), 3); + let probed: Vec = search + .iter() + .filter(|decision| decision["decision"] == "probes") + .map(|decision| decision["id"].as_u64().unwrap()) + .collect(); + assert_eq!(probed.len(), 2, "{search:#?}"); + assert_ne!( + probed[0], probed[1], + "each arm records under its own scan: {search:#?}" ); } From 8df5788072382c110c0922ba7376ee60d3b07b70 Mon Sep 17 00:00:00 2001 From: Ragnor Comerford Date: Mon, 5 Oct 2026 14:31:07 +0200 Subject: [PATCH 16/29] planner: refuse an rrf over two connected bindings without a reorder fix P001's fix, declare the ranked binding first, cannot answer an rrf() whose arms rank two bindings one traversal connects: the binding declared first starts the traversal and the other is reached, so following the fix only produced the opposite hint. Such a fusion is now refused before either arm is ranked, as P005, which names both bindings and the fused call and offers no fix. The reorder hint stays for a single ranked binding. The issue-786 owner test now runs the fused shape in both declaration orders and requires the same fixless refusal. --- .../planner-refusal-diagnostic.fixed.md | 4 +- crates/omnigraph-compiler/src/query/codes.rs | 1 + crates/omnigraph-planner/src/optimizer.rs | 65 ++++++++++++++++++- .../omnigraph/tests/engine_v2_plan_replay.rs | 30 +++++++++ docs/user/queries/diagnostics.md | 1 + 5 files changed, 99 insertions(+), 2 deletions(-) diff --git a/changelog.d/planner-refusal-diagnostic.fixed.md b/changelog.d/planner-refusal-diagnostic.fixed.md index a52b8ca09..c4bb438f3 100644 --- a/changelog.d/planner-refusal-diagnostic.fixed.md +++ b/changelog.d/planner-refusal-diagnostic.fixed.md @@ -1,6 +1,8 @@ - A query shape the planner refuses by design, a `nearest()` or `bm25()` ranking on a binding a traversal reaches, is a bad request with the planner code `P001`, the refused expression and a fix that answers the query: - declare that binding first in `match`. See the [diagnostics guide][p001-diagnostics]. + declare that binding first in `match`. An `rrf()` whose arms rank two + bindings one traversal connects is refused as `P005` without a fix, since no + declaration order serves both arms. See the [diagnostics guide][p001-diagnostics]. [p001-diagnostics]: ../docs/user/queries/diagnostics.md diff --git a/crates/omnigraph-compiler/src/query/codes.rs b/crates/omnigraph-compiler/src/query/codes.rs index ac85789b1..26b65b82d 100644 --- a/crates/omnigraph-compiler/src/query/codes.rs +++ b/crates/omnigraph-compiler/src/query/codes.rs @@ -76,6 +76,7 @@ codes! { P002 = "an edge alternation or wildcard traversal needs a finite traversal work limit"; P003 = "the traversal work limit is out of range or recorded twice"; P004 = "an edge alternation or wildcard traversal cannot run in CSR traversal mode"; + P005 = "an rrf() ranks two bindings one traversal connects"; } #[cfg(test)] diff --git a/crates/omnigraph-planner/src/optimizer.rs b/crates/omnigraph-planner/src/optimizer.rs index 8d3bcbf74..fb6a7739f 100644 --- a/crates/omnigraph-planner/src/optimizer.rs +++ b/crates/omnigraph-planner/src/optimizer.rs @@ -9,7 +9,7 @@ use arrow_schema::{DataType, Field, Schema, SchemaRef}; use omnigraph_compiler::QueryDiagnostic; use omnigraph_compiler::ir::{IRExpr, IROp, IROrdering, IRProjection, QueryIR}; use omnigraph_compiler::query::ast::AggFunc; -use omnigraph_compiler::query::codes::{P001, P002, P004}; +use omnigraph_compiler::query::codes::{P001, P002, P004, P005}; use omnigraph_compiler::settings::Traversal; use omnigraph_compiler::traversal::{EDGE_TYPE_COLUMN, EdgeSelection}; @@ -1124,6 +1124,32 @@ pub fn physical_plan( /// The nodes of the pipeline under `root` that constrain its rows: every /// node reachable from `root` without entering an `AntiJoin`'s inner tree. +/// Whether a traversal of the top-level pipeline under `root` connects +/// bindings `a` and `b`, through any chain of `Expand`s. +fn traversal_connects(plan: &PhysicalPlan, root: NodeId, a: &str, b: &str) -> bool { + let edges: Vec<(&str, &str)> = top_level(plan, root) + .into_iter() + .filter_map(|id| match plan.node(id) { + Some(PhysicalNode::Expand { src, dst, .. }) => Some((src.as_str(), dst.as_str())), + _ => None, + }) + .collect(); + let mut reached = vec![a]; + let mut grew = true; + while grew { + grew = false; + for (src, dst) in &edges { + for (from, to) in [(src, dst), (dst, src)] { + if reached.contains(from) && !reached.contains(to) { + reached.push(to); + grew = true; + } + } + } + } + reached.contains(&b) +} + fn top_level(plan: &PhysicalPlan, root: NodeId) -> Vec { let mut nodes = Vec::new(); let mut pending = vec![root]; @@ -2545,6 +2571,43 @@ impl Lowering<'_> { })?; let limit = limit.and_then(|limit| usize::try_from(limit).ok()); let [primary_arm, secondary_arm] = arms; + if primary_arm.binding != secondary_arm.binding + && traversal_connects( + &self.physical, + primary, + &primary_arm.binding, + &secondary_arm.binding, + ) + { + // One of two bindings a traversal connects always starts + // it and the other is reached, and engine v2 ranks only the + // binding a traversal starts from: no declaration order + // serves both arms, so the refusal offers none. + let call = |arm: &SearchArm| { + let function = match arm.kind { + RankKind::Nearest => "nearest", + RankKind::Bm25 => "bm25", + }; + format!( + "{function}(${}.{}, {})", + arm.binding, arm.property, arm.query + ) + }; + return Err(PlanError::Unsupported(Box::new( + QueryDiagnostic::plan( + P005, + format!( + "`rrf()` ranks `${}` and `${}`, which one traversal connects; engine v2 ranks only the binding a traversal starts from, so one arm always ranks a traversal destination and no declaration order serves both", + primary_arm.binding, secondary_arm.binding + ), + ) + .with_expression(format!( + "rrf({}, {})", + call(primary_arm), + call(secondary_arm) + )), + ))); + } let mut lowered_arms = Vec::with_capacity(2); let mut bm25_scans = Vec::new(); let mut primary_scan = None; diff --git a/crates/omnigraph/tests/engine_v2_plan_replay.rs b/crates/omnigraph/tests/engine_v2_plan_replay.rs index 07863d61a..9e5216179 100644 --- a/crates/omnigraph/tests/engine_v2_plan_replay.rs +++ b/crates/omnigraph/tests/engine_v2_plan_replay.rs @@ -1053,6 +1053,36 @@ query nearest_destination($q: Vector(4)) { ); } + // An `rrf()` whose arms rank two bindings one traversal connects is + // refused whichever binding is declared first, with no reorder fix to + // follow: the binding declared first starts the traversal and the other + // is reached, so a reorder hint would only send the caller back and forth. + for (name, first, second) in [("d_first", "$d", "$t"), ("t_first", "$t", "$d")] { + let source = format!( + "query {name}() {{\n match {{ {first}: Doc {second}: Doc $d knows $t }}\n return {{ $d.slug, $t.slug }}\n order {{ rrf(bm25($d.text, \"needle\"), bm25($t.text, \"needle\")) }}\n limit 1\n}}\n" + ); + let refused = db + .query(ReadTarget::branch("main"), &source, name, &ParamMap::new()) + .await + .expect_err("both declaration orders refuse the fused shape"); + let diagnostic = refused.diagnostic().unwrap_or_else(|| { + panic!("{name}: a plan refusal carries its diagnostic: {refused:?}") + }); + assert_eq!(diagnostic.code.as_str(), "P005", "{name}: {refused}"); + assert_eq!( + diagnostic.fix, None, + "{name}: no declaration order serves both arms" + ); + assert_eq!( + diagnostic + .stage + .as_ref() + .and_then(|stage| stage.expression.as_deref()), + Some(r#"rrf(bm25($d.text, "needle"), bm25($t.text, "needle"))"#), + "{name}" + ); + } + // A refusal raised while the query is resolved, before lowering, keeps // its diagnostic too: the statistics pass that resolves the query first // must not turn it into a planner defect. An edge wildcard refuses CSR diff --git a/docs/user/queries/diagnostics.md b/docs/user/queries/diagnostics.md index f14101682..b26495797 100644 --- a/docs/user/queries/diagnostics.md +++ b/docs/user/queries/diagnostics.md @@ -128,3 +128,4 @@ an internal error. | `P002` | An edge alternation or wildcard traversal has no finite traversal work limit. The fix sets `traversal_work_limit` before the query. | | `P003` | The traversal work limit is outside `1..=9223372036854775807`, or a plan records it twice. | | `P004` | An edge alternation or wildcard traversal was asked to run in CSR traversal mode; it runs in `auto` or `indexed` mode. | +| `P005` | An `rrf()` ranks two bindings that one traversal connects. One of them always starts the traversal and the other is reached, and engine v2 ranks only the binding a traversal starts from, so no declaration order serves both arms; the refusal offers no fix. | From 9f37eca717a122882b842630583f524594a65f9d Mon Sep 17 00:00:00 2001 From: Ragnor Comerford Date: Mon, 5 Oct 2026 15:20:36 +0200 Subject: [PATCH 17/29] planner: refuse an unwritten row cut; release replaced derivation nodes Two review findings on plan acceptance. The row-cut check compared a written `limit` with the root's cut but let any cut through when the query wrote none: a saved count plan with an added Limit(0) replayed as no rows. A read plan now carries exactly the query's cut, one at the root when it writes `limit` and none anywhere when it does not, a correlated block's tree included. The derivation arena stored each rule's successors and kept the nodes they replaced, so absorbing N conjuncts one at a time retained O(N^2) predicate data while every counter passed; a 526 KB query held 68 MB of literals. A replaced node is unreachable already (a rule naming it is refused), so it is now released before its successor is stored, and the arena keeps only each role's current node. RFC 0047's claim that the byte and node limits bound evidence memory is replaced by that linear bound. Regressions: validate/tests.rs::an_unwritten_row_cut_fails_the_row_cut, validate/tests.rs::the_derivation_arena_keeps_only_current_nodes, and engine_v2_plan_replay.rs::an_unwritten_row_cut_refuses_the_replay_of_a_count. --- .../src/validate/invariants.rs | 19 ++++++ .../omnigraph-planner/src/validate/subset.rs | 16 ++++- .../omnigraph-planner/src/validate/tests.rs | 62 +++++++++++++++++++ .../omnigraph/tests/engine_v2_plan_replay.rs | 38 ++++++++++++ docs/dev/execution.md | 4 +- docs/rfcs/0047-search-plan-truth.md | 16 ++++- 6 files changed, 151 insertions(+), 4 deletions(-) diff --git a/crates/omnigraph-planner/src/validate/invariants.rs b/crates/omnigraph-planner/src/validate/invariants.rs index 41c436b02..66a2495a9 100644 --- a/crates/omnigraph-planner/src/validate/invariants.rs +++ b/crates/omnigraph-planner/src/validate/invariants.rs @@ -601,6 +601,25 @@ impl Requirements { matcher: &Matcher<'_>, budget: &mut Budget, ) -> Result<(), ValidationError> { + // A row cut is the query's `limit` and nothing else: one, at the + // root (checked next), when the query writes it, and none anywhere + // when it does not, a correlated block's tree included. + budget.visit(u64::try_from(plan.live().count()).unwrap_or(u64::MAX))?; + let cuts = plan + .live() + .filter(|(_, node)| { + matches!(node, PhysicalNode::Limit { .. } | PhysicalNode::Page { .. }) + }) + .count(); + let allowed = usize::from(self.limit.is_some()); + if cuts != allowed { + return Err(ValidationError::violated( + "row cut", + format!( + "the plan cuts its rows {cuts} times; the query's `limit` allows {allowed}" + ), + )); + } let mut id = plan.root(); if let Some(limit) = self.limit { let rows = usize::try_from(limit).unwrap_or(usize::MAX); diff --git a/crates/omnigraph-planner/src/validate/subset.rs b/crates/omnigraph-planner/src/validate/subset.rs index 4317bfe8f..b3bf55a29 100644 --- a/crates/omnigraph-planner/src/validate/subset.rs +++ b/crates/omnigraph-planner/src/validate/subset.rs @@ -510,7 +510,10 @@ impl Tracer { } /// The checker's arena: every node the derivation reached, `None` where a -/// rule removed one. +/// rule removed or replaced one. Only the current node of each role is kept, +/// so the arena's expressions partition the query's own and its memory is +/// linear in the query; a replaced node is released before its successor is +/// stored, and a rule naming it is refused like any other stale reference. struct Arena { nodes: Vec>, current: std::collections::HashMap, @@ -546,6 +549,12 @@ fn pinned(plan: &PhysicalPlan, type_key: &str) -> Option> { .map(|pin| pin.as_ref().map(|pin| pin.version)) } +#[cfg(test)] +thread_local! { + /// The nodes the last checked derivation's arena still held at its end. + pub(super) static RETAINED_NODES: std::cell::Cell = const { std::cell::Cell::new(0) }; +} + /// Check `derivation` for `member` against the candidate `plan`. pub(crate) fn check( derivation: &Derivation, @@ -582,6 +591,8 @@ pub(crate) fn check( node.role() ))); } + #[cfg(test)] + RETAINED_NODES.with(|retained| retained.set(arena.nodes.iter().flatten().count())); let candidate = chain_of(plan, input)?; if reached.len() != candidate.len() || reached @@ -1046,6 +1057,9 @@ fn apply( }; for (role, successor) in successors { budget.node()?; + if let Some(replaced) = arena.current.get(&role).copied() { + arena.nodes[replaced] = None; + } let index = arena.nodes.len(); match successor { Some(node) => { diff --git a/crates/omnigraph-planner/src/validate/tests.rs b/crates/omnigraph-planner/src/validate/tests.rs index b22be112f..3244adc92 100644 --- a/crates/omnigraph-planner/src/validate/tests.rs +++ b/crates/omnigraph-planner/src/validate/tests.rs @@ -387,6 +387,43 @@ fn another_limit_fails_the_row_cut() { assert_eq!(check, "row cut"); } +/// A row cut the query does not write is refused wherever it sits: over the +/// root of a query without `limit`, and below the root of one with it. +#[test] +fn an_unwritten_row_cut_fails_the_row_cut() { + let cut_over = |plan: &mut PhysicalPlan, input: usize| { + let properties = plan.properties(input).cloned(); + let cut = plan.add(PhysicalNode::Limit { input, rows: 0 }); + if let Some(properties) = properties { + plan.set_properties(cut, properties); + } + cut + }; + + let unlimited = Fixture::new( + "query q() { match { $d: Doc $d.title contains \"graph\" } return { $d.slug } }", + &[], + ); + let mut plan = unlimited.plan(); + let root = plan.root(); + let cut = cut_over(&mut plan, root); + plan.set_root(cut); + let (check, detail) = unlimited.refused(plan); + assert_eq!(check, "row cut", "{detail}"); + + let limited = filtered(); + let mut plan = limited.plan(); + let Some(PhysicalNode::Limit { input, .. }) = plan.node(plan.root()).cloned() else { + panic!("the limited plan's root is its cut"); + }; + let below = cut_over(&mut plan, input); + if let Some(PhysicalNode::Limit { input, .. }) = plan.node_mut(plan.root()) { + *input = below; + } + let (check, detail) = limited.refused(plan); + assert_eq!(check, "row cut", "{detail}"); +} + /// An order key constant for every row (a parameter, `now()`) orders /// nothing; when `return` projects it under an alias the planner binds the key /// to that alias, and the order check treats the alias as the constant it @@ -765,6 +802,31 @@ fn derivation_cost_grows_with_the_query() { } } +/// The checker keeps only each role's current node: absorbing 64 conjuncts +/// one at a time replaces the filter and the scan 64 times, and every +/// replaced node is released, so the arena ends holding no more than the +/// chain's six roles rather than a copy of the predicate per step. +#[test] +fn the_derivation_arena_keeps_only_current_nodes() { + let filter: Vec = (0..64) + .map(|index| format!("$d.title != \"{index}\"")) + .collect(); + let query = format!( + "query q() {{ match {{ $d: Doc {} }} return {{ $d.slug }} order {{ $d.year }} limit 10 }}", + filter.join(" ") + ); + let fixture = Fixture::new(&query, &[]); + let derivation = fixture.derivation().expect("an exact-subset member"); + assert!( + derivation.steps.len() >= 64, + "every conjunct is absorbed by its own step: {}", + derivation.steps.len() + ); + fixture.accept(fixture.plan()).unwrap(); + let retained = super::subset::RETAINED_NODES.with(std::cell::Cell::get); + assert!(retained <= 6, "the arena retained {retained} nodes"); +} + /// A rule may name only a node the derivation has reached and not yet /// replaced: a forward reference and a superseded node are both refused. #[test] diff --git a/crates/omnigraph/tests/engine_v2_plan_replay.rs b/crates/omnigraph/tests/engine_v2_plan_replay.rs index b805bcfa7..545b61294 100644 --- a/crates/omnigraph/tests/engine_v2_plan_replay.rs +++ b/crates/omnigraph/tests/engine_v2_plan_replay.rs @@ -1268,6 +1268,44 @@ query instants() { ); } +/// A saved plan carrying a row cut the query does not write is invalid +/// evidence: a `Limit(0)` over the unfiltered count would replay as no rows +/// where the query returns its one count row. +#[tokio::test] +async fn an_unwritten_row_cut_refuses_the_replay_of_a_count() { + let dir = tempfile::tempdir().unwrap(); + let db = people(&dir).await; + let mut run = db + .query_inspected( + ReadTarget::branch("main"), + PEOPLE_QUERIES, + "count_people", + &ParamMap::new(), + ) + .await + .unwrap(); + let plan = &mut run.plan.plan; + let root = plan.root(); + let properties = plan.properties(root).cloned(); + let cut = plan.add(PhysicalNode::Limit { + input: root, + rows: 0, + }); + if let Some(properties) = properties { + plan.set_properties(cut, properties); + } + plan.set_root(cut); + let refused = db + .replay_bound_plan( + ReadTarget::branch("main"), + &run.replay_envelope(PEOPLE_QUERIES, "count_people"), + ) + .await + .err() + .expect("the added cut is refused"); + assert!(refused.to_string().contains("row cut"), "{refused}"); +} + #[tokio::test] async fn an_insert_after_planning_refuses_the_replay_of_a_count() { let dir = tempfile::tempdir().unwrap(); diff --git a/docs/dev/execution.md b/docs/dev/execution.md index d0634f316..ba1272176 100644 --- a/docs/dev/execution.md +++ b/docs/dev/execution.md @@ -650,7 +650,9 @@ rules it applies (`AbsorbScanFilter`, `PruneScanColumns`, `Lower`, `RankBm25Scan`) over arena references; the validator checks that the IR is the declaration's lowering, rebuilds the canonical chain `Limit(Sort(Project([Search](Filter(Scan)))))`, re-applies every rule -against its precondition and requires the result to equal the plan. A member +against its precondition and requires the result to equal the plan. The +checker's arena keeps only each role's current node, releasing a replaced one +before storing its successor, so its memory is linear in the query. A member is accepted as `exact_subset`; every other plan as `invariants_only`. A fresh plan that fails a check is a planner defect (an internal error). A diff --git a/docs/rfcs/0047-search-plan-truth.md b/docs/rfcs/0047-search-plan-truth.md index d1d7d2450..cebdba3df 100644 --- a/docs/rfcs/0047-search-plan-truth.md +++ b/docs/rfcs/0047-search-plan-truth.md @@ -591,8 +591,12 @@ charged per expression and plan node checked and per conjunct an absorption copies. A member's derivation grows linearly with its conjuncts (the `instrument:` test `derivation_cost_grows_with_the_query` prints bytes, steps, nodes, visits and time; 256 conjuncts take about 58 KB, 261 steps and 11 ms). -No planning-time memory pool exists, so evidence memory is bounded by the byte -and node limits instead of being charged to the query pool. The planner +No planning-time memory pool exists. The checker's arena keeps only each +role's current node and releases a replaced node before storing its +successor, so its retained expressions partition the query's own: evidence +memory is linear in the compiled query, whose size the request limits bound, +while the node limit bounds the arena's slots and the visit charge bounds the +copying work. The planner always records a member's derivation; no optional rewrite is skipped for budget, and exhaustion on a fresh plan is a resource outcome (`ResourceLimitExceeded`). @@ -1026,3 +1030,11 @@ None. blocks; evidence memory is bounded by byte and node limits because no planning pool exists; the fusion and aggregate order checks land with step 5. +- 2026-10-05: two review findings on step 2a, both fixed with regressions. + The row-cut check refused a wrong `limit` but accepted a cut the query + does not write; a plan now carries exactly the query's cut, one at the + root or none anywhere. The derivation arena kept every replaced node, so + its retained predicate data grew quadratically with the conjuncts while + every counter passed; it now releases a replaced node, and the statement + that the byte and node limits bound evidence memory is replaced by the + linear bound above. From 5569283223dc3132afd6c97abcbe700be34fea4c Mon Sep 17 00:00:00 2001 From: Ragnor Comerford Date: Tue, 6 Oct 2026 10:32:02 +0200 Subject: [PATCH 18/29] release notes: open v0.12.1 now that v0.12.0 is published v0.12.0 is tagged and its snapshot generated, but release.json still named it, so the documentation check compared every note added since with that published snapshot and failed. The release procedure's post-publication step moves the configuration to the next version on the published base; this is the same change #871 and #878 carry. --- changelog.d/release.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/changelog.d/release.json b/changelog.d/release.json index 788c18fb1..7079c1508 100644 --- a/changelog.d/release.json +++ b/changelog.d/release.json @@ -1,5 +1,5 @@ { - "version": "v0.12.0", - "base": "refs/tags/v0.11.0", + "version": "v0.12.1", + "base": "refs/tags/v0.12.0", "legacy": null } From 3426d7cbb8b0eebf86c14ff4b81eb4f7534801d8 Mon Sep 17 00:00:00 2001 From: Ragnor Comerford Date: Tue, 6 Oct 2026 13:45:17 +0200 Subject: [PATCH 19/29] planner: every node declares its row cut, and acceptance justifies each The row-cut check counted `Limit` and `Page` nodes, so a capped `Sort` below the final sort cut rows that acceptance never saw: a saved ordered plan with such a sort replayed as no rows. `PhysicalNode::row_cut` now names each node's count cut in one match over every variant (a `Limit`'s or `Page`'s rows, a `Sort`'s `fetch`, a `RankFuse`'s limit, a ranked scan's candidate cap), so a node kind that gains a cut cannot compile without declaring it. `check_row_cuts` walks every live node and accepts a declared cut only where the query requires it: the root `Limit` and the final sort's `fetch` at the query's `limit`, the fusion at that `limit`, and a ranked scan's cap, which the search check already matches to its declared policy. Every other cut is refused, the final sort keeping its top-k. Regressions: validate/tests.rs::an_unwritten_row_cut_fails_the_row_cut (capped sorts below the final sort of an ordered and of a limited query) and engine_v2_plan_replay.rs::an_unwritten_row_cut_refuses_the_replay (the count's root cut and `liked`'s capped sort). --- crates/omnigraph-planner/src/physical.rs | 27 +++++ .../src/validate/invariants.rs | 70 ++++++++---- .../omnigraph-planner/src/validate/tests.rs | 58 +++++++++- .../omnigraph/tests/engine_v2_plan_replay.rs | 103 ++++++++++++------ docs/dev/execution.md | 9 +- docs/rfcs/0047-search-plan-truth.md | 7 ++ 6 files changed, 216 insertions(+), 58 deletions(-) diff --git a/crates/omnigraph-planner/src/physical.rs b/crates/omnigraph-planner/src/physical.rs index 063fdcd7d..53233539b 100644 --- a/crates/omnigraph-planner/src/physical.rs +++ b/crates/omnigraph-planner/src/physical.rs @@ -682,6 +682,33 @@ pub enum PhysicalNode { } impl PhysicalNode { + /// The most rows this node keeps of its input whatever any predicate + /// holds: its count cut. Every variant is listed, so a node kind that + /// gains a cut cannot compile without declaring it here, and plan + /// acceptance justifies every declared cut against the query. + pub fn row_cut(&self) -> Option { + match self { + Self::Limit { rows, .. } | Self::Page { rows, .. } => Some(*rows), + Self::Sort { fetch, .. } => *fetch, + Self::RankFuse { limit, .. } => *limit, + Self::Scan { ranked, .. } => ranked.as_ref().and_then(|ranked| ranked.fetch), + Self::MetadataCount { .. } + | Self::SortMergeJoin { .. } + | Self::HashJoin { .. } + | Self::HydrateByAddress { .. } + | Self::RowCompare { .. } + | Self::ClassifyThreeWay { .. } + | Self::CrossJoin { .. } + | Self::ContainsJoin { .. } + | Self::Filter { .. } + | Self::Expand { .. } + | Self::AntiJoin { .. } + | Self::OuterReference { .. } + | Self::Projection { .. } + | Self::Aggregate { .. } => None, + } + } + pub fn name(&self) -> &'static str { match self { Self::Scan { .. } => "Scan", diff --git a/crates/omnigraph-planner/src/validate/invariants.rs b/crates/omnigraph-planner/src/validate/invariants.rs index 66a2495a9..1caebf975 100644 --- a/crates/omnigraph-planner/src/validate/invariants.rs +++ b/crates/omnigraph-planner/src/validate/invariants.rs @@ -138,6 +138,7 @@ impl Requirements { } let top = pipeline(plan, plan.root(), budget)?; self.check_search(plan, &matcher, budget)?; + self.check_row_cuts(plan, budget)?; self.check_returns(plan, &top, &matcher, budget)?; self.check_order_and_cut(plan, &top, &matcher, budget)?; check_policies(plan, budget)?; @@ -426,6 +427,56 @@ impl Requirements { Ok(()) } + /// Every count cut the plan declares (`PhysicalNode::row_cut`) is one the + /// query requires, where it requires it: the root `Limit` and the final + /// sort's `fetch` carry the query's `limit`, a fusion keeps that `limit`, + /// and a ranked scan's candidate cap is the one its declared policy sets + /// (checked with its access). Any other cut is refused, a capped sort + /// below the final one and a cut in a correlated block's tree included. + fn check_row_cuts( + &self, + plan: &PhysicalPlan, + budget: &mut Budget, + ) -> Result<(), ValidationError> { + budget.visit(u64::try_from(plan.live().count()).unwrap_or(u64::MAX))?; + let limit = self + .limit + .map(|limit| usize::try_from(limit).unwrap_or(usize::MAX)); + let root = plan.root(); + let mut id = match plan.node(root) { + Some(PhysicalNode::Limit { input, .. }) => *input, + _ => root, + }; + while let Some(PhysicalNode::Projection { input, .. }) = plan.node(id) { + id = *input; + } + let final_sort = matches!(plan.node(id), Some(PhysicalNode::Sort { .. })).then_some(id); + for (id, node) in plan.live() { + let Some(cut) = node.row_cut() else { + continue; + }; + let required = match node { + PhysicalNode::Limit { .. } => id == root && limit == Some(cut), + PhysicalNode::Sort { .. } => Some(id) == final_sort && limit == Some(cut), + PhysicalNode::RankFuse { .. } => limit == Some(cut), + PhysicalNode::Scan { + ranked: Some(_), .. + } => true, + _ => false, + }; + if !required { + return Err(ValidationError::violated( + "row cut", + format!( + "node {id} ({}) keeps at most {cut} rows; the query's `limit` is {limit:?} and requires no such cut there", + node.name() + ), + )); + } + } + Ok(()) + } + /// The candidate cap, probe cap and overfetch ladder a ranked scan /// declares, from the cut its scope requires: an ordering `nearest` /// fetches the `limit` and may widen it by the declared ladder, a @@ -601,25 +652,6 @@ impl Requirements { matcher: &Matcher<'_>, budget: &mut Budget, ) -> Result<(), ValidationError> { - // A row cut is the query's `limit` and nothing else: one, at the - // root (checked next), when the query writes it, and none anywhere - // when it does not, a correlated block's tree included. - budget.visit(u64::try_from(plan.live().count()).unwrap_or(u64::MAX))?; - let cuts = plan - .live() - .filter(|(_, node)| { - matches!(node, PhysicalNode::Limit { .. } | PhysicalNode::Page { .. }) - }) - .count(); - let allowed = usize::from(self.limit.is_some()); - if cuts != allowed { - return Err(ValidationError::violated( - "row cut", - format!( - "the plan cuts its rows {cuts} times; the query's `limit` allows {allowed}" - ), - )); - } let mut id = plan.root(); if let Some(limit) = self.limit { let rows = usize::try_from(limit).unwrap_or(usize::MAX); diff --git a/crates/omnigraph-planner/src/validate/tests.rs b/crates/omnigraph-planner/src/validate/tests.rs index 3244adc92..320e1a5f9 100644 --- a/crates/omnigraph-planner/src/validate/tests.rs +++ b/crates/omnigraph-planner/src/validate/tests.rs @@ -387,18 +387,50 @@ fn another_limit_fails_the_row_cut() { assert_eq!(check, "row cut"); } -/// A row cut the query does not write is refused wherever it sits: over the -/// root of a query without `limit`, and below the root of one with it. +/// Every count cut a plan declares must be one the query requires, where it +/// requires it. Refused: a `Limit` over the root of a query without `limit`, +/// a second `Limit` below a written one, and a capped `Sort` below the final +/// sort, with and without a written `limit`; the final sort's own top-k under +/// a written `limit` stays accepted. #[test] fn an_unwritten_row_cut_fails_the_row_cut() { - let cut_over = |plan: &mut PhysicalPlan, input: usize| { + let cut_over = |plan: &mut PhysicalPlan, node: PhysicalNode, input: usize| { let properties = plan.properties(input).cloned(); - let cut = plan.add(PhysicalNode::Limit { input, rows: 0 }); + let cut = plan.add(node); if let Some(properties) = properties { plan.set_properties(cut, properties); } cut }; + let limit_over = |plan: &mut PhysicalPlan, input: usize| { + cut_over(plan, PhysicalNode::Limit { input, rows: 0 }, input) + }; + // A capped sort spliced below the plan's final sort, ordered as it is. + let capped_below_final = |plan: &mut PhysicalPlan| { + let final_sort = node_ids(plan, |node| matches!(node, PhysicalNode::Sort { .. }))[0]; + let Some(PhysicalNode::Sort { + input, + order_by, + tiebreak, + .. + }) = plan.node(final_sort).cloned() + else { + unreachable!("the id names a sort"); + }; + let capped = cut_over( + plan, + PhysicalNode::Sort { + input, + order_by, + fetch: Some(0), + tiebreak, + }, + input, + ); + if let Some(PhysicalNode::Sort { input, .. }) = plan.node_mut(final_sort) { + *input = capped; + } + }; let unlimited = Fixture::new( "query q() { match { $d: Doc $d.title contains \"graph\" } return { $d.slug } }", @@ -406,7 +438,7 @@ fn an_unwritten_row_cut_fails_the_row_cut() { ); let mut plan = unlimited.plan(); let root = plan.root(); - let cut = cut_over(&mut plan, root); + let cut = limit_over(&mut plan, root); plan.set_root(cut); let (check, detail) = unlimited.refused(plan); assert_eq!(check, "row cut", "{detail}"); @@ -416,12 +448,26 @@ fn an_unwritten_row_cut_fails_the_row_cut() { let Some(PhysicalNode::Limit { input, .. }) = plan.node(plan.root()).cloned() else { panic!("the limited plan's root is its cut"); }; - let below = cut_over(&mut plan, input); + let below = limit_over(&mut plan, input); if let Some(PhysicalNode::Limit { input, .. }) = plan.node_mut(plan.root()) { *input = below; } let (check, detail) = limited.refused(plan); assert_eq!(check, "row cut", "{detail}"); + + let ordered = Fixture::new( + "query q() { match { $d: Doc } return { $d.slug } order { $d.year } }", + &[], + ); + for fixture in [&ordered, &limited] { + fixture + .accept(fixture.plan()) + .expect("the planned final sort is the cut the query requires"); + let mut plan = fixture.plan(); + capped_below_final(&mut plan); + let (check, detail) = fixture.refused(plan); + assert_eq!(check, "row cut", "{detail}"); + } } /// An order key constant for every row (a parameter, `now()`) orders diff --git a/crates/omnigraph/tests/engine_v2_plan_replay.rs b/crates/omnigraph/tests/engine_v2_plan_replay.rs index 545b61294..cc73db8d8 100644 --- a/crates/omnigraph/tests/engine_v2_plan_replay.rs +++ b/crates/omnigraph/tests/engine_v2_plan_replay.rs @@ -1268,42 +1268,81 @@ query instants() { ); } -/// A saved plan carrying a row cut the query does not write is invalid -/// evidence: a `Limit(0)` over the unfiltered count would replay as no rows -/// where the query returns its one count row. +/// A saved plan carrying a count cut the query does not require is invalid +/// evidence, wherever the cut sits: a `Limit(0)` over the unfiltered count +/// would replay as no rows where the query returns its one count row, and a +/// capped sort below `liked`'s final sort as no rows where it returns three. #[tokio::test] -async fn an_unwritten_row_cut_refuses_the_replay_of_a_count() { +async fn an_unwritten_row_cut_refuses_the_replay() { let dir = tempfile::tempdir().unwrap(); let db = people(&dir).await; - let mut run = db - .query_inspected( - ReadTarget::branch("main"), - PEOPLE_QUERIES, - "count_people", - &ParamMap::new(), - ) - .await - .unwrap(); - let plan = &mut run.plan.plan; - let root = plan.root(); - let properties = plan.properties(root).cloned(); - let cut = plan.add(PhysicalNode::Limit { - input: root, - rows: 0, - }); - if let Some(properties) = properties { - plan.set_properties(cut, properties); + for query in ["count_people", "liked"] { + let mut run = db + .query_inspected( + ReadTarget::branch("main"), + PEOPLE_QUERIES, + query, + &ParamMap::new(), + ) + .await + .unwrap(); + assert!(!rows_of(&run.result).is_empty(), "{query} returns rows"); + let plan = &mut run.plan.plan; + let final_sort = plan + .live() + .find(|(_, node)| matches!(node, PhysicalNode::Sort { .. })) + .map(|(id, node)| (id, node.clone())); + assert_eq!(final_sort.is_some(), query == "liked", "{query}"); + match final_sort { + Some(( + id, + PhysicalNode::Sort { + input, + order_by, + tiebreak, + .. + }, + )) => { + let properties = plan.properties(input).cloned(); + let capped = plan.add(PhysicalNode::Sort { + input, + order_by, + fetch: Some(0), + tiebreak, + }); + if let Some(properties) = properties { + plan.set_properties(capped, properties); + } + if let Some(PhysicalNode::Sort { input, .. }) = plan.node_mut(id) { + *input = capped; + } + } + _ => { + let root = plan.root(); + let properties = plan.properties(root).cloned(); + let cut = plan.add(PhysicalNode::Limit { + input: root, + rows: 0, + }); + if let Some(properties) = properties { + plan.set_properties(cut, properties); + } + plan.set_root(cut); + } + } + let refused = db + .replay_bound_plan( + ReadTarget::branch("main"), + &run.replay_envelope(PEOPLE_QUERIES, query), + ) + .await + .err() + .unwrap_or_else(|| panic!("{query}: the added cut is refused")); + assert!( + refused.to_string().contains("row cut"), + "{query}: {refused}" + ); } - plan.set_root(cut); - let refused = db - .replay_bound_plan( - ReadTarget::branch("main"), - &run.replay_envelope(PEOPLE_QUERIES, "count_people"), - ) - .await - .err() - .expect("the added cut is refused"); - assert!(refused.to_string().contains("row cut"), "{refused}"); } #[tokio::test] diff --git a/docs/dev/execution.md b/docs/dev/execution.md index ba1272176..a27bf77e4 100644 --- a/docs/dev/execution.md +++ b/docs/dev/execution.md @@ -632,7 +632,14 @@ catalog, not from the IR: binding identity, search identity and its declared approximation, every eligibility conjunct (checked per `rrf` arm, since each arm copies the match), correlated blocks, the projection and the origin of every projected score, order and cut, and the policies the plan declares -for its search. A matcher relates each written expression to the IR through +for its search. Every node declares its count cut (`PhysicalNode::row_cut`, +a match over every variant: a `Limit`'s or `Page`'s rows, a `Sort`'s +`fetch`, a `RankFuse`'s limit, a ranked scan's candidate cap), and the +validator justifies each one by its position: the root `Limit` and the final +`Sort` (below the root `Limit` and its projections) at the query's `limit`, a +fusion at that `limit`, and a ranked scan's cap as its declared policy sets +it. Any other cut, a capped sort below the final one or a cut in a +correlated block's tree, is refused as `row cut`. A matcher relates each written expression to the IR through the compiler's lowering patterns (property leaves on physical columns, `in` as a swapped `contains`, a bare search call as `= true`, a folded constant as the value the engine's evaluator gives it). The order check recomputes the diff --git a/docs/rfcs/0047-search-plan-truth.md b/docs/rfcs/0047-search-plan-truth.md index cebdba3df..b94f463a6 100644 --- a/docs/rfcs/0047-search-plan-truth.md +++ b/docs/rfcs/0047-search-plan-truth.md @@ -1038,3 +1038,10 @@ None. every counter passed; it now releases a replaced node, and the statement that the byte and node limits bound evidence memory is replaced by the linear bound above. +- 2026-10-06: a further review finding: the row-cut check counted `Limit` + and `Page` nodes only, so a capped `Sort` below the final sort cut rows + unseen. Every node kind now declares its count cut through one exhaustive + match, so a kind that gains a cut cannot compile without declaring it, + and acceptance justifies each declared cut by its position (root `Limit` + and final sort `fetch` at the query's `limit`, the fusion's limit, a + ranked scan's policy cap) and refuses every other. From d76469b511d821648a690da575ef5357ebd5d67d Mon Sep 17 00:00:00 2001 From: Ragnor Comerford Date: Fri, 9 Oct 2026 11:23:59 -0700 Subject: [PATCH 20/29] planner: say what the exhaustive row-cut match does and does not force The match forces a classification for a new node variant; a field that gives an existing variant a cut compiles unchanged, so its arm still needs a hand audit. The doc comment and RFC 0047's decision log claimed more. --- crates/omnigraph-planner/src/physical.rs | 5 +++-- docs/rfcs/0047-search-plan-truth.md | 3 ++- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/crates/omnigraph-planner/src/physical.rs b/crates/omnigraph-planner/src/physical.rs index 53233539b..a722e57bb 100644 --- a/crates/omnigraph-planner/src/physical.rs +++ b/crates/omnigraph-planner/src/physical.rs @@ -683,8 +683,9 @@ pub enum PhysicalNode { impl PhysicalNode { /// The most rows this node keeps of its input whatever any predicate - /// holds: its count cut. Every variant is listed, so a node kind that - /// gains a cut cannot compile without declaring it here, and plan + /// holds: its count cut. Every variant is listed, so a new node kind + /// cannot compile without a classification here; a field that gives an + /// existing kind a cut still needs its arm changed by hand. Plan /// acceptance justifies every declared cut against the query. pub fn row_cut(&self) -> Option { match self { diff --git a/docs/rfcs/0047-search-plan-truth.md b/docs/rfcs/0047-search-plan-truth.md index b94f463a6..fb25291fe 100644 --- a/docs/rfcs/0047-search-plan-truth.md +++ b/docs/rfcs/0047-search-plan-truth.md @@ -1041,7 +1041,8 @@ None. - 2026-10-06: a further review finding: the row-cut check counted `Limit` and `Page` nodes only, so a capped `Sort` below the final sort cut rows unseen. Every node kind now declares its count cut through one exhaustive - match, so a kind that gains a cut cannot compile without declaring it, + match, so a new kind cannot compile without a classification (a field + that gives an existing kind a cut still needs its arm audited by hand), and acceptance justifies each declared cut by its position (root `Limit` and final sort `fetch` at the query's `limit`, the fusion's limit, a ranked scan's policy cap) and refuses every other. From 29a21deb21b7b0be448d1dfa89893010933ca796 Mon Sep 17 00:00:00 2001 From: Ragnor Comerford Date: Fri, 9 Oct 2026 19:49:02 -0700 Subject: [PATCH 21/29] test: import ManifestErrorKind beside main's refusal assertions after the merge --- crates/omnigraph/tests/engine_v2_plan_replay.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/omnigraph/tests/engine_v2_plan_replay.rs b/crates/omnigraph/tests/engine_v2_plan_replay.rs index f1831ed38..758a37a3c 100644 --- a/crates/omnigraph/tests/engine_v2_plan_replay.rs +++ b/crates/omnigraph/tests/engine_v2_plan_replay.rs @@ -7,7 +7,7 @@ mod helpers; use omnigraph::db::{Omnigraph, ReadTarget}; -use omnigraph::error::OmniError; +use omnigraph::error::{ManifestErrorKind, OmniError}; use omnigraph::loader::LoadMode; use omnigraph_compiler::ir::ParamMap; use omnigraph_compiler::query::ast::Literal; From bd84602a2d5274d7b9eb0c1f53fabbe07f386670 Mon Sep 17 00:00:00 2001 From: Ragnor Comerford Date: Sat, 10 Oct 2026 21:26:51 -0700 Subject: [PATCH 22/29] planner: accept plans against the typed lowering of the query Main typed the IR (explicit casts, typed literals and parameters), which broke acceptance's AST-to-IR matching and its constant evaluator. Plan and requirement now compare as typed IR: the compiler's lowering of the declaration (filter constants folded) states every conjunct, traversal, block predicate, return item and order key in the form a plan carries it, and the declaration anchors how many of each the query writes and which rank call a projected score names. - Phase 1 of acceptance runs the plan's own well-formedness checks (aggregate specs, output schemas, fusion row keys). - Eligibility is two-sided: every required conjunct is placed, and every placed conjunct is required, a constant true, or the canonical identity of a written single String key equality, re-derived through a KeyIds hook (scan_access's key narrowing). - Traversal identity: endpoints, edge selection, endpoint types, hop bounds and edge binding of every traversal. - Order: the final comparator is recomputed as the planner binds it; a fused search leads with _rrf descending, a search-led aggregate appends its group keys, and a fusion no longer declares an order or a row cut. - Late hydration (HydrateColumns) is checked against the written return items and gets a checked exact-subset rule, HydrateReturnColumns. - Exact-subset membership is decided on the typed IR, admitting a constant cast of a literal or parameter. - RULES_VERSION and SEMANTICS_VERSION move to 2. --- crates/omnigraph-planner/src/gate.rs | 7 +- crates/omnigraph-planner/src/lib.rs | 14 +- crates/omnigraph-planner/src/optimizer.rs | 16 +- crates/omnigraph-planner/src/physical.rs | 8 +- .../src/validate/invariants.rs | 829 ++++++++++-------- crates/omnigraph-planner/src/validate/mod.rs | 35 +- .../omnigraph-planner/src/validate/replay.rs | 4 +- .../src/validate/requirements.rs | 626 ++++--------- .../omnigraph-planner/src/validate/subset.rs | 511 ++++++----- .../omnigraph-planner/src/validate/tests.rs | 65 +- 10 files changed, 1043 insertions(+), 1072 deletions(-) diff --git a/crates/omnigraph-planner/src/gate.rs b/crates/omnigraph-planner/src/gate.rs index 739e87e62..bdd84722c 100644 --- a/crates/omnigraph-planner/src/gate.rs +++ b/crates/omnigraph-planner/src/gate.rs @@ -461,12 +461,11 @@ pub async fn accept_query_explained( } /// One traced planning run of a read query: the rewritten logical plan, the -/// optimized physical plan with its assumptions, the passes that fired, and -/// the derivation the optimizer recorded for an exact-fragment chain. +/// optimized physical plan with its assumptions and the passes that fired, +/// and the derivation the optimizer recorded for an exact-fragment chain. pub(crate) struct Traced { pub logical: LogicalPlan, pub optimized: Optimized, - pub fired: Vec<&'static str>, pub derivation: Option, } @@ -490,7 +489,6 @@ pub(crate) fn plan_traced( Ok(Traced { logical, optimized, - fired, derivation: tracer.finish(), }) } @@ -505,7 +503,6 @@ async fn plan_and_accept( let Traced { logical, mut optimized, - fired: _, derivation, } = plan_traced(input.ir, source, bounds)?; // Scan access is part of the plan acceptance checks: a probe or a key diff --git a/crates/omnigraph-planner/src/lib.rs b/crates/omnigraph-planner/src/lib.rs index 40f89beec..4e50fd41b 100644 --- a/crates/omnigraph-planner/src/lib.rs +++ b/crates/omnigraph-planner/src/lib.rs @@ -70,10 +70,10 @@ pub use operation::{Operation, PageBudgetSpec, ScopeSpec, Side, TableRef}; pub use optimizer::{Bounds, physical_plan, rewrite}; pub use output::{NodeObjectType, validate_output_schemas}; pub use physical::{ - Assumptions, DatasetPin, Eligibility, EmptyEligible, Estimate, GatePolicy, Hop, HydratedBinding, - HydratedColumn, NearestPolicy, NodeId, OrderKey, OverfetchRung, PhysicalNode, PhysicalPlan, - Prefilter, PrefilterMode, Properties, ROW_ADDRESS_PREFIX, RankArm, RankKind, RankScope, - RankedAccess, ScanInput, StatisticSource, + Assumptions, DatasetPin, Eligibility, EmptyEligible, Estimate, GatePolicy, Hop, + HydratedBinding, HydratedColumn, NearestPolicy, NodeId, OrderKey, OverfetchRung, PhysicalNode, + PhysicalPlan, Prefilter, PrefilterMode, Properties, ROW_ADDRESS_PREFIX, RankArm, RankKind, + RankScope, RankedAccess, ScanInput, StatisticSource, }; pub use registry::{Coverage, Entry, Route, Shape}; pub use route::RouteOverride; @@ -83,9 +83,9 @@ pub use source::{ MemorySource, NodeTypeSpec, PlanSource, SideId, }; pub use validate::{ - AcceptInput, AcceptedBoundPlan, AcceptedPlan, ConstantEvaluator, Evidence, ReplayEnvelope, - ReplayQuery, ReplayRefusal, ValidationError, ValidationLimits, ValidationScope, - ValidationSummary, accept_replay, catalog_digest, decode_replay, + AcceptInput, AcceptedBoundPlan, AcceptedPlan, Evidence, KeyIds, ReplayEnvelope, ReplayQuery, + ReplayRefusal, ValidationError, ValidationLimits, ValidationScope, ValidationSummary, + accept_replay, catalog_digest, decode_replay, }; #[cfg(test)] diff --git a/crates/omnigraph-planner/src/optimizer.rs b/crates/omnigraph-planner/src/optimizer.rs index 6231c97d7..1c76c9822 100644 --- a/crates/omnigraph-planner/src/optimizer.rs +++ b/crates/omnigraph-planner/src/optimizer.rs @@ -603,7 +603,7 @@ fn search_arm_reads(expr: &IRExpr, out: &mut Vec) { } } -fn rank_score(binding: &str, kind: RankKind) -> IRExpr { +pub(crate) fn rank_score(binding: &str, kind: RankKind) -> IRExpr { IRExpr::PropAccess { variable: binding.to_string(), property: kind.score().0.to_string(), @@ -634,7 +634,7 @@ fn fused_score(binding: &str) -> IRExpr { /// A key the type checker bound to a `return` item (T42, RFC /// 2026-09-24-shared-expression-model, "Order key binding") as the `AliasRef` /// of that item's column; a property or alias key as written. -fn bind_order_key(key: &IRExpr, return_exprs: &[IRProjection]) -> IRExpr { +pub(crate) fn bind_order_key(key: &IRExpr, return_exprs: &[IRProjection]) -> IRExpr { if matches!(key, IRExpr::PropAccess { .. } | IRExpr::AliasRef(_, _)) { return key.clone(); } @@ -1168,7 +1168,7 @@ pub(crate) fn physical_plan_with( } = lowering; physical.set_root(root); let late_query = - query && materialize_returns_late(&mut physical, plan, source, &mut decisions)?; + query && materialize_returns_late(&mut physical, plan, source, &mut decisions, tracer)?; if late_materialization || late_query { fired.push(PASS_LATE_MATERIALIZATION); } @@ -1989,6 +1989,7 @@ fn materialize_returns_late( logical: &LogicalPlan, source: &dyn PlanSource, decisions: &mut Vec, + tracer: &mut Tracer, ) -> Result { let root = physical.root(); let Some(&PhysicalNode::Limit { input, rows: limit }) = physical.node(root) else { @@ -2186,6 +2187,15 @@ fn materialize_returns_late( spec.projection = Some(read); } } + if let [hydrated] = bindings.as_slice() { + tracer.record_with( + &[Role::Scan], + &[Role::Scan, Role::Hydrate], + Rule::HydrateReturnColumns { + columns: hydrated.columns.clone(), + }, + ); + } let hydrate = physical.add(PhysicalNode::HydrateColumns { input: root, bindings, diff --git a/crates/omnigraph-planner/src/physical.rs b/crates/omnigraph-planner/src/physical.rs index 8508a8987..0f7e72823 100644 --- a/crates/omnigraph-planner/src/physical.rs +++ b/crates/omnigraph-planner/src/physical.rs @@ -730,12 +730,16 @@ impl PhysicalNode { match self { Self::Limit { rows, .. } | Self::Page { rows, .. } => Some(*rows), Self::Sort { fetch, .. } => *fetch, - Self::RankFuse { limit, .. } => *limit, Self::Scan { ranked, .. } => ranked.as_ref().and_then(|ranked| ranked.fetch), - Self::MetadataCount { .. } + // A fusion emits every arm row; its `limit` sizes only the + // nearest arm's window, which that arm's scan declares as its + // fetch. + Self::RankFuse { .. } + | Self::MetadataCount { .. } | Self::SortMergeJoin { .. } | Self::HashJoin { .. } | Self::HydrateByAddress { .. } + | Self::HydrateColumns { .. } | Self::RowCompare { .. } | Self::ClassifyThreeWay { .. } | Self::CrossJoin { .. } diff --git a/crates/omnigraph-planner/src/validate/invariants.rs b/crates/omnigraph-planner/src/validate/invariants.rs index 1caebf975..e9eed8ef8 100644 --- a/crates/omnigraph-planner/src/validate/invariants.rs +++ b/crates/omnigraph-planner/src/validate/invariants.rs @@ -1,28 +1,30 @@ //! The checks every accepted plan passes (RFC 0047, "Checks have explicit -//! boundaries"): binding identity, search identity and its declared -//! approximation, retained eligibility predicates, correlated blocks, the -//! projection and the origin of every projected score, the required order -//! and the row cut, and the policy data the plan declares for its search. -//! They establish that the plan keeps what the query requires; they do not -//! prove row-selection equivalence, which only the exact subset's checked -//! derivation does. +//! boundaries"): binding and traversal identity, the eligibility predicates +//! (every required conjunct placed, every placed conjunct required), +//! correlated blocks, search identity and its declared approximation, the +//! projection and the origin of every projected score, late hydration, the +//! required order and the row cut, and the policy data the plan declares for +//! its search. They establish that the plan keeps what the query requires +//! and adds nothing it does not; they do not prove row-selection +//! equivalence, which only the exact subset's checked derivation does. //! -//! The order of a fused (`rrf`) search and of a search-ordered aggregate is -//! not checked here: the plan does not yet implement the total order those -//! shapes require (RFC 0047, "One total order"), and the step that does -//! extends these checks. +//! Requirements and plan are compared as typed IR (see `requirements.rs`). +//! The planner adds one predicate the query does not state: the canonical +//! `@id` of a written single String key equality (`scan_access::narrow_key`), +//! which is admitted only when the validator derives the same id from the +//! written value. + +use std::collections::HashSet; use omnigraph_compiler::ir::{IRExpr, IROrdering, IRProjection}; -use omnigraph_compiler::query::ast::{Expr, Literal}; +use omnigraph_compiler::query::ast::{CompOp, Expr, Literal, RRF_COLUMN}; use omnigraph_compiler::query::typecheck::BoundVariable; +use omnigraph_compiler::types::{ExprType, ScalarType}; use super::budget::Budget; -use super::requirements::{ - Matcher, Position, Requirements, Retrieval, Search, result_column, score_column, -}; +use super::requirements::{Requirements, Retrieval, Search, constant_true}; use super::{AcceptInput, ValidationError}; use crate::logical::{ColumnRef, EDGE_TYPE_MEMBER, IDENTITY_MEMBER}; -use crate::lower::ContainsJoinFields; use crate::optimizer::{RRF_NEAREST_ARM_K, derived_order}; use crate::physical::{ Assumptions, Eligibility, EmptyEligible, NodeId, OverfetchRung, PhysicalNode, PhysicalPlan, @@ -73,34 +75,45 @@ fn constrained_pipelines( } } -/// Every conjunct a pipeline's nodes test, wherever placement put it. -fn conjuncts(plan: &PhysicalPlan, nodes: &[NodeId]) -> Vec { +/// Every conjunct a pipeline's nodes test, wherever placement put it, with +/// the node that tests it. +fn conjuncts(plan: &PhysicalPlan, nodes: &[NodeId]) -> Vec<(NodeId, IRExpr)> { let mut out = Vec::new(); for id in nodes { match plan.node(*id) { Some(PhysicalNode::Scan { spec, .. }) => { if let Some(filter) = &spec.filter { - out.extend(filter.gq_filters()); + out.extend( + filter + .gq_filters() + .into_iter() + .flat_map(IRExpr::into_conjuncts) + .map(|conjunct| (*id, conjunct)), + ); } } Some( PhysicalNode::Filter { filters, .. } | PhysicalNode::CrossJoin { filters, .. }, ) => { - out.extend(filters.iter().cloned()); + out.extend( + filters + .iter() + .cloned() + .flat_map(IRExpr::into_conjuncts) + .map(|conjunct| (*id, conjunct)), + ); } Some(PhysicalNode::ContainsJoin { - haystack, - needle, - residual, - .. + conjunct, residual, .. }) => { - let fields = ContainsJoinFields { - haystack: (&haystack.0, &haystack.1), - needle: (&needle.0, &needle.1), - residual, - }; - out.push(fields.conjunct()); - out.extend(residual.iter().cloned()); + out.push((*id, conjunct.clone())); + out.extend( + residual + .iter() + .cloned() + .flat_map(IRExpr::into_conjuncts) + .map(|conjunct| (*id, conjunct)), + ); } _ => {} } @@ -122,6 +135,16 @@ fn ranked_scans(plan: &PhysicalPlan) -> Vec<(NodeId, &str, &RankedAccess)> { .collect() } +/// The type `scan_access::narrow_key` gives the identity column and its +/// value: a non-null String. +fn narrowed_type() -> ExprType { + ExprType::Value { + scalar: ScalarType::String, + list: false, + nullable: false, + } +} + impl Requirements { pub(crate) fn check( &self, @@ -129,26 +152,27 @@ impl Requirements { input: &AcceptInput<'_>, budget: &mut Budget, ) -> Result<(), ValidationError> { - let matcher = Matcher::new(input, self); let pipelines = constrained_pipelines(plan, budget)?; for nodes in &pipelines { self.check_bindings(plan, nodes)?; - self.check_eligibility(plan, nodes, &matcher, budget)?; - self.check_blocks(plan, nodes, &matcher, budget)?; + self.check_eligibility(plan, nodes, input, budget)?; + self.check_blocks(plan, nodes)?; } let top = pipeline(plan, plan.root(), budget)?; - self.check_search(plan, &matcher, budget)?; + self.check_search(plan)?; self.check_row_cuts(plan, budget)?; - self.check_returns(plan, &top, &matcher, budget)?; - self.check_order_and_cut(plan, &top, &matcher, budget)?; + self.check_returns(plan, &top)?; + self.check_hydration(plan, &top)?; + self.check_order_and_cut(plan, &top, input, budget)?; check_policies(plan, budget)?; Ok(()) } /// Every written binding is read with its type: a node binding by a scan /// (or a pushed-down count) of its node type, an edge binding by the - /// traversal that names it over its edge types; every traversal clause - /// is one traversal. + /// traversal that names it over its edge types; every traversal the query + /// writes runs once with its endpoints, edge selection, endpoint types, + /// hop bounds and edge binding, and no other traversal runs. fn check_bindings(&self, plan: &PhysicalPlan, nodes: &[NodeId]) -> Result<(), ValidationError> { for (name, bound) in &self.bindings { let found = nodes.iter().any(|id| match (plan.node(*id), bound) { @@ -190,78 +214,192 @@ impl Requirements { )); } } - let expands = nodes - .iter() - .filter(|id| matches!(plan.node(**id), Some(PhysicalNode::Expand { .. }))) - .count(); - if expands != self.traversals { + let mut required: Vec<&super::requirements::Traversal> = self.traversals.iter().collect(); + for id in nodes { + let Some(PhysicalNode::Expand { + src, + dst, + edges, + src_type, + dst_type, + min_hops, + max_hops, + edge_binding, + .. + }) = plan.node(*id) + else { + continue; + }; + let position = required.iter().position(|traversal| { + traversal.src == *src + && traversal.dst == *dst + && traversal.edges == *edges + && traversal.src_type == *src_type + && traversal.dst_type == *dst_type + && traversal.min_hops == *min_hops + && traversal.max_hops == *max_hops + && traversal.edge_binding == *edge_binding + }); + match position { + Some(position) => { + required.remove(position); + } + None => { + return Err(ValidationError::violated( + "traversal identity", + format!( + "the plan traverses `${src}` to `${dst}` over {edges:?} ({min_hops}..{max_hops:?} hops); the query writes no such traversal" + ), + )); + } + } + } + if let Some(missing) = required.first() { return Err(ValidationError::violated( - "binding identity", + "traversal identity", format!( - "the query writes {} traversals; the plan runs {expands}", - self.traversals + "the query traverses `${}` to `${}`; the plan runs no such traversal", + missing.src, missing.dst ), )); } Ok(()) } - /// Every written eligibility conjunct is tested somewhere in each - /// pipeline the query's rows come from; a constant `true` constrains - /// nothing and needs no place. + /// The conjuncts a pipeline tests are exactly the ones the query + /// requires: every required conjunct is tested somewhere in it, and every + /// conjunct it tests is required, a constant `true` (which constrains + /// nothing) or the canonical identity of a written key equality on the + /// same scan. fn check_eligibility( &self, plan: &PhysicalPlan, nodes: &[NodeId], - matcher: &Matcher<'_>, + input: &AcceptInput<'_>, budget: &mut Budget, ) -> Result<(), ValidationError> { let placed = conjuncts(plan, nodes); - let mut written_conjuncts = Vec::new(); - for clause in &self.eligibility { - matcher.conjuncts(clause, &mut written_conjuncts); + budget.visit(u64::try_from(placed.len() + self.eligibility.len()).unwrap_or(u64::MAX))?; + let placed_set: HashSet<&IRExpr> = placed.iter().map(|(_, conjunct)| conjunct).collect(); + if let Some(missing) = self + .eligibility + .iter() + .find(|required| !placed_set.contains(required)) + { + return Err(ValidationError::violated( + "predicate retention", + format!("the plan tests no conjunct for `{missing}`"), + )); } - // Placement keeps written order within each node, so the search - // resumes after the last match: in-order placement costs one match - // per conjunct, and any other order still finds its conjunct. - let mut cursor = 0; - for written in written_conjuncts { - if matcher.constant(written) && matcher.value(written) == Some(Literal::Bool(true)) { + let required: HashSet<&IRExpr> = self.eligibility.iter().collect(); + for (id, conjunct) in &placed { + if constant_true(conjunct) + || required.contains(conjunct) + || self.narrowed_key(plan, *id, conjunct, &required, input) + { continue; } - let mut found = false; - for offset in 0..placed.len() { - let index = (cursor + offset) % placed.len(); - if matcher.matches(written, &placed[index], Position::Filter, budget)? { - found = true; - cursor = index + 1; - break; - } - } - if !found { - return Err(ValidationError::violated( - "predicate retention", - format!("the plan tests no conjunct for `{written}`"), - )); - } + return Err(ValidationError::violated( + "predicate origin", + format!("the plan tests `{conjunct}`, which the query does not require"), + )); } Ok(()) } - /// One correlated block per written block in each pipeline the query's - /// rows come from, each with the aggregate and comparison the query - /// wrote. - fn check_blocks( + /// Whether `conjunct`, placed on the scan `id`, is the identity equality + /// `scan_access::narrow_key` adds for a written equality on the scanned + /// type's single String key: the binding's id column equal, as a + /// non-null String, to the id the loader derives from the written value. + fn narrowed_key( &self, plan: &PhysicalPlan, - top: &[NodeId], - matcher: &Matcher<'_>, - budget: &mut Budget, - ) -> Result<(), ValidationError> { - let mut planned: Vec<_> = top + id: NodeId, + conjunct: &IRExpr, + required: &HashSet<&IRExpr>, + input: &AcceptInput<'_>, + ) -> bool { + let Some(PhysicalNode::Scan { spec, .. }) = plan.node(id) else { + return false; + }; + let (Some(binding), Some(type_name)) = + (spec.binding.as_deref(), spec.table.node_type_name()) + else { + return false; + }; + let Some(( + IRExpr::PropAccess { + variable, + property, + ty, + }, + CompOp::Eq, + IRExpr::Literal(Literal::String(narrowed), literal_ty), + )) = conjunct.comparison_parts() + else { + return false; + }; + if variable != binding + || property != input.catalog.system_columns.id + || *ty != narrowed_type() + || *literal_ty != narrowed_type() + { + return false; + } + let Some([key]) = input + .catalog + .node_types + .get(type_name) + .and_then(|node| node.key.as_deref()) + else { + return false; + }; + let Some(filter) = &spec.filter else { + return false; + }; + filter + .gq_filters() + .into_iter() + .flat_map(IRExpr::into_conjuncts) + .filter(|written| required.contains(written)) + .any(|written| { + let Some((left, CompOp::Eq, right)) = written.comparison_parts() else { + return false; + }; + [(left, right), (right, left)] + .into_iter() + .any(|(side, value)| { + let keyed = matches!(side, IRExpr::PropAccess { variable, property, .. } + if variable == binding && property == key); + let literal = match value { + IRExpr::Literal(literal, _) => Some(literal), + IRExpr::Param(name, _) => input.params.get(name), + _ => None, + }; + keyed + && literal.is_some_and(|literal| { + input + .keys + .canonical_key_id(&spec.table.type_key, literal, value.ty()) + .as_deref() + == Some(narrowed.as_str()) + }) + }) + }) + } + + /// The correlated blocks of each pipeline the query's rows come from are + /// exactly the written ones, each over its outer binding with the + /// predicate the query states. + fn check_blocks(&self, plan: &PhysicalPlan, nodes: &[NodeId]) -> Result<(), ValidationError> { + let mut planned: Vec<(&String, &omnigraph_compiler::ir::SubqueryPredicate)> = nodes .iter() .filter_map(|id| match plan.node(*id) { - Some(PhysicalNode::AntiJoin { predicate, .. }) => Some(predicate), + Some(PhysicalNode::AntiJoin { + outer_var, + predicate, + .. + }) => Some((outer_var, predicate)), _ => None, }) .collect(); @@ -276,16 +414,9 @@ impl Requirements { )); } for block in &self.blocks { - let mut found = None; - for (index, predicate) in planned.iter().enumerate() { - if predicate.func == block.func - && predicate.op == block.op - && matcher.matches(&block.right, &predicate.right, Position::Plain, budget)? - { - found = Some(index); - break; - } - } + let found = planned.iter().position(|(outer_var, predicate)| { + **outer_var == block.outer_var && **predicate == block.predicate + }); match found { Some(index) => { planned.remove(index); @@ -294,8 +425,8 @@ impl Requirements { return Err(ValidationError::violated( "predicate retention", format!( - "no correlated block of the plan compares `{} {} {}`", - block.func, block.op, block.right + "no correlated block of the plan over `${}` decides as the query's block does", + block.outer_var ), )); } @@ -307,12 +438,7 @@ impl Requirements { /// The plan ranks exactly the retrievals the leading `order` key names, /// each on its binding and property, with its argument and the /// approximation its contract declares, in the scope it orders. - fn check_search( - &self, - plan: &PhysicalPlan, - matcher: &Matcher<'_>, - budget: &mut Budget, - ) -> Result<(), ValidationError> { + fn check_search(&self, plan: &PhysicalPlan) -> Result<(), ValidationError> { let ranked = ranked_scans(plan); let fuse = plan.live().find_map(|(_, node)| match node { PhysicalNode::RankFuse { arms, k, .. } => Some((arms, k)), @@ -337,7 +463,7 @@ impl Requirements { )); } for (scope, retrieval) in &expected { - let Some((_, binding, access)) = + let Some((id, binding, access)) = ranked.iter().find(|(_, _, access)| access.scope == *scope) else { return Err(ValidationError::violated( @@ -348,7 +474,7 @@ impl Requirements { let same = *binding == retrieval.binding && access.kind == retrieval.kind && access.property == retrieval.property - && matcher.argument(&retrieval.query, &access.query, budget)?; + && access.query == retrieval.query; if !same { return Err(ValidationError::violated( "search identity", @@ -375,14 +501,10 @@ impl Requirements { ), )); } - let type_key = ranked - .iter() - .find(|(_, _, other)| std::ptr::eq(*other, *access)) - .and_then(|(id, _, _)| match plan.node(*id) { - Some(PhysicalNode::Scan { spec, .. }) => Some(spec.table.type_key.as_str()), - _ => None, - }) - .unwrap_or_default(); + let type_key = match plan.node(*id) { + Some(PhysicalNode::Scan { spec, .. }) => spec.table.type_key.as_str(), + _ => "", + }; self.check_access(plan, type_key, access)?; } match (&self.search, fuse) { @@ -398,12 +520,7 @@ impl Requirements { )); } } - let same_k = match (k, planned_k) { - (None, None) => true, - (Some(written), Some(planned)) => matcher.argument(written, planned, budget)?, - _ => false, - }; - if !same_k { + if planned_k != k { return Err(ValidationError::violated( "search identity", "the fusion's `k` is not the one the query writes", @@ -428,9 +545,9 @@ impl Requirements { } /// Every count cut the plan declares (`PhysicalNode::row_cut`) is one the - /// query requires, where it requires it: the root `Limit` and the final - /// sort's `fetch` carry the query's `limit`, a fusion keeps that `limit`, - /// and a ranked scan's candidate cap is the one its declared policy sets + /// query requires, where it requires it: the root `Limit` (under a root + /// hydration) and the final sort's `fetch` carry the query's `limit`, and + /// a ranked scan's candidate cap is the one its declared policy sets /// (checked with its access). Any other cut is refused, a capped sort /// below the final one and a cut in a correlated block's tree included. fn check_row_cuts( @@ -442,7 +559,7 @@ impl Requirements { let limit = self .limit .map(|limit| usize::try_from(limit).unwrap_or(usize::MAX)); - let root = plan.root(); + let root = below_hydration(plan, plan.root()); let mut id = match plan.node(root) { Some(PhysicalNode::Limit { input, .. }) => *input, _ => root, @@ -458,7 +575,6 @@ impl Requirements { let required = match node { PhysicalNode::Limit { .. } => id == root && limit == Some(cut), PhysicalNode::Sort { .. } => Some(id) == final_sort && limit == Some(cut), - PhysicalNode::RankFuse { .. } => limit == Some(cut), PhysicalNode::Scan { ranked: Some(_), .. } => true, @@ -563,13 +679,7 @@ impl Requirements { /// The plan's projection is the query's `return`, item for item, and /// every projected score is the score of the retrieval it names. - fn check_returns( - &self, - plan: &PhysicalPlan, - top: &[NodeId], - matcher: &Matcher<'_>, - budget: &mut Budget, - ) -> Result<(), ValidationError> { + fn check_returns(&self, plan: &PhysicalPlan, top: &[NodeId]) -> Result<(), ValidationError> { let projections = projections_of(plan, top); let [returns] = projections.as_slice() else { return Err(ValidationError::violated( @@ -590,50 +700,38 @@ impl Requirements { ), )); } - for (written, projected) in self.returns.iter().zip(returns.iter()) { - let alias = written - .alias - .clone() - .or_else(|| meta_field_result_key(&written.expr)); - if projected.alias != alias - || !matcher.matches(&written.expr, &projected.expr, Position::Return, budget)? - { + for (required, projected) in self.returns.iter().zip(returns.iter()) { + if required != projected { return Err(ValidationError::violated( "projection", format!( "the plan returns `{}`; the query returns `{}`", - projected.expr, written.expr + projected.expr, required.expr ), )); } } - for (_, call) in super::requirements::projected_rank_calls(&self.returns) { - let Some(projected) = projected_retrieval(call) else { - continue; - }; - let mut origin = false; - for retrieval in self.retrievals() { - if retrieval.kind == projected.kind - && retrieval.binding == projected.binding - && retrieval.property == projected.property - && retrieval.query == projected.query - { - origin = true; - break; - } - } - let planned = ranked_scans(plan).into_iter().any(|(_, binding, access)| { - binding == projected.binding - && access.kind == projected.kind - && access.property == projected.property + for (_, call) in &self.projected_ranks { + let origin = self.written_search.as_ref().is_some_and(|lead| { + lead == call + || matches!(lead, Expr::Rrf { primary, secondary, .. } + if **primary == *call || **secondary == *call) }); + let planned = match call { + Expr::Rrf { .. } => plan + .live() + .any(|(_, node)| matches!(node, PhysicalNode::RankFuse { .. })), + call => projected_retrieval(call).is_some_and(|(kind, binding, property)| { + ranked_scans(plan).into_iter().any(|(_, scanned, access)| { + scanned == binding && access.kind == kind && access.property == property + }) + }), + }; if !origin || !planned { return Err(ValidationError::violated( "score origin", format!( - "the projected `{call}` reads `${}.{}`, which no retrieval of the query with these arguments produces", - projected.binding, - score_column(projected.kind) + "the projected `{call}` reads a score no retrieval of the query with these arguments produces" ), )); } @@ -641,18 +739,145 @@ impl Requirements { Ok(()) } + /// A root hydration (`HydrateColumns`) reads only written return items: + /// each deferred column is the bare property the query returns at its + /// position, under that item's column, read from the pinned table of the + /// binding's own scan; only cuts and sorts sit between it and the + /// projection. + fn check_hydration(&self, plan: &PhysicalPlan, top: &[NodeId]) -> Result<(), ValidationError> { + for id in top { + let Some(PhysicalNode::HydrateColumns { input, bindings }) = plan.node(*id) else { + continue; + }; + if *id != plan.root() { + return Err(ValidationError::violated( + "hydration", + "a hydration below the plan's root reads columns after rows are decided", + )); + } + let mut below = *input; + while let Some(PhysicalNode::Limit { input, .. } | PhysicalNode::Sort { input, .. }) = + plan.node(below) + { + below = *input; + } + if !matches!(plan.node(below), Some(PhysicalNode::Projection { .. })) { + return Err(ValidationError::violated( + "hydration", + "a hydration does not sit over the query's projection", + )); + } + for hydrated in bindings { + let scanned = top.iter().any(|scan| { + matches!(plan.node(*scan), Some(PhysicalNode::Scan { spec, .. }) + if spec.binding.as_deref() == Some(hydrated.binding.as_str()) + && spec.table == hydrated.table) + }); + let declared = matches!(self.bindings.get(&hydrated.binding), + Some(BoundVariable::Node { type_name }) + if hydrated.table.node_type_name() == Some(type_name.as_str())); + if !scanned || !declared { + return Err(ValidationError::violated( + "hydration", + format!( + "the hydration of `${}` reads {}, which is not the table its scan reads", + hydrated.binding, hydrated.table.type_key + ), + )); + } + for column in &hydrated.columns { + let returned = self.returns.get(column.position).is_some_and(|item| { + item.column == column.output + && matches!(&item.expr, IRExpr::PropAccess { variable, property, .. } + if *variable == hydrated.binding && *property == column.property) + }); + if !returned { + return Err(ValidationError::violated( + "hydration", + format!( + "the hydration places `${}.{}` at return position {} as `{}`, which the query does not return there", + hydrated.binding, column.property, column.position, column.output + ), + )); + } + } + } + } + Ok(()) + } + + /// The final sort's comparator as RFC 0047 defines it: a ranking's score + /// (a fused ranking's `_rrf` descending) then the written keys; over a + /// search-led aggregate the written keys then every group key not written, + /// ascending; no sort where the query orders nothing. + fn required_keys(&self) -> Option> { + if self.aggregate { + if self.order.is_empty() { + return None; + } + let mut keys = self.order.clone(); + if self.search.is_some() { + for projection in &self.returns { + if matches!(projection.expr, IRExpr::Aggregate { .. }) { + continue; + } + let written = keys.iter().any(|key| match &key.expr { + IRExpr::AliasRef(alias, _) => *alias == projection.column, + expr => *expr == projection.expr, + }); + if !written { + keys.push(IROrdering { + expr: IRExpr::AliasRef( + projection.column.clone(), + projection.ty.clone(), + ), + descending: false, + }); + } + } + } + return Some(keys); + } + let lead = match &self.search { + None if self.order.is_empty() => return None, + None => Vec::new(), + Some(Search::Rank(retrieval)) => { + let (column, descending) = retrieval.kind.score(); + vec![(retrieval.binding.as_str(), column, descending)] + } + Some(Search::Fuse { arms, .. }) => vec![(arms[0].binding.as_str(), RRF_COLUMN, true)], + }; + Some( + lead.into_iter() + .map(|(binding, column, descending)| IROrdering { + expr: IRExpr::PropAccess { + variable: binding.to_string(), + property: column.to_string(), + ty: ExprType::Value { + scalar: ScalarType::F64, + list: false, + nullable: false, + }, + }, + descending, + }) + .chain(self.order.iter().cloned()) + .collect(), + ) + } + /// The plan's rows leave in the order the query requires and are cut - /// where it requires: the final sort's comparator is the search's score, - /// the remaining written keys and the binding identities, the `limit` - /// cuts its output, and nothing between them reorders or drops rows. + /// where it requires: the final sort's comparator is + /// [`Self::required_keys`] then the binding identities, the `limit` cuts + /// its output, and nothing between them reorders or drops rows. fn check_order_and_cut( &self, plan: &PhysicalPlan, top: &[NodeId], - matcher: &Matcher<'_>, + input: &AcceptInput<'_>, budget: &mut Budget, ) -> Result<(), ValidationError> { - let mut id = plan.root(); + let mut id = below_hydration(plan, plan.root()); if let Some(limit) = self.limit { let rows = usize::try_from(limit).unwrap_or(usize::MAX); match plan.node(id) { @@ -665,36 +890,7 @@ impl Requirements { } } } - let fused = matches!(self.search, Some(Search::Fuse { .. })); - if fused { - let planned = top.iter().find_map(|id| match plan.node(*id) { - Some(PhysicalNode::RankFuse { limit, .. }) => Some(*limit), - _ => None, - }); - let limit = self.limit.and_then(|limit| usize::try_from(limit).ok()); - if planned != Some(limit) { - return Err(ValidationError::violated( - "row cut", - format!("the fusion keeps {planned:?} rows; the query's `limit` is {limit:?}"), - )); - } - return Ok(()); - } - let score = match &self.search { - Some(Search::Rank(retrieval)) => Some(retrieval), - _ => None, - }; - if score.is_some() && self.aggregate { - return Ok(()); - } - // A key is constant by what it names: an alias by the `return` item it - // aliases, on the written side and on the planned side alike. - let written: Vec<&omnigraph_compiler::query::ast::Ordering> = self - .order - .iter() - .filter(|key| !matcher.constant(self.resolved_key(&key.expr))) - .collect(); - if score.is_none() && self.order.is_empty() { + let Some(required) = self.required_keys() else { let sorted = top .iter() .any(|id| matches!(plan.node(*id), Some(PhysicalNode::Sort { .. }))); @@ -705,7 +901,7 @@ impl Requirements { )); } return Ok(()); - } + }; loop { match plan.node(id) { Some(PhysicalNode::Sort { .. }) => break, @@ -743,113 +939,39 @@ impl Requirements { ), )); } - let mut keys = order_by.iter(); - if let Some(retrieval) = score { - let (column, descending) = retrieval.kind.score(); - let leads = keys.next().is_some_and(|key| { - key.descending == descending - && matches!(&key.expr, IRExpr::PropAccess { variable, property } - if *variable == retrieval.binding && property == column) - }); - if !leads { - return Err(ValidationError::violated( - "order", - format!( - "the final sort does not lead with the {:?} score of `${}`", - retrieval.kind, retrieval.binding - ), - )); - } - } - let returns = projection_of(plan, top) - .map(Vec::as_slice) - .unwrap_or_default(); - let planned_constant = |expr: &IRExpr| match expr { - IRExpr::AliasRef(column) => returns + let lead = required.len() - self.order.len().min(required.len()); + let same = order_by.len() == required.len() + && order_by .iter() - .find(|projection| result_column(projection).as_deref() == Some(column)) - .is_some_and(|projection| constant_ir(&projection.expr)), - other => constant_ir(other), - }; - let rest: Vec<&IROrdering> = keys.filter(|key| !planned_constant(&key.expr)).collect(); - if rest.len() != written.len() { + .zip(&required) + .enumerate() + .all(|(index, (planned, key))| { + if !self.aggregate && index < lead { + // A score key reads the ranked scan's (or the fusion's) + // score column of the binding, in the score's direction. + planned.descending == key.descending + && matches!((&planned.expr, &key.expr), ( + IRExpr::PropAccess { variable, property, .. }, + IRExpr::PropAccess { variable: v, property: p, .. }, + ) if variable == v && property == p) + } else { + planned == key + } + }); + if !same { return Err(ValidationError::violated( "order", format!( - "the final sort has {} written keys; the query writes {} that are not constant", - rest.len(), - written.len() + "the final sort orders by {}; the query requires {}", + keys_text(order_by), + keys_text(&required) ), )); } - for (key, planned) in written.iter().zip(&rest) { - if key.descending != planned.descending - || !self.same_key(&key.expr, planned, returns, matcher, budget)? - { - return Err(ValidationError::violated( - "order", - format!( - "the final sort orders by `{}`; the query writes `{}`", - planned.expr, key.expr - ), - )); - } - } - self.check_tiebreak(order_by, tiebreak, returns, matcher)?; + self.check_tiebreak(order_by, tiebreak, input)?; Ok(()) } - /// Whether a planned sort key is the written key: as written, or the - /// `return` item the planner bound it to by its result column, which the - /// written key must lower to (two differently written expressions may - /// lower to one, and the planner binds the first such item). In a - /// grouped query a written meta-field key binds by name to the unaliased - /// return item whose column it names, as the compiler lowers it. - fn same_key( - &self, - written: &Expr, - planned: &IROrdering, - returns: &[IRProjection], - matcher: &Matcher<'_>, - budget: &mut Budget, - ) -> Result { - if let IRExpr::AliasRef(column) = &planned.expr - && !matches!(written, Expr::AliasRef(alias) if alias == column) - { - let Some(index) = returns - .iter() - .position(|projection| result_column(projection).as_deref() == Some(column)) - else { - return Ok(false); - }; - let Some(item) = self.returns.get(index) else { - return Ok(false); - }; - let by_name = self.aggregate - && matches!(written, Expr::PropAccess { .. }) - && item.alias.is_none() - && meta_field_result_key(written).as_deref() == Some(column) - && meta_field_result_key(&item.expr).as_deref() == Some(column); - return Ok(by_name - || item.expr == *written - || matcher.matches(written, &returns[index].expr, Position::Return, budget)?); - } - matcher.matches(written, &planned.expr, Position::Plain, budget) - } - - /// A written order key with an alias resolved to the `return` item it - /// names. - fn resolved_key<'e>(&'e self, key: &'e Expr) -> &'e Expr { - match key { - Expr::AliasRef(alias) => self - .returns - .iter() - .find(|projection| projection.alias.as_deref() == Some(alias)) - .map_or(key, |projection| &projection.expr), - other => other, - } - } - /// The sort's metadata keys complete its comparator as RFC 0047 defines: /// every written binding's identity (`@type` then `@id` for a selected /// edge) in binding-name order, ascending, nulls first. Three planner @@ -857,13 +979,12 @@ impl Requirements { /// comparator already holds, every key when the query returns only order /// keys (rows that tie are indistinguishable), and identities of /// bindings the query does not name (anonymous traversal endpoints) - /// after every named one. + /// after every named one. A sort of group rows appends none. fn check_tiebreak( &self, order_by: &[IROrdering], tiebreak: &[ColumnRef], - returns: &[IRProjection], - matcher: &Matcher<'_>, + input: &AcceptInput<'_>, ) -> Result<(), ValidationError> { if self.aggregate { if !tiebreak.is_empty() { @@ -874,58 +995,40 @@ impl Requirements { } return Ok(()); } - let covered = self.returns.iter().all(|projection| { - self.order - .iter() - .any(|key| *self.resolved_key(&key.expr) == projection.expr) - || match &self.search { - Some(Search::Rank(retrieval)) => projected_retrieval(&projection.expr) - .is_some_and(|projected| { - projected.kind == retrieval.kind - && projected.binding == retrieval.binding - && projected.property == retrieval.property - && projected.query == retrieval.query - }), - _ => false, - } + let ranked_score = |index: usize| { + matches!(self.search, Some(Search::Rank(_))) + && self.projected_ranks.iter().any(|(position, call)| { + *position == index && self.written_search.as_ref() == Some(call) + }) + }; + let covered = self.returns.iter().enumerate().all(|(index, projection)| { + ranked_score(index) + || order_by.iter().any(|key| match &key.expr { + IRExpr::AliasRef(alias, _) => *alias == projection.column, + expr => *expr == projection.expr, + }) }); - // The same claim over the lowered expressions: an expression a key - // lowers to, compared with what each returned item lowers to, since - // two differently written expressions may lower to one. - fn lowered_key<'k>(key: &'k IROrdering, returns: &'k [IRProjection]) -> &'k IRExpr { - match &key.expr { - IRExpr::AliasRef(alias) => returns - .iter() - .find(|projection| result_column(projection).as_deref() == Some(alias)) - .map_or(&key.expr, |projection| &projection.expr), - expr => expr, - } - } - let covered_lowered = !returns.is_empty() - && returns.iter().all(|projection| { - order_by - .iter() - .any(|key| *lowered_key(key, returns) == projection.expr) - }); - if covered || covered_lowered { + if covered { return Ok(()); } + let columns = input.catalog.system_columns; let keyed = |column: &ColumnRef| { let physical = match column.property.as_deref() { - Some(IDENTITY_MEMBER) => matcher.physical(SYSTEM_ID), - Some(EDGE_TYPE_MEMBER) => matcher.physical(EDGE_TYPE_MEMBER), + Some(IDENTITY_MEMBER) => columns.id, + Some(EDGE_TYPE_MEMBER) => omnigraph_compiler::traversal::EDGE_TYPE_COLUMN, _ => return false, }; order_by.iter().any(|key| { let expr = match &key.expr { - IRExpr::AliasRef(alias) => returns + IRExpr::AliasRef(alias, _) => self + .returns .iter() - .find(|projection| projection.alias.as_deref() == Some(alias)) + .find(|projection| projection.column == *alias) .map_or(&key.expr, |projection| &projection.expr), expr => expr, }; - matches!(expr, IRExpr::PropAccess { variable, property } - if *variable == column.binding && *property == physical) + matches!(expr, IRExpr::PropAccess { variable, property, .. } + if *variable == column.binding && property == physical) }) }; let mut expected = Vec::new(); @@ -957,8 +1060,6 @@ impl Requirements { } } -const SYSTEM_ID: &str = "@id"; - /// The adaptive search policies the plan declares (RFC 0047, "Execution and /// replay use the same acceptance path"): every `nearest` scan's probe /// ladder terminates and keeps its correctness fallbacks, and every @@ -1126,6 +1227,14 @@ fn check_prefilter( Ok(()) } +/// The node under a root hydration, else the root. +fn below_hydration(plan: &PhysicalPlan, root: NodeId) -> NodeId { + match plan.node(root) { + Some(PhysicalNode::HydrateColumns { input, .. }) => *input, + _ => root, + } +} + /// Every node of `top` that projects the query's `return`. fn projections_of<'p>(plan: &'p PhysicalPlan, top: &[NodeId]) -> Vec<&'p Vec> { top.iter() @@ -1140,41 +1249,27 @@ fn projections_of<'p>(plan: &'p PhysicalPlan, top: &[NodeId]) -> Vec<&'p Vec(plan: &'p PhysicalPlan, top: &[NodeId]) -> Option<&'p Vec> { - match projections_of(plan, top).as_slice() { - [returns] => Some(*returns), - _ => None, - } -} - -/// Whether a planned key reads no row: ordering by it orders nothing. -fn constant_ir(expr: &IRExpr) -> bool { - match expr { - IRExpr::Literal(_) | IRExpr::Param(_) => true, - IRExpr::Binary { left, right, .. } => constant_ir(left) && constant_ir(right), - IRExpr::Not(inner) | IRExpr::IsNull { expr: inner, .. } => constant_ir(inner), - _ => false, - } +fn keys_text(keys: &[IROrdering]) -> String { + keys.iter() + .map(|key| format!("{}{}", key.expr, if key.descending { " desc" } else { "" })) + .collect::>() + .join(", ") } fn exactness(approximate: bool) -> &'static str { if approximate { "approximate" } else { "exact" } } -/// The retrieval a projected rank call names. -fn projected_retrieval(expr: &Expr) -> Option { - super::requirements::retrieval_of(expr) -} - -/// The alias the compiler gives an unaliased meta-field projection: the -/// logical `var.@prop`. -fn meta_field_result_key(expr: &Expr) -> Option { +/// The (kind, binding, property) a projected `nearest` or `bm25` call reads. +fn projected_retrieval(expr: &Expr) -> Option<(RankKind, &str, &str)> { match expr { - Expr::PropAccess { variable, property } if property.starts_with('@') => { - Some(format!("{variable}.{property}")) - } - Expr::Aggregate { arg, .. } => meta_field_result_key(arg), + Expr::Nearest { + variable, property, .. + } => Some((RankKind::Nearest, variable, property)), + Expr::Bm25 { field, .. } => match field.as_ref() { + Expr::PropAccess { variable, property } => Some((RankKind::Bm25, variable, property)), + _ => None, + }, _ => None, } } diff --git a/crates/omnigraph-planner/src/validate/mod.rs b/crates/omnigraph-planner/src/validate/mod.rs index bcbea2678..2021ec044 100644 --- a/crates/omnigraph-planner/src/validate/mod.rs +++ b/crates/omnigraph-planner/src/validate/mod.rs @@ -23,7 +23,7 @@ pub use replay::{ REPLAY_VERSION, RULES_VERSION, ReplayEnvelope, ReplayQuery, ReplayRefusal, SEMANTICS_VERSION, accept_replay, catalog_digest, decode_replay, }; -pub use requirements::{ConstantEvaluator, Requirements}; +pub use requirements::{KeyIds, Requirements}; pub use subset::{ChainNode, ChainRanking, Derivation, Rule, Step}; use omnigraph_compiler::CheckedQuery; @@ -106,15 +106,16 @@ impl std::fmt::Display for ValidationError { /// Everything acceptance reads beside the plan source: the checked /// declaration the requirements come from, the catalog it was checked -/// against, the IR the planner plans (constants of filter positions already -/// folded to their bound values), the bound parameter values (`now()` -/// among them), the evaluator of a folded constant, and the limits. +/// against, the typed IR the planner plans (constants of filter positions +/// already folded to their bound values), the bound parameter values +/// (`now()` among them), the derivation of a key's canonical id, and the +/// limits. pub struct AcceptInput<'a> { pub checked: &'a CheckedQuery, pub catalog: &'a Catalog, pub ir: &'a QueryIR, pub params: &'a ParamMap, - pub constants: &'a dyn ConstantEvaluator, + pub keys: &'a dyn KeyIds, pub limits: ValidationLimits, } @@ -224,15 +225,33 @@ pub(crate) fn accept( }) } -/// The checks of one acceptance, shared by fresh planning and replay: the -/// invariants for every plan, and for a member of the exact fragment its -/// derivation, which a member must carry. +/// The checks of one acceptance, shared by fresh planning and replay, in +/// three phases: the plan is well formed (its aggregate specs, output +/// schemas and fusion row keys agree with its own nodes), it keeps exactly +/// what the query requires (the invariants), and a member of the exact +/// fragment carries its checked derivation. pub(crate) fn check( plan: &PhysicalPlan, input: &AcceptInput<'_>, derivation: Option, budget: &mut Budget, ) -> Result<(ValidationScope, Option), ValidationError> { + budget.visit(u64::try_from(plan.live().count()).unwrap_or(u64::MAX))?; + let formed = |check: &'static str, result: Result<(), crate::error::PlanError>| { + result.map_err(|error| ValidationError::violated(check, error.to_string())) + }; + formed( + "aggregate spec", + crate::aggregate::validate_aggregate_specs(plan), + )?; + formed( + "output schema", + crate::output::validate_output_schemas(plan), + )?; + formed( + "fusion row keys", + crate::optimizer::validate_rank_fuse_row_tiebreaks(plan), + )?; let requirements = Requirements::derive(input, budget)?; requirements.check(plan, input, budget)?; let Some(member) = subset::membership(input) else { diff --git a/crates/omnigraph-planner/src/validate/replay.rs b/crates/omnigraph-planner/src/validate/replay.rs index 5c748d9b7..c6b7142a8 100644 --- a/crates/omnigraph-planner/src/validate/replay.rs +++ b/crates/omnigraph-planner/src/validate/replay.rs @@ -26,10 +26,10 @@ pub const REPLAY_VERSION: u32 = 1; /// The catalogue of checks and rewrite rules acceptance applies. A rule id /// keeps its meaning within one version. -pub const RULES_VERSION: u32 = 1; +pub const RULES_VERSION: u32 = 2; /// The compiler and operator semantics a plan was built under. -pub const SEMANTICS_VERSION: u32 = 1; +pub const SEMANTICS_VERSION: u32 = 2; /// The query a saved plan came from: its source and declaration name. Its /// parameter values are the plan's own bound values. diff --git a/crates/omnigraph-planner/src/validate/requirements.rs b/crates/omnigraph-planner/src/validate/requirements.rs index 76f5c4acb..ab6233e84 100644 --- a/crates/omnigraph-planner/src/validate/requirements.rs +++ b/crates/omnigraph-planner/src/validate/requirements.rs @@ -1,72 +1,66 @@ //! What a checked read declaration requires of every plan for it (RFC 0047, -//! "Requirements come from the checked query"), derived from the declaration -//! as written, its type context and the catalog, never from the IR: the IR -//! has already lost meaning (a projected `bm25()` is a score column, an -//! inline match is a comparison, constants are folded). The requirements are -//! a derived validation input, never an editable description of the query. -//! -//! The [`Matcher`] relates a written expression to the IR expression a plan -//! carries through the compiler's defined lowering patterns: property leaves -//! on their physical columns, parameters as parameters, `in` as `contains` -//! with its operands swapped, `contains` over a String as `StringContains`, -//! a bare search call as `call = true`, a projected rank call as its score -//! column, and a constant subtree as the literal it evaluates to under the -//! bound parameters. It checks a lowering; it does not lower. +//! "Requirements come from the checked query"). The declaration fixes what +//! the query asks for: its bindings and their types, how many traversals, +//! correlated blocks, return items and order keys it writes, its limit, and +//! which rank call a projected score names. The compiler's typed lowering of +//! that declaration (`AcceptInput::ir`, its filter constants folded under the +//! bound values) states each expression in the form a plan carries it: every +//! conjunct, traversal, block predicate, return item and order key a plan +//! places is a verbatim copy of an IR expression, so plan and requirement +//! compare as typed IR, explicit casts and exact-integer carriers included. +//! The declaration is consulted only where the lowering erases meaning: a +//! projected rank call becomes its score column. use std::collections::{BTreeMap, HashSet}; -use omnigraph_compiler::catalog::Catalog; -use omnigraph_compiler::ir::{IRExpr, IRProjection}; -use omnigraph_compiler::query::ast::{ - AggFunc, BinaryOp, Clause, CompOp, DISTANCE_COLUMN, Expr, Literal, NOW_PARAM_NAME, Ordering, - Projection, SCORE_COLUMN, -}; +use omnigraph_compiler::ir::{IRExpr, IROp, IROrdering, IRProjection, SubqueryPredicate}; +use omnigraph_compiler::query::ast::{Clause, Expr, Literal, Projection}; use omnigraph_compiler::query::typecheck::BoundVariable; -use omnigraph_compiler::traversal::{EDGE_TYPE_COLUMN, EDGE_TYPE_META}; -use omnigraph_compiler::{SYSTEM_COLUMNS_META, SystemColumns}; +use omnigraph_compiler::traversal::EdgeSelection; +use omnigraph_compiler::types::ExprType; use super::budget::Budget; use super::{AcceptInput, ValidationError}; use crate::physical::RankKind; -/// The value of a constant expression (literals, bound parameters, `now()`, -/// and comparisons, `and`, `or`, `not` and null tests over them) under the -/// bound parameters, or `None` when it has none. The engine implements it -/// with the evaluator its constant folding uses, so a folded literal is -/// checked against the same rules that produced it. -pub trait ConstantEvaluator { - fn evaluate(&self, expr: &IRExpr) -> Option; +/// The canonical id of a node keyed by one String property, as every write +/// path derives it from the key's value. The engine implements it with the +/// loader's derivation, so a key narrowing the planner adds (an `@id` +/// equality beside the written key equality) is re-derived, not trusted. +pub trait KeyIds { + /// The id of the node of `type_key` whose key is `value` (typed `ty`), or + /// `None` when the type is not keyed by one String property. + fn canonical_key_id(&self, type_key: &str, value: &Literal, ty: &ExprType) -> Option; } -/// One retrieval a search order names: `nearest($v.p, q)` or `bm25($v.p, q)`. -#[derive(Debug, Clone)] +/// One retrieval a search order names: `nearest($v.p, q)` or `bm25($v.p, q)`, +/// with the query argument as the plan's ranked scan carries it. +#[derive(Debug, Clone, PartialEq)] pub(crate) struct Retrieval { pub kind: RankKind, pub binding: String, pub property: String, - pub query: Expr, -} - -/// The retrieval a rank call names, `None` for any other expression. -pub(crate) fn retrieval_of(expr: &Expr) -> Option { - Retrieval::of(expr) + pub query: IRExpr, } impl Retrieval { - fn of(expr: &Expr) -> Option { + pub(crate) fn of(expr: &IRExpr) -> Option { match expr { - Expr::Nearest { + IRExpr::Nearest { variable, property, query, + .. } => Some(Self { kind: RankKind::Nearest, binding: variable.clone(), property: property.clone(), query: query.as_ref().clone(), }), - Expr::Bm25 { field, query } => match field.as_ref() { - Expr::PropAccess { variable, property } => Some(Self { + IRExpr::Bm25 { field, query, .. } => match field.as_ref() { + IRExpr::PropAccess { + variable, property, .. + } => Some(Self { kind: RankKind::Bm25, binding: variable.clone(), property: property.clone(), @@ -91,16 +85,29 @@ pub(crate) enum Search { Rank(Retrieval), Fuse { arms: [Retrieval; 2], - k: Option, + k: Option, }, } -/// A correlated block of the top-level scope: the aggregate it compares. -#[derive(Debug, Clone)] +/// One traversal of the top-level scope, as the IR states it. +#[derive(Debug, Clone, PartialEq)] +pub(crate) struct Traversal { + pub src: String, + pub dst: String, + pub edges: EdgeSelection, + pub src_type: String, + pub dst_type: String, + pub min_hops: u32, + pub max_hops: Option, + pub edge_binding: Option, +} + +/// A correlated block of the top-level scope: its outer binding and the +/// predicate over the aggregate of its matches. +#[derive(Debug, Clone, PartialEq)] pub(crate) struct Block { - pub func: AggFunc, - pub op: CompOp, - pub right: Expr, + pub outer_var: String, + pub predicate: SubqueryPredicate, } /// What every plan for one checked declaration must preserve. @@ -111,18 +118,26 @@ pub struct Requirements { /// Edge bindings of a traversal over more than one edge type, whose /// identity is the pair (`@type`, `@id`). pub(crate) selected_edges: HashSet, - /// Traversal clauses of the top-level scope. - pub(crate) traversals: usize, + /// Traversals of the top-level scope, in written order. + pub(crate) traversals: Vec, /// Correlated blocks of the top-level scope, in written order. pub(crate) blocks: Vec, pub(crate) search: Option, - /// Every eligibility clause of the top-level scope as written, an - /// inline binding match spelled as its comparison; [`Matcher::conjuncts`] - /// splits a clause the way its lowering does. - pub(crate) eligibility: Vec, - pub(crate) returns: Vec, - /// The `order` keys after a leading search key. - pub(crate) order: Vec, + /// Every eligibility conjunct of the top-level scope (inline matches, + /// `match` filters and the filters of a deferred traversal destination), + /// split at its top-level `and`; a constant `true` constrains nothing and + /// is left out. + pub(crate) eligibility: Vec, + pub(crate) returns: Vec, + /// The written return items that project a rank call, by position, with + /// the call as written. + pub(crate) projected_ranks: Vec<(usize, Expr)>, + /// The leading `order` key as written, when it is a search. + pub(crate) written_search: Option, + /// The `order` keys after a leading search key that order something (a + /// literal key does not), each bound to a `return` item as the planner + /// binds it. + pub(crate) order: Vec, pub(crate) limit: Option, pub(crate) aggregate: bool, } @@ -134,56 +149,106 @@ impl Requirements { ) -> Result { let decl = input.checked.decl(); let types = input.checked.types(); + let ir = input.ir; let mut eligibility = Vec::new(); - let mut traversals = 0; + let mut traversals = Vec::new(); let mut blocks = Vec::new(); - let mut selected_edges = HashSet::new(); - for clause in &decl.match_clause { + for op in &ir.pipeline { budget.visit(1)?; - match clause { - Clause::Binding(binding) => { - let node_type = input - .catalog - .node_types - .get(&binding.type_name) - .ok_or_else(|| { - ValidationError::violated( + match op { + IROp::NodeScan { + variable, + type_name, + filters, + } => { + match types.bindings.get(variable) { + Some(BoundVariable::Node { + type_name: declared, + }) if declared == type_name => {} + _ => { + return Err(ValidationError::violated( "binding identity", format!( - "`${}` binds `{}`, which the catalog does not hold", - binding.variable, binding.type_name + "the lowering scans `${variable}: {type_name}`, which the query does not bind" ), - ) - })?; - for matched in &binding.prop_matches { - let list = node_type - .properties - .get(&matched.prop_name) - .is_some_and(|property| property.list); - eligibility.push(Expr::comparison( - Expr::PropAccess { - variable: binding.variable.clone(), - property: matched.prop_name.clone(), - }, - if list { CompOp::Contains } else { CompOp::Eq }, - matched.value.clone(), - )); + )); + } } + push_conjuncts(&mut eligibility, filters); } - Clause::Traversal(_) => {} - Clause::Filter(filter) => { - eligibility.push(filter.clone().with_search_predicates_spelled()); + IROp::Filter(filter) => { + push_conjuncts(&mut eligibility, std::slice::from_ref(filter)) } - Clause::Subquery(block) => blocks.push(Block { - func: block.func, - op: block.op, - right: block.right.clone(), + IROp::Expand { + src_var, + dst_var, + edges, + src_type, + dst_type, + min_hops, + max_hops, + dst_filters, + edge_binding, + } => { + traversals.push(Traversal { + src: src_var.clone(), + dst: dst_var.clone(), + edges: edges.clone(), + src_type: src_type.clone(), + dst_type: dst_type.clone(), + min_hops: *min_hops, + max_hops: *max_hops, + edge_binding: edge_binding.clone(), + }); + push_conjuncts(&mut eligibility, dst_filters); + } + IROp::AntiJoin { + outer_var, + predicate, + .. + } => blocks.push(Block { + outer_var: outer_var.clone(), + predicate: predicate.clone(), }), } } + // The lowering states each expression; the declaration fixes how many + // of each the query asks for. + let written_blocks = decl + .match_clause + .iter() + .filter(|clause| matches!(clause, Clause::Subquery(_))) + .count(); + let counts = [ + ("traversals", types.traversals.len(), traversals.len()), + ("correlated blocks", written_blocks, blocks.len()), + ( + "return items", + decl.return_clause.len(), + ir.return_exprs.len(), + ), + ("order keys", decl.order_clause.len(), ir.order_by.len()), + ]; + for (what, written, lowered) in counts { + if written != lowered { + return Err(ValidationError::violated( + "lowering", + format!("the query writes {written} {what}; its lowering holds {lowered}"), + )); + } + } + if ir.limit != decl.limit { + return Err(ValidationError::violated( + "lowering", + format!( + "the query's `limit` is {:?}; its lowering's is {:?}", + decl.limit, ir.limit + ), + )); + } + let mut selected_edges = HashSet::new(); for traversal in &types.traversals { budget.visit(1)?; - traversals += 1; if let Some(edge) = traversal .edge_binding .as_deref() @@ -193,12 +258,13 @@ impl Requirements { selected_edges.insert(edge.to_string()); } } - let mut order = decl.order_clause.clone(); - let search = match order.first().map(|key| &key.expr) { - Some(Expr::Rrf { + let mut keys = ir.order_by.as_slice(); + let search = match keys.first().map(|key| &key.expr) { + Some(IRExpr::Rrf { primary, secondary, k, + .. }) => match (Retrieval::of(primary), Retrieval::of(secondary)) { (Some(primary), Some(secondary)) => Some(Search::Fuse { arms: [primary, secondary], @@ -210,8 +276,20 @@ impl Requirements { None => None, }; if search.is_some() { - order.remove(0); + keys = &keys[1..]; } + let order = keys + .iter() + .filter(|key| !matches!(key.expr, IRExpr::Literal(_, _))) + .map(|key| IROrdering { + expr: crate::optimizer::bind_order_key(&key.expr, &ir.return_exprs), + descending: key.descending, + }) + .collect(); + let written_search = search + .as_ref() + .and(decl.order_clause.first()) + .map(|key| key.expr.clone()); Ok(Self { bindings: types .bindings @@ -223,356 +301,34 @@ impl Requirements { blocks, search, eligibility, - returns: decl.return_clause.clone(), + returns: ir.return_exprs.clone(), + projected_ranks: projected_rank_calls(&decl.return_clause) + .into_iter() + .map(|(index, call)| (index, call.clone())) + .collect(), + written_search, order, - limit: decl.limit, - aggregate: decl - .return_clause + limit: ir.limit, + aggregate: ir + .return_exprs .iter() - .any(|projection| matches!(projection.expr, Expr::Aggregate { .. })), + .any(|projection| matches!(projection.expr, IRExpr::Aggregate { .. })), }) } - - /// Every retrieval the query names, in written order. - pub(crate) fn retrievals(&self) -> Vec<&Retrieval> { - match &self.search { - None => Vec::new(), - Some(Search::Rank(retrieval)) => vec![retrieval], - Some(Search::Fuse { arms, .. }) => arms.iter().collect(), - } - } -} - -/// Where an expression sits, which decides the lowering pattern it follows. -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -pub(crate) enum Position { - /// A `match` conjunct: constant subtrees are folded to their value. - Filter, - /// A `return` item: a rank call is its score column. - Return, - /// An `order` key or a call argument: lowered as written. - Plain, -} - -/// Relates written expressions to the IR expressions of a plan. -pub(crate) struct Matcher<'a> { - pub catalog: &'a Catalog, - pub params: HashSet<&'a str>, - pub bindings: &'a BTreeMap, - pub constants: &'a dyn ConstantEvaluator, } -impl<'a> Matcher<'a> { - pub(crate) fn new(input: &'a AcceptInput<'a>, requirements: &'a Requirements) -> Self { - Self { - catalog: input.catalog, - params: input - .checked - .decl() - .params - .iter() - .map(|param| param.name.as_str()) - .collect(), - bindings: &requirements.bindings, - constants: input.constants, - } - } - - fn system_columns(&self) -> SystemColumns { - self.catalog.system_columns - } - - /// The physical column a written property leaf reads. - pub(crate) fn physical(&self, property: &str) -> String { - let columns = self.system_columns(); - match property { - EDGE_TYPE_META => EDGE_TYPE_COLUMN.to_string(), - name if name == SYSTEM_COLUMNS_META.id => columns.id.to_string(), - name if name == SYSTEM_COLUMNS_META.src => columns.src.to_string(), - name if name == SYSTEM_COLUMNS_META.dst => columns.dst.to_string(), - other => other.to_string(), +/// `filters` split at their top-level `and`, a constant `true` left out. +fn push_conjuncts(out: &mut Vec, filters: &[IRExpr]) { + for conjunct in filters.iter().cloned().flat_map(IRExpr::into_conjuncts) { + if !constant_true(&conjunct) { + out.push(conjunct); } } - - /// The one edge type a single-type edge binding's `@type` reads, which - /// the lowering writes as a String literal. - fn single_edge_type(&self, variable: &str, property: &str) -> Option<&str> { - if property != EDGE_TYPE_META { - return None; - } - match self.bindings.get(variable) { - Some(BoundVariable::Edge { type_names }) => match type_names.as_slice() { - [name] => Some(name), - _ => None, - }, - _ => None, - } - } - - /// Whether `expr` reads only literals, parameters, the clock and the - /// `@type` of a single-type edge binding. - pub(crate) fn constant(&self, expr: &Expr) -> bool { - match expr { - Expr::Literal(_) | Expr::Now => true, - Expr::PropAccess { variable, property } => { - self.single_edge_type(variable, property).is_some() - } - Expr::Variable(name) => self.params.contains(name.as_str()), - Expr::Binary { left, right, .. } => self.constant(left) && self.constant(right), - Expr::In { needle, list } => self.constant(needle) && self.constant(list), - Expr::Not(inner) | Expr::IsNull { expr: inner, .. } => self.constant(inner), - _ => false, - } - } - - /// A constant written subtree as the IR expression its value is - /// computed from, for the evaluator. - fn constant_ir(&self, expr: &Expr) -> Option { - Some(match expr { - Expr::Literal(literal) => IRExpr::Literal(literal.clone()), - Expr::PropAccess { variable, property } => IRExpr::Literal(Literal::String( - self.single_edge_type(variable, property)?.to_string(), - )), - Expr::Now => IRExpr::Param(NOW_PARAM_NAME.to_string()), - Expr::Variable(name) if self.params.contains(name.as_str()) => { - IRExpr::Param(name.clone()) - } - Expr::Binary { left, op, right } => IRExpr::Binary { - left: Box::new(self.constant_ir(left)?), - op: *op, - right: Box::new(self.constant_ir(right)?), - }, - Expr::In { needle, list } => IRExpr::Binary { - left: Box::new(self.constant_ir(list)?), - op: BinaryOp::Compare(CompOp::Contains), - right: Box::new(self.constant_ir(needle)?), - }, - Expr::Not(inner) => IRExpr::Not(Box::new(self.constant_ir(inner)?)), - Expr::IsNull { expr, negated } => IRExpr::IsNull { - expr: Box::new(self.constant_ir(expr)?), - negated: *negated, - }, - _ => return None, - }) - } - - /// A written clause split into the conjuncts its lowering tests: the - /// top-level `and` chain, except that a constant `and` is folded into one - /// value before the split. - pub(crate) fn conjuncts<'e>(&self, clause: &'e Expr, out: &mut Vec<&'e Expr>) { - match clause { - Expr::Binary { - left, - op: BinaryOp::And, - right, - } if !self.constant(clause) => { - self.conjuncts(left, out); - self.conjuncts(right, out); - } - other => out.push(other), - } - } - - /// The value of a constant written expression under the bound - /// parameters. - pub(crate) fn value(&self, expr: &Expr) -> Option { - self.constants.evaluate(&self.constant_ir(expr)?) - } - - /// Whether `ir` is the lowering of `written` at `position`. - pub(crate) fn matches( - &self, - written: &Expr, - ir: &IRExpr, - position: Position, - budget: &mut Budget, - ) -> Result { - budget.visit(1)?; - if let IRExpr::Literal(value) = ir - && !matches!(written, Expr::Literal(_)) - && self.constant(written) - { - return Ok(self - .constant_ir(written) - .and_then(|constant| self.constants.evaluate(&constant)) - .is_some_and(|evaluated| evaluated == *value)); - } - if position == Position::Return - && let Some((variable, column)) = written.score_column() - { - return Ok(matches!( - ir, - IRExpr::PropAccess { variable: v, property: p } if v == variable && p == column - )); - } - Ok(match (written, ir) { - (Expr::Now, IRExpr::Param(name)) => name == NOW_PARAM_NAME, - (Expr::Literal(left), IRExpr::Literal(right)) => left == right, - (Expr::Variable(name), IRExpr::Param(param)) => { - self.params.contains(name.as_str()) && name == param - } - (Expr::Variable(name), IRExpr::Variable(variable)) => { - !self.params.contains(name.as_str()) && name == variable - } - (Expr::AliasRef(left), IRExpr::AliasRef(right)) => left == right, - (Expr::PropAccess { variable, property }, IRExpr::Literal(Literal::String(name))) => { - self.single_edge_type(variable, property) == Some(name.as_str()) - } - ( - Expr::PropAccess { variable, property }, - IRExpr::PropAccess { - variable: v, - property: p, - }, - ) => variable == v && self.physical(property) == *p, - ( - Expr::Binary { left, op, right }, - IRExpr::Binary { - left: l, - op: o, - right: r, - }, - ) => { - same_op(*op, *o) - && self.matches(left, l, position, budget)? - && self.matches(right, r, position, budget)? - } - ( - Expr::In { needle, list }, - IRExpr::Binary { - left: l, - op: o, - right: r, - }, - ) => { - same_op(BinaryOp::Compare(CompOp::Contains), *o) - && self.matches(list, l, position, budget)? - && self.matches(needle, r, position, budget)? - } - (Expr::Not(inner), IRExpr::Not(i)) => self.matches(inner, i, position, budget)?, - ( - Expr::IsNull { expr, negated }, - IRExpr::IsNull { - expr: e, - negated: n, - }, - ) => negated == n && self.matches(expr, e, position, budget)?, - (Expr::Aggregate { func, arg }, IRExpr::Aggregate { func: f, arg: a }) => { - func == f && self.matches(arg, a, Position::Plain, budget)? - } - ( - Expr::Nearest { - variable, - property, - query, - }, - IRExpr::Nearest { - variable: v, - property: p, - query: q, - }, - ) => { - variable == v && property == p && self.matches(query, q, Position::Plain, budget)? - } - (Expr::Search { field, query }, IRExpr::Search { field: f, query: q }) - | (Expr::MatchText { field, query }, IRExpr::MatchText { field: f, query: q }) - | (Expr::Bm25 { field, query }, IRExpr::Bm25 { field: f, query: q }) => { - self.matches(field, f, Position::Plain, budget)? - && self.matches(query, q, Position::Plain, budget)? - } - ( - Expr::Fuzzy { - field, - query, - max_edits, - }, - IRExpr::Fuzzy { - field: f, - query: q, - max_edits: m, - }, - ) => { - self.matches(field, f, Position::Plain, budget)? - && self.matches(query, q, Position::Plain, budget)? - && match (max_edits, m) { - (None, None) => true, - (Some(written), Some(ir)) => { - self.matches(written, ir, Position::Plain, budget)? - } - _ => false, - } - } - ( - Expr::Rrf { - primary, - secondary, - k, - }, - IRExpr::Rrf { - primary: p, - secondary: s, - k: kk, - }, - ) => { - self.matches(primary, p, Position::Plain, budget)? - && self.matches(secondary, s, Position::Plain, budget)? - && match (k, kk) { - (None, None) => true, - (Some(written), Some(ir)) => { - self.matches(written, ir, Position::Plain, budget)? - } - _ => false, - } - } - _ => false, - }) - } - - /// Whether `ir` carries the written query argument of a retrieval: as - /// written, or a constant argument as its bound value. - pub(crate) fn argument( - &self, - written: &Expr, - ir: &IRExpr, - budget: &mut Budget, - ) -> Result { - self.matches(written, ir, Position::Plain, budget) - } -} - -/// `op` as the lowering spells it: `contains` over a String left operand -/// becomes `StringContains`; every other operator is kept. -fn same_op(written: BinaryOp, ir: BinaryOp) -> bool { - written == ir - || written == BinaryOp::Compare(CompOp::Contains) - && ir == BinaryOp::Compare(CompOp::StringContains) -} - -/// The column a projected rank expression lowers to, as `(binding, column)`. -pub(crate) fn score_column(kind: RankKind) -> &'static str { - match kind { - RankKind::Nearest => DISTANCE_COLUMN, - RankKind::Bm25 => SCORE_COLUMN, - } } -/// The `return` item an order key names by its result column, as the -/// planner binds a non-property key: the alias, else the expression's own -/// column. -pub(crate) fn result_column(projection: &IRProjection) -> Option { - fn column(expr: &IRExpr) -> Option { - match expr { - IRExpr::PropAccess { variable, property } => Some(format!("{variable}.{property}")), - IRExpr::Variable(name) | IRExpr::Param(name) => Some(name.clone()), - IRExpr::Literal(_) => Some("literal".to_string()), - IRExpr::Aggregate { arg, .. } => column(arg), - _ => None, - } - } - projection - .alias - .clone() - .or_else(|| column(&projection.expr)) +/// Whether a conjunct is the literal `true`, which constrains nothing. +pub(crate) fn constant_true(conjunct: &IRExpr) -> bool { + matches!(conjunct, IRExpr::Literal(Literal::Bool(true), _)) } /// Every written return item that projects a rank call, at its root or in diff --git a/crates/omnigraph-planner/src/validate/subset.rs b/crates/omnigraph-planner/src/validate/subset.rs index b3bf55a29..9c81fb052 100644 --- a/crates/omnigraph-planner/src/validate/subset.rs +++ b/crates/omnigraph-planner/src/validate/subset.rs @@ -1,12 +1,13 @@ //! The exact fragment and its checked derivation (RFC 0047, "Checked //! rewrites for exact search"). Membership is a syntax-and-type check over -//! the checked declaration: one binding of one node type, eligibility over -//! Boolean, integer and String properties, literals and parameters, an -//! optional leading `bm25()` over a String property, direct property sort -//! keys, a limit, and a projection of properties and the selected score. +//! the declaration's typed lowering: one binding of one node type, +//! eligibility over Boolean, integer and String properties, literals and +//! parameters (a literal or parameter cast to the property's integer type +//! included), an optional leading `bm25()` over a String property, direct +//! property sort keys, a limit, and a projection of properties and the +//! selected score. //! -//! For a member, the validator first checks that the IR is the lowering of -//! the declaration, then builds the canonical chain from it +//! For a member, the validator builds the canonical chain from the typed IR //! (`Limit(n, Sort(K, Project(V, [Search(a)] Filter(p, Scan(C)))))`, leaf //! first), applies the optimizer's recorded rule applications one by one, //! each checked against its precondition and reconstructed by the validator, @@ -20,16 +21,19 @@ use std::collections::BTreeSet; use omnigraph_compiler::catalog::NodeType; use omnigraph_compiler::ir::{IRExpr, IROp, IROrdering, IRProjection, QueryIR}; -use omnigraph_compiler::query::ast::{BinaryOp, Clause, CompOp, Expr, Literal}; -use omnigraph_compiler::types::{PropType, ScalarType}; +use omnigraph_compiler::query::ast::{BinaryOp, CompOp, Literal}; +use omnigraph_compiler::types::{ExprType, PropType, ScalarType}; use serde::{Deserialize, Serialize}; use super::budget::Budget; -use super::requirements::{Matcher, Position, Requirements}; +use super::requirements::Requirements; use super::{AcceptInput, ValidationError}; use crate::logical::{ColumnRef, IDENTITY_MEMBER, LogicalNode, LogicalPlan}; use crate::mirror::{ExprMirror, OrderingMirror, ProjectionMirror}; -use crate::physical::{Eligibility, PhysicalNode, PhysicalPlan, RankKind, RankScope, ScanInput}; +use crate::optimizer::ROW_ADDR; +use crate::physical::{ + Eligibility, HydratedColumn, PhysicalNode, PhysicalPlan, RankKind, RankScope, ScanInput, +}; use crate::source::FullTextCoverage; /// A member of the exact fragment: its one binding and node type. @@ -41,25 +45,39 @@ pub(crate) struct Member { /// The scalar types the fragment's comparisons and keys admit. fn admitted(property: &PropType) -> bool { - !property.list - && matches!( - property.scalar, - ScalarType::Bool - | ScalarType::I32 - | ScalarType::I64 - | ScalarType::U32 - | ScalarType::U64 - | ScalarType::String - ) + !property.list && admitted_scalar(property.scalar) +} + +fn admitted_scalar(scalar: ScalarType) -> bool { + matches!( + scalar, + ScalarType::Bool + | ScalarType::I32 + | ScalarType::I64 + | ScalarType::U32 + | ScalarType::U64 + | ScalarType::String + ) } -/// An operand's type as the fragment compares it: an integer literal -/// compares with any integer property, every other operand only with its -/// own scalar type. +/// The scalar of a non-list value type the fragment admits. +fn admitted_type(ty: &ExprType) -> Option { + match ty { + ExprType::Value { + scalar, + list: false, + .. + } if admitted_scalar(*scalar) => Some(*scalar), + _ => None, + } +} + +/// An operand's type as the fragment compares it. The lowering makes both +/// sides of a comparison one type (an explicit cast of a constant where +/// they differ), so every operand compares only with its own scalar type. #[derive(Debug, Clone, Copy, PartialEq, Eq)] enum Operand { Scalar(ScalarType), - IntegerLiteral, Null, } @@ -67,10 +85,7 @@ impl Operand { fn integer(self) -> bool { matches!( self, - Self::IntegerLiteral - | Self::Scalar( - ScalarType::I32 | ScalarType::I64 | ScalarType::U32 | ScalarType::U64 - ) + Self::Scalar(ScalarType::I32 | ScalarType::I64 | ScalarType::U32 | ScalarType::U64) ) } @@ -81,7 +96,6 @@ impl Operand { fn same_type(self, other: Self) -> bool { match (self, other) { (Self::Null, _) | (_, Self::Null) => true, - (Self::IntegerLiteral, other) | (other, Self::IntegerLiteral) => other.integer(), (left, right) => left == right, } } @@ -90,7 +104,6 @@ impl Operand { struct Fragment<'a> { binding: &'a str, node_type: &'a NodeType, - params: &'a [omnigraph_compiler::query::ast::Param], } impl Fragment<'_> { @@ -102,38 +115,44 @@ impl Fragment<'_> { admitted(declared).then_some(declared.scalar) } - fn operand(&self, expr: &Expr) -> Option { + fn operand(&self, expr: &IRExpr) -> Option { match expr { - Expr::PropAccess { variable, property } => { - self.property(variable, property).map(Operand::Scalar) - } - Expr::Literal(Literal::Bool(_)) => Some(Operand::Scalar(ScalarType::Bool)), - Expr::Literal(Literal::String(_)) => Some(Operand::Scalar(ScalarType::String)), - Expr::Literal(Literal::Integer(_)) => Some(Operand::IntegerLiteral), - Expr::Literal(Literal::Null) => Some(Operand::Null), - Expr::Variable(name) => { - let param = self.params.iter().find(|param| param.name == *name)?; - let declared = PropType::from_param_type_name(¶m.type_name, param.nullable)?; - admitted(&declared).then_some(Operand::Scalar(declared.scalar)) + IRExpr::PropAccess { + variable, property, .. + } => self.property(variable, property).map(Operand::Scalar), + IRExpr::Literal(Literal::Null, _) => Some(Operand::Null), + IRExpr::Literal(Literal::Bool(_) | Literal::String(_) | Literal::Integer(_), ty) + | IRExpr::Param(_, ty) => admitted_type(ty).map(Operand::Scalar), + // A constant cast to an admitted type: the scan translates the + // cast as the in-memory filter evaluates it. + IRExpr::Cast { expr, ty } + if matches!( + expr.as_ref(), + IRExpr::Literal(Literal::Integer(_), _) | IRExpr::Param(_, _) + ) => + { + admitted_type(ty).map(Operand::Scalar) } _ => None, } } /// Whether `expr` is an admitted eligibility predicate. - fn predicate(&self, expr: &Expr) -> bool { + fn predicate(&self, expr: &IRExpr) -> bool { match expr { - Expr::Binary { + IRExpr::Binary { left, op: BinaryOp::And | BinaryOp::Or, right, + .. } => self.predicate(left) && self.predicate(right), - Expr::Not(inner) => self.predicate(inner), - Expr::IsNull { expr, .. } => self.operand(expr).is_some(), - Expr::Binary { + IRExpr::Not(inner, _) => self.predicate(inner), + IRExpr::IsNull { expr, .. } => self.operand(expr).is_some(), + IRExpr::Binary { left, op: BinaryOp::Compare(op), right, + .. } => { let (Some(left), Some(right)) = (self.operand(left), self.operand(right)) else { return false; @@ -154,11 +173,14 @@ impl Fragment<'_> { /// Whether `expr` is the leading `bm25()` the fragment admits: over a /// String property of the binding, with a String literal or parameter. - fn bm25(&self, expr: &Expr) -> bool { - let Expr::Bm25 { field, query } = expr else { + fn bm25(&self, expr: &IRExpr) -> bool { + let IRExpr::Bm25 { field, query, .. } = expr else { return false; }; - let Expr::PropAccess { variable, property } = field.as_ref() else { + let IRExpr::PropAccess { + variable, property, .. + } = field.as_ref() + else { return false; }; self.property(variable, property) == Some(ScalarType::String) @@ -166,72 +188,72 @@ impl Fragment<'_> { } } -/// The member the checked declaration of `input` is, or `None` for a query +/// The member the typed lowering of `input` is, or `None` for a query /// outside the fragment. pub(crate) fn membership(input: &AcceptInput<'_>) -> Option { - let decl = input.checked.decl(); - let mut binding = None; - for clause in &decl.match_clause { - match clause { - Clause::Binding(found) if binding.is_none() => binding = Some(found), - Clause::Filter(_) => {} - _ => return None, - } - } - let binding = binding?; - let node_type = input.catalog.node_types.get(&binding.type_name)?; + let ir = input.ir; + let mut pipeline = ir.pipeline.iter(); + let Some(IROp::NodeScan { + variable, + type_name, + filters, + }) = pipeline.next() + else { + return None; + }; + let node_type = input.catalog.node_types.get(type_name)?; let fragment = Fragment { - binding: &binding.variable, + binding: variable, node_type, - params: &decl.params, }; - for matched in &binding.prop_matches { - let declared = node_type.properties.get(&matched.prop_name)?; - let comparison = Expr::comparison( - Expr::PropAccess { - variable: binding.variable.clone(), - property: matched.prop_name.clone(), - }, - CompOp::Eq, - matched.value.clone(), - ); - if !admitted(declared) || !fragment.predicate(&comparison) { - return None; - } + if !filters.iter().all(|filter| fragment.predicate(filter)) { + return None; } - for clause in &decl.match_clause { - if let Clause::Filter(filter) = clause - && !fragment.predicate(filter) - { - return None; + for op in pipeline { + match op { + IROp::Filter(filter) if fragment.predicate(filter) => {} + _ => return None, } } - let mut order = decl.order_clause.iter(); - let leading = decl.order_clause.first().map(|key| &key.expr); + let mut order = ir.order_by.iter(); + let leading = ir.order_by.first().map(|key| &key.expr); let ranked = leading.is_some_and(|lead| fragment.bm25(lead)); if ranked { order.next(); } for key in order { - let Expr::PropAccess { variable, property } = &key.expr else { + let IRExpr::PropAccess { + variable, property, .. + } = &key.expr + else { return None; }; fragment.property(variable, property)?; } - for projection in &decl.return_clause { + // The lowering makes a projected `bm25()` its score column; the + // declaration says which call it was. + let decl = input.checked.decl(); + let written_lead = decl.order_clause.first().map(|key| &key.expr); + for (index, projection) in ir.return_exprs.iter().enumerate() { let direct = match &projection.expr { - Expr::PropAccess { variable, property } => { - fragment.property(variable, property).is_some() + IRExpr::PropAccess { + variable, property, .. + } if fragment.property(variable, property).is_some() => true, + _ => { + ranked + && decl + .return_clause + .get(index) + .is_some_and(|written| Some(&written.expr) == written_lead) } - call => ranked && Some(call) == leading, }; if !direct { return None; } } Some(Member { - binding: binding.variable.clone(), - type_name: binding.type_name.clone(), + binding: variable.clone(), + type_name: type_name.clone(), }) } @@ -280,6 +302,13 @@ pub enum ChainNode { rows: usize, lowered: bool, }, + /// Return columns read by row address above the limit; physical from + /// its creation. + Hydrate { + binding: String, + type_key: String, + columns: Vec, + }, } /// A chain node's place, leaf first; a chain holds at most one of each. @@ -291,6 +320,7 @@ pub(crate) enum Role { Projection, Sort, Limit, + Hydrate, } impl ChainNode { @@ -302,6 +332,7 @@ impl ChainNode { Self::Projection { .. } => Role::Projection, Self::Sort { .. } => Role::Sort, Self::Limit { .. } => Role::Limit, + Self::Hydrate { .. } => Role::Hydrate, } } @@ -312,6 +343,7 @@ impl ChainNode { | Self::Projection { lowered, .. } | Self::Sort { lowered, .. } | Self::Limit { lowered, .. } => *lowered, + Self::Hydrate { .. } => true, Self::Search { .. } => false, } } @@ -323,7 +355,7 @@ impl ChainNode { | Self::Projection { lowered, .. } | Self::Sort { lowered, .. } | Self::Limit { lowered, .. } => *lowered = true, - Self::Search { .. } => {} + Self::Search { .. } | Self::Hydrate { .. } => {} } } } @@ -348,6 +380,13 @@ pub enum Rule { /// placing eligibility before scoring only under recorded full /// full-text coverage; `at` `[search, scan]`. RankBm25Scan { eligibility: Eligibility }, + /// The physical `Limit(n > 0, …Projection(V, …Scan(C)))` reads each of + /// `columns`, a bare property of the binding that `V` returns at its + /// position and no other node reads, by row address above the limit: + /// the scan reads `_rowaddr` in their place, and a hydration over the + /// limit places them; `at` `[scan]`, successors the scan and the + /// hydration. + HydrateReturnColumns { columns: Vec }, } /// One recorded rule application. @@ -504,6 +543,27 @@ impl Tracer { derivation.steps.push(Step { at, rule }); } + /// Record `rule` applied to the current nodes of `at`, whose successors + /// are the current nodes of `successors` (a role it creates included). + pub(crate) fn record_with(&mut self, at: &[Role], successors: &[Role], rule: Rule) { + let Some(derivation) = self.derivation.as_mut() else { + return; + }; + let mut indices = Vec::with_capacity(at.len()); + for role in at { + let Some(index) = self.current.get(role).copied() else { + self.derivation = None; + return; + }; + indices.push(index); + } + for role in successors { + self.current.insert(*role, self.next); + self.next += 1; + } + derivation.steps.push(Step { at: indices, rule }); + } + pub(crate) fn finish(self) -> Option { self.derivation } @@ -564,15 +624,15 @@ pub(crate) fn check( member: &Member, budget: &mut Budget, ) -> Result<(), ValidationError> { + let _ = requirements; let ir = input.ir; - let matcher = Matcher::new(input, requirements); - lowered_ir(ir, input, member, &matcher, budget)?; let mut arena = canonical(ir, plan, member, input)?; for step in &derivation.steps { budget.step()?; apply(&mut arena, step, plan, member, input, budget)?; } let reached: Vec<&ChainNode> = [ + Role::Hydrate, Role::Limit, Role::Sort, Role::Projection, @@ -607,110 +667,6 @@ pub(crate) fn check( Ok(()) } -/// Stage one: the IR is the lowering of the member's declaration, the -/// pattern the compiler defines for one binding. -fn lowered_ir( - ir: &QueryIR, - input: &AcceptInput<'_>, - member: &Member, - matcher: &Matcher<'_>, - budget: &mut Budget, -) -> Result<(), ValidationError> { - let decl = input.checked.decl(); - let mut pipeline = ir.pipeline.iter(); - let Some(IROp::NodeScan { - variable, - type_name, - filters, - }) = pipeline.next() - else { - return Err(invalid( - "the IR does not start with the binding's scan".into(), - )); - }; - if *variable != member.binding || *type_name != member.type_name { - return Err(invalid(format!( - "the IR scans `${variable}: {type_name}`; the query binds `${}: {}`", - member.binding, member.type_name - ))); - } - let binding = decl.match_clause.iter().find_map(|clause| match clause { - Clause::Binding(binding) => Some(binding), - _ => None, - }); - let matches = binding.map_or(&[][..], |binding| binding.prop_matches.as_slice()); - if filters.len() != matches.len() { - return Err(invalid( - "the IR scan's inline filters are not the binding's matches".into(), - )); - } - for (matched, filter) in matches.iter().zip(filters) { - let written = Expr::comparison( - Expr::PropAccess { - variable: member.binding.clone(), - property: matched.prop_name.clone(), - }, - CompOp::Eq, - matched.value.clone(), - ); - if !matcher.matches(&written, filter, Position::Filter, budget)? { - return Err(invalid(format!("the IR tests `{filter}` for `{written}`"))); - } - } - let clauses: Vec<&Expr> = decl - .match_clause - .iter() - .filter_map(|clause| match clause { - Clause::Filter(filter) => Some(filter), - _ => None, - }) - .collect(); - let rest: Vec<&IROp> = pipeline.collect(); - if rest.len() != clauses.len() { - return Err(invalid( - "the IR's filters are not the query's clauses".into(), - )); - } - for (clause, op) in clauses.iter().zip(rest) { - let IROp::Filter(filter) = op else { - return Err(invalid("the IR holds an op the fragment does not".into())); - }; - if !matcher.matches(clause, filter, Position::Filter, budget)? { - return Err(invalid(format!("the IR tests `{filter}` for `{clause}`"))); - } - } - if ir.return_exprs.len() != decl.return_clause.len() { - return Err(invalid("the IR's return is not the query's".into())); - } - for (written, lowered) in decl.return_clause.iter().zip(&ir.return_exprs) { - if written.alias != lowered.alias - || !matcher.matches(&written.expr, &lowered.expr, Position::Return, budget)? - { - return Err(invalid(format!( - "the IR returns `{}`; the query returns `{}`", - lowered.expr, written.expr - ))); - } - } - if ir.order_by.len() != decl.order_clause.len() { - return Err(invalid("the IR's order is not the query's".into())); - } - for (written, lowered) in decl.order_clause.iter().zip(&ir.order_by) { - if written.descending != lowered.descending - || !matcher.matches(&written.expr, &lowered.expr, Position::Plain, budget)? - { - return Err(invalid(format!( - "the IR orders by `{}`; the query by `{}`", - lowered.expr, written.expr - ))); - } - } - if ir.limit != decl.limit { - return Err(invalid("the IR's limit is not the query's".into())); - } - Ok(()) -} - /// The canonical chain of a member, from its checked IR: the scan of the /// pinned table, every conjunct in written order (a repeated one once), the /// leading `bm25()`, the projection, the sort of the remaining keys with the @@ -739,7 +695,7 @@ fn canonical( } let mut keys: &[IROrdering] = &ir.order_by; let search = match keys.first().map(|key| &key.expr) { - Some(IRExpr::Bm25 { field, query }) => match field.as_ref() { + Some(IRExpr::Bm25 { field, query, .. }) => match field.as_ref() { IRExpr::PropAccess { property, .. } => { keys = &keys[1..]; Some(ChainNode::Search { @@ -775,7 +731,7 @@ fn canonical( if sorted { let written: Vec<&IROrdering> = keys .iter() - .filter(|key| !matches!(key.expr, IRExpr::Literal(_))) + .filter(|key| !matches!(key.expr, IRExpr::Literal(_, _))) .collect(); nodes.push(ChainNode::Sort { keys: written @@ -814,7 +770,7 @@ fn canonical_tiebreak(ir: &QueryIR, member: &Member, input: &AcceptInput<'_>) -> .all(|projection| keys.contains(&projection.expr.to_string())); let id = input.catalog.system_columns.id; let keyed = ir.order_by.iter().any(|key| { - matches!(&key.expr, IRExpr::PropAccess { variable, property } + matches!(&key.expr, IRExpr::PropAccess { variable, property, .. } if *variable == member.binding && property == id) }); if covered || keyed { @@ -992,15 +948,11 @@ fn apply( _ => None, }); if ranked.is_some() { - let (column, descending) = RankKind::Bm25.score(); keys.insert( 0, OrderingMirror::from(&IROrdering { - expr: IRExpr::PropAccess { - variable: member.binding.clone(), - property: column.to_string(), - }, - descending, + expr: crate::optimizer::rank_score(&member.binding, RankKind::Bm25), + descending: RankKind::Bm25.score().1, }), ); } @@ -1051,6 +1003,113 @@ fn apply( }); vec![(Role::Search, None), (Role::Scan, Some(node))] } + (Rule::HydrateReturnColumns { columns }, [scan]) => { + let mut node = arena.node(*scan, Role::Scan)?.clone(); + let ChainNode::Scan { + type_key, + filter, + projection: Some(read), + ranked: None, + lowered: true, + .. + } = &mut node + else { + return Err(invalid( + "only a physical, unranked scan with a projection defers columns".into(), + )); + }; + let current = |role: Role| { + arena + .current + .get(&role) + .and_then(|index| arena.nodes[*index].as_ref()) + }; + if current(Role::Hydrate).is_some() + || !matches!(current(Role::Limit), Some(ChainNode::Limit { rows, lowered: true }) if *rows > 0) + { + return Err(invalid( + "return columns are hydrated once, above a nonzero physical limit".into(), + )); + } + let Some(ChainNode::Projection { + exprs, + lowered: true, + }) = current(Role::Projection) + else { + return Err(invalid( + "return columns are hydrated over a physical projection".into(), + )); + }; + let returns: Vec = + exprs.iter().cloned().map(IRProjection::from).collect(); + let node_type = input + .catalog + .node_types + .get(&member.type_name) + .ok_or_else(|| invalid("the member's type left the catalog".into()))?; + budget.visit(u64::try_from(columns.len() + returns.len()).unwrap_or(u64::MAX))?; + // Every read below the hydration that must still come from the + // scan: filters, sort keys, and every return item not deferred. + let mut reads = Vec::new(); + for conjunct in filter.iter() { + crate::optimizer::reads_of_expr(&IRExpr::from(conjunct.clone()), &mut reads); + } + if let Some(ChainNode::Filter { conjuncts, .. }) = current(Role::Filter) { + for conjunct in conjuncts { + crate::optimizer::reads_of_expr(&IRExpr::from(conjunct.clone()), &mut reads); + } + } + if let Some(ChainNode::Sort { keys, .. }) = current(Role::Sort) { + for key in keys { + crate::optimizer::reads_of_expr( + &IROrdering::from(key.clone()).expr, + &mut reads, + ); + } + } + for (position, item) in returns.iter().enumerate() { + if !columns.iter().any(|column| column.position == position) { + crate::optimizer::reads_of_expr(&item.expr, &mut reads); + } + } + for column in columns { + let returned = returns.get(column.position).is_some_and(|item| { + item.column == column.output + && matches!(&item.expr, IRExpr::PropAccess { variable, property, .. } + if *variable == member.binding && *property == column.property) + }); + let read_below = reads.iter().any(|read| { + read.binding == member.binding + && read.property.as_deref() == Some(column.property.as_str()) + }); + if !returned + || read_below + || !read.contains(&column.property) + || node_type + .key + .as_deref() + .unwrap_or_default() + .iter() + .any(|key| *key == column.property) + || column.property == input.catalog.system_columns.id + { + return Err(invalid(format!( + "`${}.{}` at return position {} cannot be read by row address", + member.binding, column.property, column.position + ))); + } + } + read.retain(|name| !columns.iter().any(|column| column.property == *name)); + if !read.iter().any(|name| name == ROW_ADDR) { + read.push(ROW_ADDR.to_string()); + } + let hydrate = ChainNode::Hydrate { + binding: member.binding.clone(), + type_key: type_key.clone(), + columns: columns.clone(), + }; + vec![(Role::Scan, Some(node)), (Role::Hydrate, Some(hydrate))] + } (rule, at) => { return Err(invalid(format!("rule {rule:?} names {} nodes", at.len()))); } @@ -1077,21 +1136,32 @@ fn apply( /// Whether the scan evaluates `conjunct` with the same typed semantics as /// the in-memory filter: the fragment's predicate forms over the member's -/// properties, literals and bound parameters, with no cast. +/// properties, literals and bound parameters, a constant cast included (the +/// scan translates a cast as the filter evaluates it). fn translatable(conjunct: &IRExpr, member: &Member, input: &AcceptInput<'_>) -> bool { let Some(node_type) = input.catalog.node_types.get(&member.type_name) else { return false; }; fn operand(expr: &IRExpr, member: &Member, node_type: &NodeType) -> bool { match expr { - IRExpr::PropAccess { variable, property } => { + IRExpr::PropAccess { + variable, property, .. + } => { *variable == member.binding && node_type.properties.get(property).is_some_and(admitted) } IRExpr::Literal( Literal::Bool(_) | Literal::String(_) | Literal::Integer(_) | Literal::Null, + _, ) - | IRExpr::Param(_) => true, + | IRExpr::Param(_, _) => true, + IRExpr::Cast { expr, ty } => { + admitted_type(ty).is_some() + && matches!( + expr.as_ref(), + IRExpr::Literal(Literal::Integer(_), _) | IRExpr::Param(_, _) + ) + } _ => false, } } @@ -1100,8 +1170,9 @@ fn translatable(conjunct: &IRExpr, member: &Member, input: &AcceptInput<'_>) -> left, op: BinaryOp::And | BinaryOp::Or, right, + .. } => translatable(left, member, input) && translatable(right, member, input), - IRExpr::Not(inner) => translatable(inner, member, input), + IRExpr::Not(inner, _) => translatable(inner, member, input), IRExpr::IsNull { expr, .. } => operand(expr, member, node_type), IRExpr::Binary { left, @@ -1110,6 +1181,7 @@ fn translatable(conjunct: &IRExpr, member: &Member, input: &AcceptInput<'_>) -> CompOp::Eq | CompOp::Ne | CompOp::Lt | CompOp::Le | CompOp::Gt | CompOp::Ge, ), right, + .. } => operand(left, member, node_type) && operand(right, member, node_type), other => operand(other, member, node_type), } @@ -1122,11 +1194,26 @@ fn chain_of( ) -> Result, ValidationError> { let mut chain = Vec::new(); let mut id = plan.root(); + let mut hydrated_table = None; loop { let node = plan .node(id) .ok_or_else(|| invalid(format!("node {id} is a tombstone")))?; let (converted, next) = match node { + PhysicalNode::HydrateColumns { input, bindings } if chain.is_empty() => { + let [hydrated] = bindings.as_slice() else { + return Err(invalid("a chain hydrates one binding".into())); + }; + hydrated_table = Some(&hydrated.table); + ( + ChainNode::Hydrate { + binding: hydrated.binding.clone(), + type_key: hydrated.table.type_key.clone(), + columns: hydrated.columns.clone(), + }, + Some(*input), + ) + } PhysicalNode::Limit { input, rows } => ( ChainNode::Limit { rows: *rows, @@ -1151,6 +1238,7 @@ fn chain_of( PhysicalNode::Projection { input, return_exprs, + .. } => ( ChainNode::Projection { exprs: return_exprs.iter().map(ProjectionMirror::from).collect(), @@ -1181,6 +1269,7 @@ fn chain_of( None => false, }; if !same_table + || hydrated_table.is_some_and(|table| *table != spec.table) || spec.fragments.is_some() || spec.runtime_filter.is_some() || spec.columns != input.catalog.system_columns diff --git a/crates/omnigraph-planner/src/validate/tests.rs b/crates/omnigraph-planner/src/validate/tests.rs index 320e1a5f9..bb9423e5e 100644 --- a/crates/omnigraph-planner/src/validate/tests.rs +++ b/crates/omnigraph-planner/src/validate/tests.rs @@ -7,7 +7,7 @@ use std::sync::Arc; use omnigraph_compiler::CheckedQuery; use omnigraph_compiler::catalog::{Catalog, build_catalog}; -use omnigraph_compiler::ir::{IRExpr, ParamMap, QueryIR, fold}; +use omnigraph_compiler::ir::{IRExpr, ParamMap, QueryIR}; use omnigraph_compiler::query::ast::Literal; use omnigraph_compiler::schema::parser::parse_schema; @@ -33,26 +33,19 @@ node Doc { edge Cites: Doc -> Doc "#; -/// The bound-literal evaluator: literals and bound parameters through the -/// compiler's fold rules. -struct Bound<'p>(&'p ParamMap); - -impl ConstantEvaluator for Bound<'_> { - fn evaluate(&self, expr: &IRExpr) -> Option { - match expr { - IRExpr::Literal(literal) => Some(literal.clone()), - IRExpr::Param(name) => self.0.get(name).cloned(), - IRExpr::Binary { left, op, right } => { - fold::evaluate(*op, &self.evaluate(left)?, &self.evaluate(right)?) - } - IRExpr::Not(inner) => match self.evaluate(inner)? { - Literal::Bool(value) => Some(Literal::Bool(!value)), - Literal::Null => Some(Literal::Null), - _ => None, - }, - IRExpr::IsNull { expr, negated } => Some(Literal::Bool( - matches!(self.evaluate(expr)?, Literal::Null) != *negated, - )), +/// The canonical id of a `Doc` by its String key, as a stand-in for the +/// loader's derivation: `id:`. +struct Keys; + +impl KeyIds for Keys { + fn canonical_key_id( + &self, + type_key: &str, + value: &Literal, + _ty: &omnigraph_compiler::types::ExprType, + ) -> Option { + match (type_key, value) { + ("node:Doc", Literal::String(key)) => Some(format!("id:{key}")), _ => None, } } @@ -93,6 +86,7 @@ impl Fixture { .iter() .map(|field| field.name().clone()) .collect(), + object_fields: node_type.node_object_fields().cloned().collect(), row_count: None, }, ); @@ -130,13 +124,12 @@ impl Fixture { plan: PhysicalPlan, derivation: Option, ) -> Result { - let constants = Bound(&self.params); let input = AcceptInput { checked: &self.checked, catalog: &self.catalog, ir: &self.ir, params: &self.params, - constants: &constants, + keys: &Keys, limits: ValidationLimits::DEFAULT, }; accept(plan, &input, derivation) @@ -213,6 +206,7 @@ fn rules(derivation: &Derivation) -> Vec<&'static str> { Rule::PruneScanColumns { .. } => "prune", Rule::Lower => "lower", Rule::RankBm25Scan { .. } => "rank", + Rule::HydrateReturnColumns { .. } => "hydrate", }) .collect() } @@ -351,7 +345,8 @@ fn another_query_argument_fails_search_identity() { .. }) = plan.node_mut(id) { - ranked.query = IRExpr::Literal(Literal::String("other".into())); + ranked.query = + IRExpr::Literal(Literal::String("other".into()), ranked.query.ty().clone()); } } let (check, detail) = fixture.refused(plan); @@ -559,6 +554,7 @@ fn a_score_read_from_another_column_fails_the_projection() { return_exprs[1].expr = IRExpr::PropAccess { variable: "d".to_string(), property: "_distance".to_string(), + ty: return_exprs[1].expr.ty().clone(), }; } } @@ -569,13 +565,12 @@ fn a_score_read_from_another_column_fails_the_projection() { #[test] fn an_exhausted_budget_is_a_resource_outcome() { let fixture = ranked(); - let constants = Bound(&fixture.params); let input = AcceptInput { checked: &fixture.checked, catalog: &fixture.catalog, ir: &fixture.ir, params: &fixture.params, - constants: &constants, + keys: &Keys, limits: ValidationLimits { work: 3, ..ValidationLimits::DEFAULT @@ -774,17 +769,24 @@ fn both_acceptance_entries_agree_even_when_a_limit_runs_out() { }, ] { let fixture = ranked(); - let constants = Bound(&fixture.params); let input = AcceptInput { checked: &fixture.checked, catalog: &fixture.catalog, ir: &fixture.ir, params: &fixture.params, - constants: &constants, + keys: &Keys, limits, }; - let plain = crate::gate::accept_query(&input, &fixture.source, &BOUNDS); - let explained = crate::gate::accept_query_explained(&input, &fixture.source, &BOUNDS); + let plain = futures::executor::block_on(crate::gate::accept_query( + &input, + &fixture.source, + &BOUNDS, + )); + let explained = futures::executor::block_on(crate::gate::accept_query_explained( + &input, + &fixture.source, + &BOUNDS, + )); match (plain, explained) { (Ok(plain), Ok((explained, explain))) => { assert_eq!(plain.plan(), explained.plan()); @@ -821,13 +823,12 @@ fn derivation_cost_grows_with_the_query() { let traced = fixture.traced(); let derivation = traced.derivation.clone().unwrap(); let bytes = serde_json::to_vec(&derivation).unwrap().len(); - let constants = Bound(&fixture.params); let input = AcceptInput { checked: &fixture.checked, catalog: &fixture.catalog, ir: &fixture.ir, params: &fixture.params, - constants: &constants, + keys: &Keys, limits: ValidationLimits::DEFAULT, }; let mut budget = Budget::new(ValidationLimits::DEFAULT); From 256dceb5f85c05465607d2a051d7ec03b5efad0c Mon Sep 17 00:00:00 2001 From: Ragnor Comerford Date: Sat, 10 Oct 2026 21:30:54 -0700 Subject: [PATCH 23/29] planner: record key narrowing as a checked rule; test both-sided eligibility Key narrowing runs after the derivation is recorded, so an exact-fragment member like `Doc { slug: $s }` on a table with an id index would carry a predicate its derivation did not explain and fail acceptance as a planner defect. finalize_scan_access now returns the identity conjuncts it adds, the planned run records a NarrowKey step, and the subset checker re-derives the id through the same function the invariants use. Tests: an unwritten conjunct fails predicate origin; a planned narrowing is accepted as exact subset while a forged id or an identity without its written key equality is refused; edited hop bounds fail traversal identity; fused and grouped searches sort as the query requires; a hydrated member keeps its exact subset and a misplaced hydration is refused. --- crates/omnigraph-planner/src/gate.rs | 33 ++- crates/omnigraph-planner/src/scan_access.rs | 26 +- crates/omnigraph-planner/src/source.rs | 27 ++ .../src/validate/invariants.rs | 85 +----- .../src/validate/requirements.rs | 72 +++++ .../omnigraph-planner/src/validate/subset.rs | 34 +++ .../omnigraph-planner/src/validate/tests.rs | 248 +++++++++++++++++- 7 files changed, 430 insertions(+), 95 deletions(-) diff --git a/crates/omnigraph-planner/src/gate.rs b/crates/omnigraph-planner/src/gate.rs index bdd84722c..8fce79902 100644 --- a/crates/omnigraph-planner/src/gate.rs +++ b/crates/omnigraph-planner/src/gate.rs @@ -27,7 +27,7 @@ use crate::source::{ AdjacencyProof, EXPAND_INDEXED_MAX_FRONTIER_ENV, EXPAND_INDEXED_MAX_HOPS_ENV, ExpandStatistics, FragmentStat, FullTextCoverage, IndexFact, NodeTypeSpec, PlanSource, SideId, }; -use crate::validate::subset::{Derivation, Tracer}; +use crate::validate::subset::{Role, Rule, Tracer}; use crate::validate::{self, AcceptInput, AcceptedPlan, ValidationError}; /// A `PlanSource` that records what the planner read through it: the @@ -466,7 +466,15 @@ pub async fn accept_query_explained( pub(crate) struct Traced { pub logical: LogicalPlan, pub optimized: Optimized, - pub derivation: Option, + pub tracer: Tracer, +} + +impl Traced { + /// The derivation recorded so far. + #[cfg(test)] + pub(crate) fn derivation(&self) -> Option { + self.tracer.derivation() + } } pub(crate) fn plan_traced( @@ -489,7 +497,7 @@ pub(crate) fn plan_traced( Ok(Traced { logical, optimized, - derivation: tracer.finish(), + tracer, }) } @@ -503,13 +511,24 @@ async fn plan_and_accept( let Traced { logical, mut optimized, - derivation, + mut tracer, } = plan_traced(input.ir, source, bounds)?; - // Scan access is part of the plan acceptance checks: a probe or a key - // narrowing it adds is validated like any other planned predicate. - crate::scan_access::finalize_scan_access(&mut optimized, source) + // Scan access is part of the plan acceptance checks: a key narrowing it + // adds is a recorded rule of an exact-fragment chain, and a predicate + // the invariants re-derive. + let narrowed = crate::scan_access::finalize_scan_access(&mut optimized, source) .await .map_err(Unrouted::of)?; + if let [conjunct] = narrowed.as_slice() { + tracer.record( + &[Role::Scan], + Rule::NarrowKey { + conjunct: crate::mirror::ExprMirror::from(conjunct), + }, + &[], + ); + } + let derivation = tracer.finish(); let rendered = explain.then(|| { LogicalView::of(&logical) .explain( diff --git a/crates/omnigraph-planner/src/scan_access.rs b/crates/omnigraph-planner/src/scan_access.rs index 6df6de075..275e2ccd0 100644 --- a/crates/omnigraph-planner/src/scan_access.rs +++ b/crates/omnigraph-planner/src/scan_access.rs @@ -11,16 +11,18 @@ use crate::source::{IndexKind, PlanSource}; pub const PASS_SCAN_ACCESS: &str = "scan_access"; pub const PASS_KEY_TO_ID: &str = "key_to_id"; +/// Record every fresh scan's access, and return the identity conjuncts the +/// key narrowing added (one per narrowed scan), which acceptance re-derives. pub async fn finalize_scan_access( optimized: &mut Optimized, source: &(dyn PlanSource + Sync), -) -> Result<(), PlanError> { +) -> Result, PlanError> { let scans: Vec<_> = optimized .physical .live() .filter_map(|(id, node)| matches!(node, PhysicalNode::Scan { .. }).then_some(id)) .collect(); - let mut rewrote_key = false; + let mut narrowed = Vec::new(); for id in &scans { let Some(PhysicalNode::Scan { source: input, @@ -62,34 +64,34 @@ pub async fn finalize_scan_access( fact.column == spec.columns.id && matches!(fact.kind, IndexKind::Btree { usable: true }) }) { - rewrote_key |= narrow_key(spec, source)?; + narrowed.extend(narrow_key(spec, source)?); } source.index_split(spec).await? }; spec.access = Some(access); } - if rewrote_key { + if !narrowed.is_empty() { optimized.fired.push(PASS_KEY_TO_ID); } if !scans.is_empty() { optimized.fired.push(PASS_SCAN_ACCESS); } - Ok(()) + Ok(narrowed) } -fn narrow_key(spec: &mut ScanSpec, source: &dyn PlanSource) -> Result { +fn narrow_key(spec: &mut ScanSpec, source: &dyn PlanSource) -> Result, PlanError> { let Some(type_name) = spec.table.type_key.strip_prefix("node:") else { - return Ok(false); + return Ok(None); }; let node = source.node_type(type_name)?; let [key] = node.key.as_slice() else { - return Ok(false); + return Ok(None); }; let Some(binding) = spec.binding.as_ref() else { - return Ok(false); + return Ok(None); }; let Some(original) = spec.filter.as_ref() else { - return Ok(false); + return Ok(None); }; for conjunct in original .gq_filters() @@ -134,10 +136,10 @@ fn narrow_key(spec: &mut ScanSpec, source: &dyn PlanSource) -> Result bool { diff --git a/crates/omnigraph-planner/src/source.rs b/crates/omnigraph-planner/src/source.rs index 50e8ffbf0..f83de18c3 100644 --- a/crates/omnigraph-planner/src/source.rs +++ b/crates/omnigraph-planner/src/source.rs @@ -298,6 +298,9 @@ pub struct MemorySource { column_data_bytes: HashMap>, query_memory_pool_bytes: u64, full_text: HashMap<(String, String), FullTextCoverage>, + /// When set, a key's canonical id renders as `id:`, a parameter + /// resolved from these values. + canonical_keys: Option, } impl MemorySource { @@ -312,6 +315,11 @@ impl MemorySource { self } + pub fn with_canonical_key_ids(mut self, params: omnigraph_compiler::ir::ParamMap) -> Self { + self.canonical_keys = Some(params); + self + } + pub fn with_index_facts(mut self, dataset_key: &str, facts: Vec) -> Self { self.index_facts.insert(dataset_key.to_string(), facts); self @@ -395,6 +403,25 @@ impl MemorySource { } impl PlanSource for MemorySource { + fn canonical_key_id( + &self, + _type_key: &str, + value: &IRExpr, + ) -> Result, PlanError> { + let Some(params) = &self.canonical_keys else { + return Ok(None); + }; + let literal = match value { + IRExpr::Literal(literal, _) => Some(literal), + IRExpr::Param(name, _) => params.get(name), + _ => None, + }; + Ok(match literal { + Some(omnigraph_compiler::query::ast::Literal::String(key)) => Some(format!("id:{key}")), + _ => None, + }) + } + fn index_facts(&self, dataset_key: &str) -> Vec { self.index_facts .get(dataset_key) diff --git a/crates/omnigraph-planner/src/validate/invariants.rs b/crates/omnigraph-planner/src/validate/invariants.rs index e9eed8ef8..b63055e64 100644 --- a/crates/omnigraph-planner/src/validate/invariants.rs +++ b/crates/omnigraph-planner/src/validate/invariants.rs @@ -17,7 +17,7 @@ use std::collections::HashSet; use omnigraph_compiler::ir::{IRExpr, IROrdering, IRProjection}; -use omnigraph_compiler::query::ast::{CompOp, Expr, Literal, RRF_COLUMN}; +use omnigraph_compiler::query::ast::{Expr, RRF_COLUMN}; use omnigraph_compiler::query::typecheck::BoundVariable; use omnigraph_compiler::types::{ExprType, ScalarType}; @@ -135,16 +135,6 @@ fn ranked_scans(plan: &PhysicalPlan) -> Vec<(NodeId, &str, &RankedAccess)> { .collect() } -/// The type `scan_access::narrow_key` gives the identity column and its -/// value: a non-null String. -fn narrowed_type() -> ExprType { - ExprType::Value { - scalar: ScalarType::String, - list: false, - nullable: false, - } -} - impl Requirements { pub(crate) fn check( &self, @@ -308,9 +298,8 @@ impl Requirements { } /// Whether `conjunct`, placed on the scan `id`, is the identity equality - /// `scan_access::narrow_key` adds for a written equality on the scanned - /// type's single String key: the binding's id column equal, as a - /// non-null String, to the id the loader derives from the written value. + /// the key narrowing adds for a written key equality on the same scan + /// ([`super::requirements::narrowed_identity`]). fn narrowed_key( &self, plan: &PhysicalPlan, @@ -322,70 +311,22 @@ impl Requirements { let Some(PhysicalNode::Scan { spec, .. }) = plan.node(id) else { return false; }; - let (Some(binding), Some(type_name)) = - (spec.binding.as_deref(), spec.table.node_type_name()) - else { - return false; - }; - let Some(( - IRExpr::PropAccess { - variable, - property, - ty, - }, - CompOp::Eq, - IRExpr::Literal(Literal::String(narrowed), literal_ty), - )) = conjunct.comparison_parts() - else { - return false; - }; - if variable != binding - || property != input.catalog.system_columns.id - || *ty != narrowed_type() - || *literal_ty != narrowed_type() - { - return false; - } - let Some([key]) = input - .catalog - .node_types - .get(type_name) - .and_then(|node| node.key.as_deref()) - else { - return false; - }; - let Some(filter) = &spec.filter else { + let (Some(binding), Some(filter)) = (spec.binding.as_deref(), spec.filter.as_ref()) else { return false; }; - filter + let written: Vec = filter .gq_filters() .into_iter() .flat_map(IRExpr::into_conjuncts) .filter(|written| required.contains(written)) - .any(|written| { - let Some((left, CompOp::Eq, right)) = written.comparison_parts() else { - return false; - }; - [(left, right), (right, left)] - .into_iter() - .any(|(side, value)| { - let keyed = matches!(side, IRExpr::PropAccess { variable, property, .. } - if variable == binding && property == key); - let literal = match value { - IRExpr::Literal(literal, _) => Some(literal), - IRExpr::Param(name, _) => input.params.get(name), - _ => None, - }; - keyed - && literal.is_some_and(|literal| { - input - .keys - .canonical_key_id(&spec.table.type_key, literal, value.ty()) - .as_deref() - == Some(narrowed.as_str()) - }) - }) - }) + .collect(); + super::requirements::narrowed_identity( + input, + &spec.table.type_key, + binding, + &written, + conjunct, + ) } /// The correlated blocks of each pipeline the query's rows come from are diff --git a/crates/omnigraph-planner/src/validate/requirements.rs b/crates/omnigraph-planner/src/validate/requirements.rs index ab6233e84..b081b5dbf 100644 --- a/crates/omnigraph-planner/src/validate/requirements.rs +++ b/crates/omnigraph-planner/src/validate/requirements.rs @@ -317,6 +317,78 @@ impl Requirements { } } +/// Whether `derived` is the identity equality the key narrowing adds to a +/// scan of `type_key` bound to `binding` whose conjuncts include `written`: +/// the binding's id column equal, as a non-null String, to the id the +/// loader derives from the value of a written equality on the type's single +/// String key. The derivation goes through `AcceptInput::keys`, so a +/// narrowed id is re-derived, never trusted. +pub(crate) fn narrowed_identity<'w>( + input: &AcceptInput<'_>, + type_key: &str, + binding: &str, + written: impl IntoIterator, + derived: &IRExpr, +) -> bool { + let narrowed_type = ExprType::Value { + scalar: omnigraph_compiler::types::ScalarType::String, + list: false, + nullable: false, + }; + let Some(( + IRExpr::PropAccess { + variable, + property, + ty, + }, + omnigraph_compiler::query::ast::CompOp::Eq, + IRExpr::Literal(Literal::String(id), literal_ty), + )) = derived.comparison_parts() + else { + return false; + }; + if variable != binding + || property != input.catalog.system_columns.id + || *ty != narrowed_type + || *literal_ty != narrowed_type + { + return false; + } + let Some([key]) = type_key + .strip_prefix("node:") + .and_then(|name| input.catalog.node_types.get(name)) + .and_then(|node| node.key.as_deref()) + else { + return false; + }; + written.into_iter().any(|conjunct| { + let Some((left, omnigraph_compiler::query::ast::CompOp::Eq, right)) = + conjunct.comparison_parts() + else { + return false; + }; + [(left, right), (right, left)] + .into_iter() + .any(|(side, value)| { + let keyed = matches!(side, IRExpr::PropAccess { variable, property, .. } + if variable == binding && property == key); + let literal = match value { + IRExpr::Literal(literal, _) => Some(literal), + IRExpr::Param(name, _) => input.params.get(name), + _ => None, + }; + keyed + && literal.is_some_and(|literal| { + input + .keys + .canonical_key_id(type_key, literal, value.ty()) + .as_deref() + == Some(id.as_str()) + }) + }) + }) +} + /// `filters` split at their top-level `and`, a constant `true` left out. fn push_conjuncts(out: &mut Vec, filters: &[IRExpr]) { for conjunct in filters.iter().cloned().flat_map(IRExpr::into_conjuncts) { diff --git a/crates/omnigraph-planner/src/validate/subset.rs b/crates/omnigraph-planner/src/validate/subset.rs index 9c81fb052..620eace0d 100644 --- a/crates/omnigraph-planner/src/validate/subset.rs +++ b/crates/omnigraph-planner/src/validate/subset.rs @@ -387,6 +387,10 @@ pub enum Rule { /// limit places them; `at` `[scan]`, successors the scan and the /// hydration. HydrateReturnColumns { columns: Vec }, + /// The physical scan also tests `conjunct`, the canonical identity of a + /// written equality on its type's single String key, which the + /// validator re-derives; `at` `[scan]`. + NarrowKey { conjunct: ExprMirror }, } /// One recorded rule application. @@ -564,6 +568,12 @@ impl Tracer { derivation.steps.push(Step { at: indices, rule }); } + /// The derivation recorded so far. + #[cfg(test)] + pub(crate) fn derivation(&self) -> Option { + self.derivation.clone() + } + pub(crate) fn finish(self) -> Option { self.derivation } @@ -1110,6 +1120,30 @@ fn apply( }; vec![(Role::Scan, Some(node)), (Role::Hydrate, Some(hydrate))] } + (Rule::NarrowKey { conjunct }, [scan]) => { + let mut node = arena.node(*scan, Role::Scan)?.clone(); + let ChainNode::Scan { + type_key, + binding, + filter, + lowered: true, + .. + } = &mut node + else { + return Err(invalid("only a physical scan narrows its key".into())); + }; + let derived = IRExpr::from(conjunct.clone()); + let written: Vec = filter.iter().cloned().map(IRExpr::from).collect(); + budget.visit(u64::try_from(written.len()).unwrap_or(u64::MAX))?; + if !super::requirements::narrowed_identity(input, type_key, binding, &written, &derived) + { + return Err(invalid(format!( + "`{derived}` is no canonical identity of a written key equality" + ))); + } + filter.push(conjunct.clone()); + vec![(Role::Scan, Some(node))] + } (rule, at) => { return Err(invalid(format!("rule {rule:?} names {} nodes", at.len()))); } diff --git a/crates/omnigraph-planner/src/validate/tests.rs b/crates/omnigraph-planner/src/validate/tests.rs index bb9423e5e..fa3e1e6a9 100644 --- a/crates/omnigraph-planner/src/validate/tests.rs +++ b/crates/omnigraph-planner/src/validate/tests.rs @@ -112,13 +112,24 @@ impl Fixture { } fn derivation(&self) -> Option { - self.traced().derivation + self.traced().derivation() } fn accept(&self, plan: PhysicalPlan) -> Result { self.accept_with(plan, self.derivation()) } + fn input(&self) -> AcceptInput<'_> { + AcceptInput { + checked: &self.checked, + catalog: &self.catalog, + ir: &self.ir, + params: &self.params, + keys: &Keys, + limits: ValidationLimits::DEFAULT, + } + } + fn accept_with( &self, plan: PhysicalPlan, @@ -207,6 +218,7 @@ fn rules(derivation: &Derivation) -> Vec<&'static str> { Rule::Lower => "lower", Rule::RankBm25Scan { .. } => "rank", Rule::HydrateReturnColumns { .. } => "hydrate", + Rule::NarrowKey { .. } => "narrow", }) .collect() } @@ -286,9 +298,10 @@ fn a_bm25_rank_before_scoring_needs_full_coverage() { ); let traced = fixture.traced(); let accepted = fixture - .accept_with(traced.optimized.physical.clone(), traced.derivation.clone()) + .accept_with(traced.optimized.physical.clone(), traced.derivation()) .expect("full coverage admits filtering before scoring"); assert_eq!(accepted.scope(), ValidationScope::ExactSubset); + let derivation = traced.derivation(); let mut plan = traced.optimized.physical; let mut assumptions = plan.assumptions().clone(); assumptions.full_text.insert( @@ -296,7 +309,7 @@ fn a_bm25_rank_before_scoring_needs_full_coverage() { crate::source::FullTextCoverage::Partial, ); plan.set_assumptions(assumptions); - match fixture.accept_with(plan, traced.derivation) { + match fixture.accept_with(plan, derivation) { Err(ValidationError::Violated { check, detail }) => { assert_eq!(check, "prerequisite", "{detail}"); } @@ -322,6 +335,233 @@ fn a_dropped_conjunct_fails_predicate_retention() { assert_eq!(check, "predicate retention", "{detail}"); } +fn value_type( + scalar: omnigraph_compiler::types::ScalarType, +) -> omnigraph_compiler::types::ExprType { + omnigraph_compiler::types::ExprType::Value { + scalar, + list: false, + nullable: false, + } +} + +/// `conjunct` appended to the filter of every scan of `plan`. +fn place_on_scans(plan: &mut PhysicalPlan, conjunct: &IRExpr) { + for id in node_ids(plan, |node| matches!(node, PhysicalNode::Scan { .. })) { + if let Some(PhysicalNode::Scan { spec, .. }) = plan.node_mut(id) { + let added = crate::optimizer::gq_conjunct(conjunct); + spec.filter = Some(match spec.filter.take() { + Some(filter) => filter.and(added), + None => added, + }); + } + } +} + +#[test] +fn an_unwritten_conjunct_fails_predicate_origin() { + let fixture = filtered(); + let mut plan = fixture.plan(); + place_on_scans( + &mut plan, + &IRExpr::Literal( + Literal::Bool(false), + value_type(omnigraph_compiler::types::ScalarType::Bool), + ), + ); + let (check, detail) = fixture.refused(plan); + assert_eq!(check, "predicate origin", "{detail}"); +} + +const KEYED: &str = r#"query q($s: String) { + match { $d: Doc { slug: $s } } + return { $d.title } +}"#; + +/// The identity equality a key narrowing adds, with `id`. +fn identity(catalog: &Catalog, id: &str) -> IRExpr { + let ty = value_type(omnigraph_compiler::types::ScalarType::String); + IRExpr::comparison( + IRExpr::PropAccess { + variable: "d".to_string(), + property: catalog.system_columns.id.to_string(), + ty: ty.clone(), + }, + omnigraph_compiler::query::ast::CompOp::Eq, + IRExpr::Literal(Literal::String(id.to_string()), ty), + ) +} + +/// A key narrowing (`scan_access::narrow_key`) is a recorded rule of the +/// exact fragment and a predicate the invariants re-derive: the planned +/// narrowing is accepted, a forged id or a narrowing with no written key +/// equality is refused. +#[test] +fn a_key_narrowing_is_accepted_only_with_the_derived_id() { + let fixture = Fixture::new(KEYED, &[("s", Literal::String("a".into()))]); + let id = fixture.catalog.system_columns.id.to_string(); + let source = fixture + .source + .clone() + .with_canonical_key_ids(fixture.params.clone()) + .with_index_facts( + "node:Doc", + vec![crate::source::IndexFact { + name: "id_idx".into(), + column: id, + kind: crate::source::IndexKind::Btree { usable: true }, + coverage: Some(crate::source::FragmentCoverage { + covered: 1, + total: 1, + }), + }], + ); + let accepted = futures::executor::block_on(crate::gate::accept_query( + &fixture.input(), + &source, + &BOUNDS, + )) + .expect("the narrowed plan is accepted"); + assert_eq!(accepted.scope(), ValidationScope::ExactSubset); + let derivation = accepted.evidence().derivation().cloned().unwrap(); + assert!(rules(&derivation).contains(&"narrow"), "{derivation:?}"); + let narrowed = identity(&fixture.catalog, "id:a"); + let placed = node_ids(accepted.plan(), |node| { + matches!(node, PhysicalNode::Scan { .. }) + }) + .into_iter() + .any(|id| match accepted.plan().node(id) { + Some(PhysicalNode::Scan { spec, .. }) => spec + .filter + .as_ref() + .is_some_and(|filter| filter.gq_filters().contains(&narrowed)), + _ => false, + }); + assert!(placed, "the plan tests the narrowed identity"); + + let mut forged = accepted.plan().clone(); + for id in node_ids(&forged, |node| matches!(node, PhysicalNode::Scan { .. })) { + if let Some(PhysicalNode::Scan { spec, .. }) = forged.node_mut(id) { + let conjuncts: Vec = spec + .filter + .as_ref() + .map(|filter| filter.gq_filters()) + .unwrap_or_default(); + let mut rebuilt: Option = None; + for conjunct in conjuncts { + let conjunct = if conjunct == narrowed { + identity(&fixture.catalog, "id:b") + } else { + conjunct + }; + let added = crate::optimizer::gq_conjunct(&conjunct); + rebuilt = Some(match rebuilt { + Some(filter) => filter.and(added), + None => added, + }); + } + spec.filter = rebuilt; + } + } + match fixture.accept_with(forged, Some(derivation)) { + Err(ValidationError::Violated { check, detail }) => { + assert_eq!(check, "predicate origin", "{detail}"); + } + other => panic!("a forged id was not refused: {other:?}"), + } + + let keyless = Fixture::new("query q() { match { $d: Doc } return { $d.title } }", &[]); + let mut plan = keyless.plan(); + place_on_scans(&mut plan, &identity(&keyless.catalog, "id:a")); + let (check, detail) = keyless.refused(plan); + assert_eq!(check, "predicate origin", "{detail}"); +} + +#[test] +fn edited_hop_bounds_fail_traversal_identity() { + let fixture = Fixture::new( + "query q() { match { $d: Doc $d cites{1,2} $e } return { $d.slug, $e.slug } }", + &[], + ); + let plan = fixture.plan(); + fixture + .accept(plan.clone()) + .expect("the planned traversal is accepted"); + let mut plan = plan; + for id in node_ids(&plan, |node| matches!(node, PhysicalNode::Expand { .. })) { + if let Some(PhysicalNode::Expand { max_hops, .. }) = plan.node_mut(id) { + *max_hops = Some(1); + } + } + let (check, detail) = fixture.refused(plan); + assert_eq!(check, "traversal identity", "{detail}"); +} + +/// The first key of the final sort, removed. +fn drop_first_sort_key(plan: &mut PhysicalPlan) { + for id in node_ids(plan, |node| matches!(node, PhysicalNode::Sort { .. })) { + if let Some(PhysicalNode::Sort { order_by, .. }) = plan.node_mut(id) { + order_by.remove(0); + } + } +} + +/// A fused search sorts by `_rrf` descending then its written keys; a +/// search-led aggregate by its written keys then every group key. +#[test] +fn fused_and_grouped_searches_sort_as_the_query_requires() { + let q = [("q", Literal::String("graph".into()))]; + let fused = Fixture::new( + "query q($q: String) { match { $d: Doc } return { $d.slug } order { rrf(bm25($d.title, $q), bm25($d.title, $q)), $d.slug desc } limit 2 }", + &q, + ); + let grouped = Fixture::new( + "query q($q: String) { match { $d: Doc } return { $d.open, count($d) as n } order { bm25($d.title, $q), n desc } }", + &q, + ); + for fixture in [fused, grouped] { + let plan = fixture.plan(); + fixture + .accept(plan.clone()) + .expect("the planned order is accepted"); + let mut plan = plan; + drop_first_sort_key(&mut plan); + let (check, detail) = fixture.refused(plan); + assert_eq!(check, "order", "{detail}"); + } +} + +/// A member whose bare return columns are hydrated by row address keeps its +/// exact subset through the recorded rule; a hydration that places a column +/// elsewhere is refused. +#[test] +fn a_hydrated_member_keeps_its_exact_subset() { + let fixture = Fixture::new( + "query q() { match { $d: Doc } return { $d.slug, $d.title } order { $d.year } limit 2 }", + &[], + ); + let plan = fixture.plan(); + assert!( + matches!( + plan.node(plan.root()), + Some(PhysicalNode::HydrateColumns { .. }) + ), + "the fixture hydrates its title" + ); + let accepted = fixture + .accept(plan.clone()) + .expect("a hydrated plan is accepted"); + assert_eq!(accepted.scope(), ValidationScope::ExactSubset); + assert!(rules(accepted.evidence().derivation().unwrap()).contains(&"hydrate")); + let mut plan = plan; + let root = plan.root(); + if let Some(PhysicalNode::HydrateColumns { bindings, .. }) = plan.node_mut(root) { + bindings[0].columns[0].position = 0; + } + let (check, detail) = fixture.refused(plan); + assert_eq!(check, "hydration", "{detail}"); +} + #[test] fn a_scan_of_another_binding_fails_binding_identity() { let fixture = filtered(); @@ -821,7 +1061,7 @@ fn derivation_cost_grows_with_the_query() { ); let fixture = Fixture::new(&query, &[]); let traced = fixture.traced(); - let derivation = traced.derivation.clone().unwrap(); + let derivation = traced.derivation().unwrap(); let bytes = serde_json::to_vec(&derivation).unwrap().len(); let input = AcceptInput { checked: &fixture.checked, From 669ca00593f3310a0938f3fcea56129cf12f0aa6 Mon Sep 17 00:00:00 2001 From: Ragnor Comerford Date: Sun, 11 Oct 2026 10:01:17 -0700 Subject: [PATCH 24/29] planner: judge order like the planner and derive full-text coverage from index facts A constant return item cannot tell rows apart, so it counts as ordered for the tie-break; an `order` clause whose keys all fold to constants still requires the identity tie-break sort; an alias key covers every return item lowering to the expression it names. Acceptance names a conjunct or sort key whose text matches the query's but whose stored types do not. `FullTextCoverage::of_facts` derives the coverage fact from the planner's index facts: full only when every inverted segment on the column reports known, complete coverage. --- crates/omnigraph-gqt-core/src/plan.rs | 4 +- crates/omnigraph-planner/src/source.rs | 28 ++++++++ .../src/validate/invariants.rs | 72 +++++++++++-------- .../src/validate/requirements.rs | 10 ++- .../omnigraph-planner/src/validate/subset.rs | 3 +- .../omnigraph-planner/src/validate/tests.rs | 58 +++++++++++++++ 6 files changed, 141 insertions(+), 34 deletions(-) diff --git a/crates/omnigraph-gqt-core/src/plan.rs b/crates/omnigraph-gqt-core/src/plan.rs index c1ac7ccc7..92f5f6a15 100644 --- a/crates/omnigraph-gqt-core/src/plan.rs +++ b/crates/omnigraph-gqt-core/src/plan.rs @@ -195,7 +195,9 @@ pub(crate) enum PlanLine { }, /// Acceptance checked the plan to `scope` (`exact_subset` or /// `invariants_only`), as the document's `validation` reports it. - ValidationScope { scope: String }, + ValidationScope { + scope: String, + }, } #[derive(Debug, Default)] diff --git a/crates/omnigraph-planner/src/source.rs b/crates/omnigraph-planner/src/source.rs index f83de18c3..8c72299fc 100644 --- a/crates/omnigraph-planner/src/source.rs +++ b/crates/omnigraph-planner/src/source.rs @@ -281,6 +281,34 @@ pub enum FullTextCoverage { Absent, } +impl FullTextCoverage { + /// The coverage the index facts of one table establish for `column`: + /// `Absent` with no index on it, `Full` only when it has a full-text + /// index and every full-text index on it has known coverage of every + /// current fragment, `Partial` otherwise. An index of unknown kind or + /// unknown coverage on the column never counts as full. + pub fn of_facts(facts: &[IndexFact], column: &str) -> Self { + let on_column: Vec<&IndexFact> = + facts.iter().filter(|fact| fact.column == column).collect(); + let full_text = on_column + .iter() + .any(|fact| fact.kind == IndexKind::Inverted); + let unknown = on_column.iter().any(|fact| fact.kind == IndexKind::Unknown); + if !full_text { + return if unknown { Self::Partial } else { Self::Absent }; + } + let full = !unknown + && on_column + .iter() + .filter(|fact| fact.kind == IndexKind::Inverted) + .all(|fact| { + fact.coverage + .is_some_and(|coverage| coverage.covered == coverage.total) + }); + if full { Self::Full } else { Self::Partial } + } +} + /// An in-memory [`PlanSource`] for planner tests and registry fixtures. #[derive(Debug, Clone, Default)] pub struct MemorySource { diff --git a/crates/omnigraph-planner/src/validate/invariants.rs b/crates/omnigraph-planner/src/validate/invariants.rs index b63055e64..9ab551690 100644 --- a/crates/omnigraph-planner/src/validate/invariants.rs +++ b/crates/omnigraph-planner/src/validate/invariants.rs @@ -121,6 +121,18 @@ fn conjuncts(plan: &PhysicalPlan, nodes: &[NodeId]) -> Vec<(NodeId, IRExpr)> { out } +/// The expression an order key names: for an alias, the return item it +/// aliases. +fn named_expr<'e>(returns: &'e [IRProjection], key: &'e IRExpr) -> &'e IRExpr { + match key { + IRExpr::AliasRef(alias, _) => returns + .iter() + .find(|projection| projection.column == *alias) + .map_or(key, |projection| &projection.expr), + expr => expr, + } +} + /// The ranked scans of the plan with their ids. fn ranked_scans(plan: &PhysicalPlan) -> Vec<(NodeId, &str, &RankedAccess)> { plan.live() @@ -276,10 +288,17 @@ impl Requirements { .iter() .find(|required| !placed_set.contains(required)) { - return Err(ValidationError::violated( - "predicate retention", - format!("the plan tests no conjunct for `{missing}`"), - )); + // The same text under other stored types is a retyped conjunct. + let text = missing.to_string(); + let detail = if placed + .iter() + .any(|(_, conjunct)| conjunct.to_string() == text) + { + format!("the plan tests `{missing}` with stored types the query does not give it") + } else { + format!("the plan tests no conjunct for `{missing}`") + }; + return Err(ValidationError::violated("predicate retention", detail)); } let required: HashSet<&IRExpr> = self.eligibility.iter().collect(); for (id, conjunct) in &placed { @@ -450,7 +469,7 @@ impl Requirements { } match (&self.search, fuse) { (Some(Search::Fuse { arms, k }), Some((planned, planned_k))) => { - for (arm, retrieval) in planned.iter().zip(arms) { + for (arm, retrieval) in planned.iter().zip(arms.iter()) { if arm.binding != retrieval.binding || arm.kind != retrieval.kind { return Err(ValidationError::violated( "search identity", @@ -753,7 +772,9 @@ impl Requirements { /// ascending; no sort where the query orders nothing. fn required_keys(&self) -> Option> { if self.aggregate { - if self.order.is_empty() { + // Under a search, groups have no order unless a key remains + // after the score; otherwise any written clause sorts them. + if !self.ordered || (self.search.is_some() && self.order.is_empty()) { return None; } let mut keys = self.order.clone(); @@ -780,7 +801,7 @@ impl Requirements { return Some(keys); } let lead = match &self.search { - None if self.order.is_empty() => return None, + None if !self.ordered => return None, None => Vec::new(), Some(Search::Rank(retrieval)) => { let (column, descending) = retrieval.kind.score(); @@ -900,14 +921,15 @@ impl Requirements { } }); if !same { - return Err(ValidationError::violated( - "order", + let (planned, wanted) = (keys_text(order_by), keys_text(&required)); + let detail = if planned == wanted { format!( - "the final sort orders by {}; the query requires {}", - keys_text(order_by), - keys_text(&required) - ), - )); + "the final sort orders by {planned} with stored types the query does not give its keys" + ) + } else { + format!("the final sort orders by {planned}; the query requires {wanted}") + }; + return Err(ValidationError::violated("order", detail)); } self.check_tiebreak(order_by, tiebreak, input)?; Ok(()) @@ -918,7 +940,7 @@ impl Requirements { /// edge) in binding-name order, ascending, nulls first. Three planner /// omissions are comparator equivalences and accepted: a key the /// comparator already holds, every key when the query returns only order - /// keys (rows that tie are indistinguishable), and identities of + /// keys or constants (rows that tie are indistinguishable), and identities of /// bindings the query does not name (anonymous traversal endpoints) /// after every named one. A sort of group rows appends none. fn check_tiebreak( @@ -942,12 +964,14 @@ impl Requirements { *position == index && self.written_search.as_ref() == Some(call) }) }; + // A key orders the expression it names, so it covers every return + // item lowering to that expression, under any alias. let covered = self.returns.iter().enumerate().all(|(index, projection)| { ranked_score(index) - || order_by.iter().any(|key| match &key.expr { - IRExpr::AliasRef(alias, _) => *alias == projection.column, - expr => *expr == projection.expr, - }) + || matches!(projection.expr, IRExpr::Literal(_, _)) + || order_by + .iter() + .any(|key| *named_expr(&self.returns, &key.expr) == projection.expr) }); if covered { return Ok(()); @@ -960,15 +984,7 @@ impl Requirements { _ => return false, }; order_by.iter().any(|key| { - let expr = match &key.expr { - IRExpr::AliasRef(alias, _) => self - .returns - .iter() - .find(|projection| projection.column == *alias) - .map_or(&key.expr, |projection| &projection.expr), - expr => expr, - }; - matches!(expr, IRExpr::PropAccess { variable, property, .. } + matches!(named_expr(&self.returns, &key.expr), IRExpr::PropAccess { variable, property, .. } if *variable == column.binding && property == physical) }) }; diff --git a/crates/omnigraph-planner/src/validate/requirements.rs b/crates/omnigraph-planner/src/validate/requirements.rs index b081b5dbf..f1af1bab9 100644 --- a/crates/omnigraph-planner/src/validate/requirements.rs +++ b/crates/omnigraph-planner/src/validate/requirements.rs @@ -27,7 +27,7 @@ use crate::physical::RankKind; /// path derives it from the key's value. The engine implements it with the /// loader's derivation, so a key narrowing the planner adds (an `@id` /// equality beside the written key equality) is re-derived, not trusted. -pub trait KeyIds { +pub trait KeyIds: Sync { /// The id of the node of `type_key` whose key is `value` (typed `ty`), or /// `None` when the type is not keyed by one String property. fn canonical_key_id(&self, type_key: &str, value: &Literal, ty: &ExprType) -> Option; @@ -84,7 +84,7 @@ impl Retrieval { pub(crate) enum Search { Rank(Retrieval), Fuse { - arms: [Retrieval; 2], + arms: Box<[Retrieval; 2]>, k: Option, }, } @@ -138,6 +138,9 @@ pub struct Requirements { /// literal key does not), each bound to a `return` item as the planner /// binds it. pub(crate) order: Vec, + /// Whether the query writes an `order` clause at all: one whose every + /// key is a constant still orders its rows by the identity tie-break. + pub(crate) ordered: bool, pub(crate) limit: Option, pub(crate) aggregate: bool, } @@ -267,7 +270,7 @@ impl Requirements { .. }) => match (Retrieval::of(primary), Retrieval::of(secondary)) { (Some(primary), Some(secondary)) => Some(Search::Fuse { - arms: [primary, secondary], + arms: Box::new([primary, secondary]), k: k.as_deref().cloned(), }), _ => None, @@ -308,6 +311,7 @@ impl Requirements { .collect(), written_search, order, + ordered: !ir.order_by.is_empty(), limit: ir.limit, aggregate: ir .return_exprs diff --git a/crates/omnigraph-planner/src/validate/subset.rs b/crates/omnigraph-planner/src/validate/subset.rs index 620eace0d..fa3e9008a 100644 --- a/crates/omnigraph-planner/src/validate/subset.rs +++ b/crates/omnigraph-planner/src/validate/subset.rs @@ -1099,8 +1099,7 @@ fn apply( .key .as_deref() .unwrap_or_default() - .iter() - .any(|key| *key == column.property) + .contains(&column.property) || column.property == input.catalog.system_columns.id { return Err(invalid(format!( diff --git a/crates/omnigraph-planner/src/validate/tests.rs b/crates/omnigraph-planner/src/validate/tests.rs index fa3e1e6a9..99105fcc9 100644 --- a/crates/omnigraph-planner/src/validate/tests.rs +++ b/crates/omnigraph-planner/src/validate/tests.rs @@ -1136,3 +1136,61 @@ fn a_reference_to_an_unreached_or_superseded_node_is_refused() { } } } + +/// A return item the lowering folds to a constant (`@type` of a binding +/// with one edge type) cannot tell two rows apart, so ordering by the other +/// returns completes the comparator without identities. +#[test] +fn a_constant_return_needs_no_tiebreak() { + let fixture = Fixture::new( + r#"query q() { + match { $p: Doc $d: Doc $p $e:cites $d } + return { $p.slug as source, $d.slug as target, $e.@type as edge_type } + order { $p.slug, $d.slug, $e.@type } +}"#, + &[], + ); + assert!( + fixture + .ir + .return_exprs + .iter() + .any(|projection| matches!(projection.expr, IRExpr::Literal(_, _))), + "the lowering folds the single edge type" + ); + fixture + .accept(fixture.plan()) + .expect("a constant return is covered by the other keys"); +} + +/// An `order` clause whose every key folds to a constant still orders its +/// rows, by the identity tie-break alone (or groups, by nothing more), so the +/// plan's sort is required rather than refused. +#[test] +fn a_constant_order_still_sorts() { + for query in [ + r#"query q() { + match { $p: Doc $d: Doc $p $e:cites $d } + return { $d.slug as target } + order { $e.@type } +}"#, + r#"query q() { + match { $p: Doc $d: Doc $p $e:cites $d } + return { $p.slug as source, count($d) as n } + order { $e.@type } +}"#, + ] { + let fixture = Fixture::new(query, &[]); + assert!( + fixture + .ir + .order_by + .iter() + .all(|key| matches!(key.expr, IRExpr::Literal(_, _))), + "the lowering folds the single edge type: {query}" + ); + fixture + .accept(fixture.plan()) + .unwrap_or_else(|error| panic!("{query}: {error:?}")); + } +} From 89406c4e4475759c80b5e1afd0eef74d802d599f Mon Sep 17 00:00:00 2001 From: Ragnor Comerford Date: Sun, 11 Oct 2026 10:01:25 -0700 Subject: [PATCH 25/29] engine: accept every plan through one gate and re-derive narrowed keys Execution no longer re-checks aggregate specs, output schemas or fusion row keys: acceptance checks them first, so a saved plan that fails one is a bad request at the replay door. The engine supplies acceptance's key-id derivation (`CatalogKeys`, the loader's canonical id) and reads full-text coverage from the index facts, at planning and again on replay; the separate `TableStore::fts_coverage` reader is gone. The replay tests move to the envelope door and assert acceptance's refusals for forged plans, leaf types and sort aliases. --- crates/omnigraph/src/engine/mod.rs | 8 +- crates/omnigraph/src/engine/plan_source.rs | 190 +++++-------- .../src/engine/plan_source/key_tests.rs | 35 ++- crates/omnigraph/src/engine/scan.rs | 1 + crates/omnigraph/src/table_store.rs | 56 +--- crates/omnigraph/tests/engine_v2.rs | 9 +- .../omnigraph/tests/engine_v2_plan_replay.rs | 261 ++++++++++-------- crates/omnigraph/tests/forbidden_apis.rs | 1 - 8 files changed, 255 insertions(+), 306 deletions(-) diff --git a/crates/omnigraph/src/engine/mod.rs b/crates/omnigraph/src/engine/mod.rs index 6971ec7fc..065812222 100644 --- a/crates/omnigraph/src/engine/mod.rs +++ b/crates/omnigraph/src/engine/mod.rs @@ -410,14 +410,10 @@ pub(crate) async fn execute( accepted: AcceptedBoundPlan, context: &EngineContext<'_>, ) -> Result { + // Acceptance checked the plan's aggregate specs, output schemas and + // fusion row keys; only what depends on this run's admission is left. let (bound, evidence) = accepted.into_parts(); - omnigraph_planner::validate_aggregate_specs(&bound.plan) - .map_err(|error| OmniError::manifest_internal(error.to_string()))?; - omnigraph_planner::validate_output_schemas(&bound.plan) - .map_err(|error| OmniError::manifest_internal(error.to_string()))?; let traversal_limit = validate_traversal_admission(&bound.plan)?; - omnigraph_planner::optimizer::validate_rank_fuse_row_tiebreaks(&bound.plan) - .map_err(|error| OmniError::manifest_internal(error.to_string()))?; let ctx = QueryContext::with_traversal_limit(bound.plan.assumptions().memory_limit, traversal_limit)?; ctx.run_owned(execute_with_context(bound, evidence, context, &ctx)) diff --git a/crates/omnigraph/src/engine/plan_source.rs b/crates/omnigraph/src/engine/plan_source.rs index ed6385c9c..507d81887 100644 --- a/crates/omnigraph/src/engine/plan_source.rs +++ b/crates/omnigraph/src/engine/plan_source.rs @@ -19,11 +19,11 @@ use omnigraph_compiler::query::ast::Literal; use omnigraph_compiler::settings::{RrfPlan, SessionSettings, Traversal}; use omnigraph_compiler::types::Direction; use omnigraph_planner::{ - AcceptInput, AcceptedBoundPlan, AcceptedPlan, AdjacencyProof, Bounds, ConstantEvaluator, - DatasetPin, EXPAND_INDEXED_MAX_FRONTIER_ENV, EXPAND_INDEXED_MAX_HOPS_ENV, ExpandStatistics, - Explain, FragmentStat, FullTextCoverage, GatePolicy, IndexFact, NodeTypeSpec, Operation, - PlanError, PlanSource, PrefilterMode, ReplayEnvelope, ReplayRefusal, SideId, TableRef, Unrouted, - ValidationLimits, + AcceptInput, AcceptedBoundPlan, AcceptedPlan, AdjacencyProof, Bounds, DatasetPin, + EXPAND_INDEXED_MAX_FRONTIER_ENV, EXPAND_INDEXED_MAX_HOPS_ENV, ExpandStatistics, Explain, + FragmentStat, FullTextCoverage, GatePolicy, IndexFact, KeyIds, NodeTypeSpec, Operation, + PlanError, PlanSource, PrefilterMode, ReplayEnvelope, ReplayRefusal, SideId, TableRef, + Unrouted, ValidationLimits, }; use super::ResolvedParams; @@ -124,9 +124,6 @@ pub(crate) struct QuerySource<'a> { gate_policy: GatePolicy, expand_caps: ExpandCaps, table_stats: HashMap, - /// The full-text coverage of every property a `bm25()` order key ranks, - /// by `(table key, property)`, read at the pinned dataset version. - full_text: HashMap<(String, String), FullTextCoverage>, /// The error the source met answering the planner's last lookup, handed /// back to the caller in place of the planner's wrapper. source_failure: std::sync::Mutex>, @@ -152,7 +149,6 @@ impl<'a> QuerySource<'a> { let params = resolve_params(&query.ir, params)?; let ir = super::constant::fold_query_constants(&query.ir, params.shared())?; let table_stats = query_table_statistics(&ir, snapshot).await?; - let full_text = ranked_full_text_coverage(&ir, snapshot).await?; let mut source = QuerySource { ir, checked: query.checked.clone(), @@ -164,7 +160,6 @@ impl<'a> QuerySource<'a> { gate_policy: gate_policy(settings), expand_caps: ExpandCaps::from_env(), table_stats, - full_text, source_failure: std::sync::Mutex::new(None), }; source @@ -233,10 +228,7 @@ impl<'a> QuerySource<'a> { impl PlanSource for QuerySource<'_> { fn full_text_coverage(&self, type_key: &str, property: &str) -> FullTextCoverage { - self.full_text - .get(&(type_key.to_string(), property.to_string())) - .copied() - .unwrap_or(FullTextCoverage::Absent) + FullTextCoverage::of_facts(&self.index_facts(type_key), property) } fn canonical_key_id( @@ -244,42 +236,15 @@ impl PlanSource for QuerySource<'_> { type_key: &str, value: &IRExpr, ) -> std::result::Result, PlanError> { - use omnigraph_compiler::{ExprType, ScalarType}; - let Some(node) = type_key - .strip_prefix("node:") - .and_then(|name| self.catalog.node_types.get(name)) - else { - return Ok(None); - }; - let Some([key]) = node.key.as_deref() else { - return Ok(None); - }; - if !node - .properties - .get(key) - .is_some_and(|ty| ty.scalar == ScalarType::String && !ty.list) - || !matches!( - value.ty(), - ExprType::Value { - scalar: ScalarType::String, - list: false, - .. - } - ) - { - return Ok(None); - } let literal = match value { IRExpr::Literal(value, _) => Some(value), IRExpr::Param(name, _) => self.params.shared().get(name), _ => None, }; - let Some(literal @ Literal::String(_)) = literal else { + let Some(literal) = literal else { return Ok(None); }; - let array = super::typed_value::typed_literal_to_array(literal, value.ty(), 1) - .map_err(|error| PlanError::Internal(error.to_string()))?; - crate::loader::canonical_key_id(&[array], 0) + canonical_key_id_of(self.catalog, type_key, literal, value.ty()) .map_err(|error| PlanError::Internal(error.to_string())) } @@ -520,13 +485,60 @@ fn resolve_params(ir: &QueryIR, params: &ParamMap) -> Result { Ok(ResolvedParams(Arc::new(resolved))) } -/// The engine's constant evaluator for acceptance: the rules its own -/// constant folding applied to the IR, under the same bound parameters. -struct BoundConstants<'p>(&'p ParamMap); +/// The canonical id of the node of `type_key` whose single String key is +/// `value` (typed `ty`), as every write path derives it; `None` when the +/// type is not keyed by one String property or the value is no String. +fn canonical_key_id_of( + catalog: &Catalog, + type_key: &str, + value: &Literal, + ty: &omnigraph_compiler::ExprType, +) -> Result> { + use omnigraph_compiler::{ExprType, ScalarType}; + let Some(node) = type_key + .strip_prefix("node:") + .and_then(|name| catalog.node_types.get(name)) + else { + return Ok(None); + }; + let Some([key]) = node.key.as_deref() else { + return Ok(None); + }; + let string_key = node + .properties + .get(key) + .is_some_and(|property| property.scalar == ScalarType::String && !property.list); + let string_value = matches!( + ty, + ExprType::Value { + scalar: ScalarType::String, + list: false, + .. + } + ); + if !string_key || !string_value || !matches!(value, Literal::String(_)) { + return Ok(None); + } + let array = super::typed_value::typed_literal_to_array(value, ty, 1)?; + crate::loader::canonical_key_id(&[array], 0) +} -impl ConstantEvaluator for BoundConstants<'_> { - fn evaluate(&self, expr: &IRExpr) -> Option { - super::constant::evaluate_constant(expr, self.0).ok() +/// Acceptance's derivation of a key's canonical id: the same function the +/// plan source narrows a key with, so a narrowed id is re-derived. A value +/// the derivation fails on yields no id, so acceptance refuses the narrowed +/// conjunct rather than admitting it. +struct CatalogKeys<'c>(&'c Catalog); + +impl KeyIds for CatalogKeys<'_> { + fn canonical_key_id( + &self, + type_key: &str, + value: &Literal, + ty: &omnigraph_compiler::ExprType, + ) -> Option { + canonical_key_id_of(self.0, type_key, value, ty) + .ok() + .flatten() } } @@ -554,13 +566,13 @@ impl QuerySource<'_> { .unwrap_or_else(|| unaccepted(reason)) } - fn accept_input<'s>(&'s self, constants: &'s BoundConstants<'s>) -> AcceptInput<'s> { + fn accept_input<'s>(&'s self, keys: &'s CatalogKeys<'s>) -> AcceptInput<'s> { AcceptInput { checked: &self.checked, catalog: self.catalog, ir: &self.ir, params: self.params.shared(), - constants, + keys, limits: ValidationLimits::DEFAULT, } } @@ -568,8 +580,8 @@ impl QuerySource<'_> { /// Plan the read and accept the plan, without explain diagnostics. pub(crate) async fn accept_query(source: &QuerySource<'_>) -> Result { - let constants = BoundConstants(source.params.shared()); - omnigraph_planner::accept_query(&source.accept_input(&constants), source, &source.bounds()) + let keys = CatalogKeys(source.catalog); + omnigraph_planner::accept_query(&source.accept_input(&keys), source, &source.bounds()) .await .map_err(|reason| source.unaccepted(reason)) } @@ -584,13 +596,13 @@ pub(crate) fn accept_replay( ) -> Result { let params = Arc::clone(&envelope.plan.values.params); let ir = super::constant::fold_query_constants(&query.ir, ¶ms)?; - let constants = BoundConstants(¶ms); + let keys = CatalogKeys(catalog); let input = AcceptInput { checked: &query.checked, catalog, ir: &ir, params: ¶ms, - constants: &constants, + keys: &keys, limits: ValidationLimits::DEFAULT, }; omnigraph_planner::accept_replay(envelope, &input).map_err(replay_refused) @@ -620,7 +632,7 @@ pub(crate) async fn replayed_coverage_holds( let actual = match snapshot.dataset(type_key) { Some(_) => { let dataset = snapshot.open_lance_dataset(type_key).await?; - crate::table_store::TableStore::fts_coverage(&dataset, property).await? + FullTextCoverage::of_facts(&indexes::gather(&dataset).await?, property) } None => FullTextCoverage::Absent, }; @@ -665,9 +677,9 @@ pub(crate) struct ExplainedQuery { /// [`accept_query`] with the accepted plan's explain document, rendered from /// the same planning run; it validates exactly what a run validates. pub(crate) async fn explain_query(source: &QuerySource<'_>) -> Result { - let constants = BoundConstants(source.params.shared()); + let keys = CatalogKeys(source.catalog); let (accepted, explain) = omnigraph_planner::accept_query_explained( - &source.accept_input(&constants), + &source.accept_input(&keys), source, &source.bounds(), ) @@ -676,68 +688,6 @@ pub(crate) async fn explain_query(source: &QuerySource<'_>) -> Result Result> { - fn bm25_targets(expr: &IRExpr, out: &mut Vec<(String, String)>) { - match expr { - IRExpr::Bm25 { field, .. } => { - if let IRExpr::PropAccess { variable, property } = field.as_ref() { - out.push((variable.clone(), property.clone())); - } - } - IRExpr::Rrf { - primary, secondary, .. - } => { - bm25_targets(primary, out); - bm25_targets(secondary, out); - } - _ => {} - } - } - fn binding_type(ops: &[IROp], binding: &str) -> Option { - ops.iter().find_map(|op| match op { - IROp::NodeScan { - variable, - type_name, - .. - } if variable == binding => Some(type_name.clone()), - IROp::Expand { - dst_var, dst_type, .. - } if dst_var == binding => Some(dst_type.clone()), - _ => None, - }) - } - let mut targets = Vec::new(); - if let Some(leading) = ir.order_by.first() { - bm25_targets(&leading.expr, &mut targets); - } - let mut coverage = HashMap::new(); - for (binding, property) in targets { - let Some(type_name) = binding_type(&ir.pipeline, &binding) else { - continue; - }; - let type_key = format!("node:{type_name}"); - if coverage.contains_key(&(type_key.clone(), property.clone())) { - continue; - } - let known = match snapshot.dataset(&type_key) { - Some(_) => { - let dataset = snapshot.open_lance_dataset(&type_key).await?; - crate::table_store::TableStore::fts_coverage(&dataset, &property).await? - } - None => FullTextCoverage::Absent, - }; - coverage.insert((type_key, property), known); - } - Ok(coverage) -} - async fn query_table_statistics( ir: &QueryIR, snapshot: &Snapshot, diff --git a/crates/omnigraph/src/engine/plan_source/key_tests.rs b/crates/omnigraph/src/engine/plan_source/key_tests.rs index 03c4ee2c2..e1f1030cd 100644 --- a/crates/omnigraph/src/engine/plan_source/key_tests.rs +++ b/crates/omnigraph/src/engine/plan_source/key_tests.rs @@ -59,12 +59,14 @@ async fn run_with_access( let stores = probes.table_stores.clone(); with_query_io_probes(probes, async { let statement = omnigraph_compiler::find_read_statement(text, "lookup").unwrap(); - let checked = - omnigraph_compiler::query::typecheck::typecheck_query(catalog, statement.decl()) - .unwrap(); - let ir = omnigraph_compiler::lower_query(catalog, statement.decl(), &checked).unwrap(); + let checked = CheckedQuery::check(catalog, statement.decl()).unwrap(); + let ir = omnigraph_compiler::lower_query(catalog, checked.decl(), checked.types()).unwrap(); + let query = CompiledQuery { + ir: Arc::new(ir), + checked, + }; let settings = SessionSettings::default(); - let source = QuerySource::gather(&ir, catalog, &snapshot, params, &settings) + let source = QuerySource::gather(&query, catalog, &snapshot, params, &settings) .await .unwrap(); assert!( @@ -76,10 +78,13 @@ async fn run_with_access( omnigraph_planner::IndexKind::Btree { usable: true } ) && fact.column == "__id") ); - let ExplainedQuery { explain, physical } = explain_query(&source).await.unwrap(); - let mut bound = crate::engine::bind::bind(physical, &source, &EmbeddingResolver::explain()) - .await - .unwrap(); + let ExplainedQuery { explain, accepted } = explain_query(&source).await.unwrap(); + let evidence = accepted.evidence().clone(); + let (mut bound, _) = + crate::engine::bind::bind(accepted, &source, &EmbeddingResolver::explain()) + .await + .unwrap() + .into_parts(); if sequential { let scans: Vec<_> = bound .plan @@ -106,7 +111,17 @@ async fn run_with_access( catalog, graph_index: Arc::new(GraphIndexHandle::none()), }; - let run = crate::engine::execute(bound, &context).await.unwrap(); + // The edited access is a cost choice acceptance admits: the plan + // reaches execution through the replay door, like any saved plan. + let envelope = omnigraph_planner::ReplayEnvelope::new( + text, + "lookup", + bound, + &evidence, + omnigraph_planner::catalog_digest(catalog), + ); + let accepted = accept_replay(&query, envelope, catalog).unwrap(); + let run = crate::engine::execute(accepted, &context).await.unwrap(); let execution = drain(&stores); Observed { planning, diff --git a/crates/omnigraph/src/engine/scan.rs b/crates/omnigraph/src/engine/scan.rs index d9bba58fe..d04ac193f 100644 --- a/crates/omnigraph/src/engine/scan.rs +++ b/crates/omnigraph/src/engine/scan.rs @@ -126,6 +126,7 @@ impl<'n> NodeRead<'n> { scalar_index: Some(true), columns, proven_empty: false, + filter_after_scoring: false, }) } diff --git a/crates/omnigraph/src/table_store.rs b/crates/omnigraph/src/table_store.rs index 9554e6745..094d22949 100644 --- a/crates/omnigraph/src/table_store.rs +++ b/crates/omnigraph/src/table_store.rs @@ -70,7 +70,7 @@ use crate::blob::{ }; use crate::dataset_index::{ has_btree_index_on, has_fts_index_on, has_vector_index_on, is_full_text_index, - user_indices_for_column, validate_full_text_demand, validate_full_text_scan, + validate_full_text_demand, validate_full_text_scan, }; use crate::db::manifest::{TableVersionMetadata, open_dataset_entry}; use crate::db::{DatasetEntry, Snapshot}; @@ -5096,60 +5096,6 @@ impl TableStore { has_fts_index_on(ds, column).await } - /// Metadata-only read (no data IO) of how far the FTS (inverted) index - /// entries on `column` together cover `ds`'s current fragments: every - /// fragment, some (an entry exists, but the bitmaps do not prove every - /// fragment), or no full-text entry at all. The FTS twin of - /// `dataset_index::key_column_index_coverage`, read while planning: rows - /// in fragments no entry covers are scored by a filter-dependent - /// batch-derived scorer instead of the index-global BM25 statistics - /// (lance `inverted/index.rs`), so a filter applied before scoring would - /// change their scores. Coverage is the UNION across same-column - /// entries: index optimization creates delta entries each covering a - /// disjoint fragment subset, and demanding one all-covering entry would - /// permanently rule out filtering before scoring on exactly the append + - /// optimize maintenance schedule production tables follow. Anything - /// short of full is always safe — eligibility then applies after - /// scoring and the gates run their postfilter plans — so every - /// unprovable case (an entry without a fragment bitmap) is partial. - pub(crate) async fn fts_coverage( - ds: &Dataset, - column: &str, - ) -> Result { - use omnigraph_planner::FullTextCoverage; - let indices = user_indices_for_column(ds, column).await?; - let fts_entries: Vec<_> = indices - .iter() - .filter(|index| { - index - .index_details - .as_ref() - .map(|details| IndexDetails(details.clone()).supports_fts()) - .unwrap_or(false) - }) - .collect(); - if fts_entries.is_empty() { - return Ok(FullTextCoverage::Absent); - } - let fts_bitmaps: Vec<_> = fts_entries - .iter() - .filter_map(|index| index.fragment_bitmap.as_ref()) - .collect(); - if fts_bitmaps.is_empty() { - return Ok(FullTextCoverage::Partial); - } - let covered = ds.fragments().iter().all(|f| { - fts_bitmaps - .iter() - .any(|bitmap| bitmap.contains(f.id as u32)) - }); - Ok(if covered { - FullTextCoverage::Full - } else { - FullTextCoverage::Partial - }) - } - pub async fn has_vector_index(&self, ds: &Dataset, column: &str) -> Result { has_vector_index_on(ds, column).await } diff --git a/crates/omnigraph/tests/engine_v2.rs b/crates/omnigraph/tests/engine_v2.rs index eefb416dd..7619e05b3 100644 --- a/crates/omnigraph/tests/engine_v2.rs +++ b/crates/omnigraph/tests/engine_v2.rs @@ -1011,12 +1011,11 @@ edge Likes: Person -> Doc .await .unwrap(); db.ensure_indices().await.unwrap(); - for (source, name, values) in [ - (ADULTS, "adults", vec![("min", "18")]), - (RANKED, "ranked", vec![("q", "graph")]), - (LIKED, "liked", vec![]), + for (source, name, params) in [ + (ADULTS, "adults", int_params(&[("min", 18)])), + (RANKED, "ranked", params(&[("q", "graph")])), + (LIKED, "liked", params(&[])), ] { - let params = params(&values); let main = || ReadTarget::branch("main"); let document = db .explain_query(main(), source, name, ¶ms) diff --git a/crates/omnigraph/tests/engine_v2_plan_replay.rs b/crates/omnigraph/tests/engine_v2_plan_replay.rs index 9eadb2e99..292c16a33 100644 --- a/crates/omnigraph/tests/engine_v2_plan_replay.rs +++ b/crates/omnigraph/tests/engine_v2_plan_replay.rs @@ -706,6 +706,8 @@ struct Replayed { plan: omnigraph_planner::BoundPlan, rows: Vec, search: Vec, + /// The run's replay envelope. + envelope: Vec, } /// One inspected run and two replays of its plan through the door (the plan @@ -738,9 +740,19 @@ async fn replayed(db: &Session, source: &str, name: &str, params: &ParamMap) -> plan: run.plan, rows, search, + envelope, } } +/// `envelope` with its plan replaced by `plan` (a serialized `BoundPlan`), +/// as a saved plan edited after its run arrives at the replay door: the +/// run's query, scope and schema with an edited plan. +fn with_plan(envelope: &[u8], plan: Value) -> Vec { + let mut edited: Value = serde_json::from_slice(envelope).unwrap(); + edited["plan"] = plan; + serde_json::to_vec(&edited).unwrap() +} + #[tokio::test] async fn a_hash_join_traversal_replays_with_its_switches() { let dir = tempfile::tempdir().unwrap(); @@ -782,6 +794,7 @@ async fn legacy_two_hop_plan_corrects_assumed_coverage_before_execution() { ); let result = rows_of(&run.result); assert_eq!(result, vec![serde_json::json!({"b.slug":"d01"})]); + let envelope = run.replay_envelope(query, "two_hops"); let mut plan = run.plan; for id in plan.plan.post_order() { if let Some(PhysicalNode::Expand { mode, policy, .. }) = plan.plan.node_mut(id) { @@ -807,8 +820,10 @@ async fn legacy_two_hop_plan_corrects_assumed_coverage_before_execution() { } }); assert_eq!(changed, 1); - let restored = serde_json::from_value(encoded).unwrap(); - let replay = db.replay_bound_plan("main", restored).await.unwrap(); + let replay = db + .replay_bound_plan("main", &with_plan(&envelope, encoded)) + .await + .unwrap(); assert_eq!(rows_of(&replay.result), result); assert!( sides(&report_rows(&replay.report)) @@ -940,26 +955,14 @@ async fn a_marker_that_disagrees_with_its_contains_join_refuses_the_replay() { /// conjunct is refused by acceptance, which finds the predicate gone. #[tokio::test] async fn a_marker_with_no_contains_join_refuses_the_replay() { - use omnigraph_compiler::ir::IRExpr; - use omnigraph_compiler::query::ast::CompOp; - use omnigraph_planner::mirror::ExprMirror; for keep_conjunct in [true, false] { let error = edited_replay_refusal(&mut |object| { if object.get("node") != Some(&Value::from("ContainsJoin")) { return; } - let side = |key: &str| { - let pair = object[key].as_array().unwrap(); - IRExpr::PropAccess { - variable: pair[0].as_str().unwrap().to_string(), - property: pair[1].as_str().unwrap().to_string(), - } - }; - let conjunct = - IRExpr::comparison(side("haystack"), CompOp::StringContains, side("needle")); let mut filters = Vec::new(); if keep_conjunct { - filters.push(serde_json::to_value(ExprMirror::from(&conjunct)).unwrap()); + filters.push(object["conjunct"].clone()); } filters.extend(object["residual"].as_array().cloned().unwrap_or_default()); object.retain(|key, _| matches!(key.as_str(), "node" | "left" | "right")); @@ -970,7 +973,7 @@ async fn a_marker_with_no_contains_join_refuses_the_replay() { let expected = if keep_conjunct { "carries a runtime filter no contains join fills" } else { - "predicate retention: the plan tests no conjunct for `$p.text contains $m.number`" + "predicate retention: the plan tests no conjunct for" }; assert!(error.to_string().contains(expected), "{error}"); } @@ -1015,8 +1018,12 @@ async fn saved_contains_conjunct_refuses_pairing_and_leaf_type_drift() { _ => None, }) .expect("the fixture plans a ContainsJoin"); + let envelope = first.replay_envelope(CITATION_QUERIES, "cited"); let replay = db - .replay_bound_plan("main", restored.clone()) + .replay_bound_plan( + "main", + &with_plan(&envelope, serde_json::to_value(&restored).unwrap()), + ) .await .unwrap(); assert_eq!(rows_of(&replay.result), rows_of(&first.result)); @@ -1087,34 +1094,28 @@ async fn saved_contains_conjunct_refuses_pairing_and_leaf_type_drift() { panic!("the saved join exists"); }; *saved = conjunct; - let altered = serde_json::from_value(serde_json::to_value(altered).unwrap()).unwrap(); - let error = match db.replay_bound_plan("main", altered).await { + let error = match db + .replay_bound_plan( + "main", + &with_plan(&envelope, serde_json::to_value(altered).unwrap()), + ) + .await + { Err(error) => error, Ok(_) => panic!("{label}: inconsistent ContainsJoin conjunct replayed"), }; - let (compiler_refusal, diagnostic) = match label { - "haystack scalar" => (true, "string predicate needs scalar String operands"), - "needle list" => (true, "comparison operands have incompatible list shapes"), - "haystack nullability" => ( - false, - "property p.text leaf type String disagrees with its catalog owner", - ), - "operator" | "swapped columns" | "other haystack" | "other needle" - | "non-comparison" => ( - false, - "contains join conjunct disagrees with its matcher columns", - ), - other => panic!("unclassified corruption {other}"), - }; - if compiler_refusal { - assert!(matches!(&error, OmniError::Compiler(_)), "{label}: {error}"); - } else { - assert!( - matches!(&error, OmniError::Manifest(error) if error.kind == ManifestErrorKind::Internal), - "{label}: {error}" - ); - } - assert!(error.to_string().contains(diagnostic), "{label}: {error}"); + // Acceptance refuses every corruption at the envelope door, before + // execution: whatever replaced it, the written conjunct is gone. + let check = "predicate retention"; + assert!( + matches!(&error, OmniError::Manifest(error) if error.kind == ManifestErrorKind::BadRequest), + "{label}: {error}" + ); + assert!( + error.to_string().contains("the saved plan is not accepted") + && error.to_string().contains(check), + "{label}: {error}" + ); } } @@ -1144,8 +1145,12 @@ async fn saved_ranked_score_refuses_owner_and_type_drift() { .filter_map(|(id, node)| node.ranked().map(|ranked| (id, ranked.kind))) .collect(); assert_eq!(scans.len(), 2); + let envelope = first.replay_envelope(DOC_QUERIES, "fused"); let replay = db - .replay_bound_plan("main", restored.clone()) + .replay_bound_plan( + "main", + &with_plan(&envelope, serde_json::to_value(&restored).unwrap()), + ) .await .unwrap(); assert_eq!(rows_of(&replay.result), rows_of(&first.result)); @@ -1187,8 +1192,13 @@ async fn saved_ranked_score_refuses_owner_and_type_drift() { panic!("the saved ranked scan exists"); }; ranked.score = replacement; - let altered = serde_json::from_value(serde_json::to_value(altered).unwrap()).unwrap(); - let error = match db.replay_bound_plan("main", altered).await { + let error = match db + .replay_bound_plan( + "main", + &with_plan(&envelope, serde_json::to_value(altered).unwrap()), + ) + .await + { Err(error) => error, Ok(_) => panic!("{kind:?} {label}: inconsistent ranked score replayed"), }; @@ -1210,6 +1220,7 @@ async fn a_nearest_ladder_replays_the_same_rungs() { plan, rows, search, + .. } = replayed(&db, DOC_QUERIES, "nearest_with_edge", ¶ms).await; assert_eq!(result.len(), 3); let decided: Vec<(&str, u64)> = search @@ -1297,8 +1308,9 @@ async fn a_hydrating_top_k_replays() { order { $d.slug desc } limit 2 }"#; - let Replayed { result, plan, rows } = - replayed(&db, source, "two_texts", &ParamMap::new()).await; + let Replayed { + result, plan, rows, .. + } = replayed(&db, source, "two_texts", &ParamMap::new()).await; assert_eq!( result, [ @@ -1337,8 +1349,12 @@ async fn the_skip_shapes_and_the_aggregate_replay() { replayed(&db, PEOPLE_QUERIES, "count_by_age", &ParamMap::new()).await; assert_eq!(result.len(), 3); assert!(rows.iter().any(|row| row["operator"] == "AggregateExec")); - let Replayed { result, plan, .. } = - replayed(&db, PEOPLE_QUERIES, "sum_ages", &ParamMap::new()).await; + let Replayed { + result, + plan, + envelope, + .. + } = replayed(&db, PEOPLE_QUERIES, "sum_ages", &ParamMap::new()).await; assert_eq!(result, vec![serde_json::json!({"total": 120.0})]); let explain = plan.plan.to_json(); assert_eq!( @@ -1370,8 +1386,10 @@ async fn the_skip_shapes_and_the_aggregate_replay() { .find(|node| node["node"] == "Aggregate") .unwrap(); node["aggregates"] = replacement; - let bound = serde_json::from_value(altered).unwrap(); - let error = match db.replay_bound_plan("main", bound).await { + let error = match db + .replay_bound_plan("main", &with_plan(&envelope, altered)) + .await + { Err(error) => error, Ok(_) => panic!("{label}: malformed aggregate specification replayed"), }; @@ -1390,6 +1408,7 @@ async fn invalid_aggregate_vector_dimensions_refuse_replay_issue_858() { .query_inspected("main", PEOPLE_QUERIES, "count_by_age", &ParamMap::new()) .await .unwrap(); + let envelope = first.replay_envelope(PEOPLE_QUERIES, "count_by_age"); let encoded = serde_json::to_value(first.plan).unwrap(); for dimension in [0_u32, i32::MAX as u32 + 1, u32::MAX] { let mut altered = encoded.clone(); @@ -1402,8 +1421,10 @@ async fn invalid_aggregate_vector_dimensions_refuse_replay_issue_858() { node["return_exprs"][0]["expr"]["signature"]["arg"] = serde_json::json!({ "kind": "value", "scalar": {"Vector": dimension}, "list": false, "nullable": false }); - let bound = serde_json::from_value(altered).unwrap(); - let error = match db.replay_bound_plan("main", bound).await { + let error = match db + .replay_bound_plan("main", &with_plan(&envelope, altered)) + .await + { Err(error) => error, Ok(_) => panic!("invalid vector dimension {dimension} replayed"), }; @@ -1432,13 +1453,20 @@ async fn metadata_count_with_an_aggregate_signature_refuses_replay_issue_858() { _ => None, }) .unwrap(); + let envelope = count.replay_envelope(PEOPLE_QUERIES, "count_people"); let mut bound = count.plan; let root = bound.plan.root(); let Some(PhysicalNode::MetadataCount { return_exprs, .. }) = bound.plan.node_mut(root) else { panic!("count_people must use MetadataCount"); }; *return_exprs = replacement; - let error = match db.replay_bound_plan("main", bound).await { + let error = match db + .replay_bound_plan( + "main", + &with_plan(&envelope, serde_json::to_value(&bound).unwrap()), + ) + .await + { Err(error) => error, Ok(_) => panic!("MetadataCount accepted a sum signature"), }; @@ -2083,6 +2111,8 @@ async fn a_replay_rechecks_recorded_full_text_coverage() { && error.to_string().contains("full-text coverage"), "{error}" ); +} + #[tokio::test] async fn saved_result_schemas_and_node_declarations_refuse_drift() { let dir = tempfile::tempdir().unwrap(); @@ -2093,10 +2123,13 @@ async fn saved_result_schemas_and_node_declarations_refuse_drift() { .query_inspected("main", query, "nodes", &ParamMap::new()) .await .unwrap(); + let envelope = run.replay_envelope(query, "nodes"); let expected = rows_of(&run.result); let encoded = serde_json::to_value(&run.plan).unwrap(); - let restored = serde_json::from_value(encoded.clone()).unwrap(); - let replay = db.replay_bound_plan("main", restored).await.unwrap(); + let replay = db + .replay_bound_plan("main", &with_plan(&envelope, encoded.clone())) + .await + .unwrap(); assert_eq!(rows_of(&replay.result), expected); let result_node = encoded["body"]["plan"]["slots"] .as_array() @@ -2151,8 +2184,10 @@ async fn saved_result_schemas_and_node_declarations_refuse_drift() { _ => unreachable!(), } } - let bound = serde_json::from_value(altered).unwrap(); - let error = match db.replay_bound_plan("main", bound).await { + let error = match db + .replay_bound_plan("main", &with_plan(&envelope, altered)) + .await + { Err(error) => error, Ok(_) => panic!("{label}: altered result declaration replayed"), }; @@ -2170,6 +2205,7 @@ async fn saved_result_schemas_and_node_declarations_refuse_drift() { .query_inspected("main", PEOPLE_QUERIES, "count_people", &ParamMap::new()) .await .unwrap(); + let envelope = run.replay_envelope(PEOPLE_QUERIES, "count_people"); let root = run.plan.plan.root(); assert!( run.plan @@ -2182,8 +2218,10 @@ async fn saved_result_schemas_and_node_declarations_refuse_drift() { ); let mut encoded = serde_json::to_value(run.plan).unwrap(); encoded["body"]["plan"]["properties"][root]["schema"][0]["nullable"] = serde_json::json!(false); - let malformed = serde_json::from_value(encoded).unwrap(); - let error = match db.replay_bound_plan("main", malformed).await { + let error = match db + .replay_bound_plan("main", &with_plan(&envelope, encoded)) + .await + { Err(error) => error, Ok(_) => panic!("MetadataCount accepted a different declared nullability"), }; @@ -2241,10 +2279,13 @@ query skipped() { .query_inspected("main", queries, name, &ParamMap::new()) .await .unwrap(); + let envelope = run.replay_envelope(queries, name); assert_eq!(run.result.num_rows(), rows, "{name}"); let encoded = serde_json::to_value(&run.plan).unwrap(); - let restored = serde_json::from_value(encoded.clone()).unwrap(); - let replay = db.replay_bound_plan("main", restored).await.unwrap(); + let replay = db + .replay_bound_plan("main", &with_plan(&envelope, encoded.clone())) + .await + .unwrap(); assert_eq!(rows_of(&replay.result), rows_of(&run.result), "{name}"); for (label, ty) in [ ( @@ -2300,9 +2341,8 @@ query skipped() { }) .sum(); assert_eq!(changed, 1, "{name}: the age predicate must reach the scan"); - let bound = serde_json::from_value(altered).unwrap(); let error = db - .replay_bound_plan("main", bound) + .replay_bound_plan("main", &with_plan(&envelope, altered)) .await .err() .unwrap_or_else(|| { @@ -2310,35 +2350,21 @@ query skipped() { "{name}: replay accepted forged property {label}, coherent={coherent}" ) }); - let diagnostic = if coherent { - assert!( - matches!(&error, OmniError::Manifest(error) if error.kind == ManifestErrorKind::Internal), - "{name}/{label}, coherent={coherent}: {error}" - ); - "property p.age leaf type" - } else { - assert!( - matches!(&error, OmniError::Compiler(_)), - "{name}/{label}: {error}" - ); - match label { - "scalar" => "comparison operands need explicit casts to one stored domain", - "nullability" => "stored expression result type violates its local rule", - other => panic!("unclassified corruption {other}"), - } - }; + // Acceptance refuses the forged leaf at the envelope door, + // whether or not its local types cohere: the stored conjunct + // is not the query's typed lowering. assert!( - error.to_string().contains(diagnostic), + matches!(&error, OmniError::Manifest(error) if error.kind == ManifestErrorKind::BadRequest), + "{name}/{label}, coherent={coherent}: {error}" + ); + assert!( + error.to_string().contains( + "the saved plan is not accepted: predicate retention: the plan tests `$p.age >" + ) && error + .to_string() + .contains("with stored types the query does not give it"), "{name}/{label}, coherent={coherent}: {error}" ); - if coherent { - assert!( - error - .to_string() - .contains("disagrees with its catalog owner"), - "{name}/{label}: {error}" - ); - } } } } @@ -2361,12 +2387,15 @@ async fn saved_count_variable_types_refuse_drift_with_metadata_or_empty_aggregat .query_inspected("main", source, name, &ParamMap::new()) .await .unwrap(); + let envelope = run.replay_envelope(source, name); if name == "empty_count" { assert_eq!(rows_of(&run.result), vec![serde_json::json!({"n":0})]); } let mut altered = serde_json::to_value(&run.plan).unwrap(); - let restored = serde_json::from_value(altered.clone()).unwrap(); - let replay = db.replay_bound_plan("main", restored).await.unwrap(); + let replay = db + .replay_bound_plan("main", &with_plan(&envelope, altered.clone())) + .await + .unwrap(); assert_eq!(rows_of(&replay.result), rows_of(&run.result), "{name}"); let node = altered["body"]["plan"]["slots"] .as_array_mut() @@ -2386,9 +2415,8 @@ async fn saved_count_variable_types_refuse_drift_with_metadata_or_empty_aggregat 1 ); assert_eq!(node["return_exprs"][0]["expr"]["signature"], signature); - let bound = serde_json::from_value(altered).unwrap(); let error = db - .replay_bound_plan("main", bound) + .replay_bound_plan("main", &with_plan(&envelope, altered)) .await .err() .unwrap_or_else(|| panic!("{name}: replay accepted a count of a different node type")); @@ -2417,11 +2445,11 @@ async fn saved_sort_alias_type_refuses_drift_even_when_limit_skips_input() { .query_inspected("main", source, "sorted", &ParamMap::new()) .await .unwrap(); + let envelope = run.replay_envelope(source, "sorted"); assert_eq!(run.result.num_rows(), 0); let mut altered = serde_json::to_value(&run.plan).unwrap(); - let restored = serde_json::from_value(altered.clone()).unwrap(); assert_eq!( - db.replay_bound_plan("main", restored) + db.replay_bound_plan("main", &with_plan(&envelope, altered.clone())) .await .unwrap() .result @@ -2444,13 +2472,21 @@ async fn saved_sort_alias_type_refuses_drift_even_when_limit_skips_input() { ), 1 ); - let bound = serde_json::from_value(altered).unwrap(); let error = db - .replay_bound_plan("main", bound) + .replay_bound_plan("main", &with_plan(&envelope, altered)) .await .err() .expect("a sort alias must agree with its retained return type"); - assert!(error.to_string().contains("alias"), "{error}"); + assert!( + matches!(&error, OmniError::Manifest(error) if error.kind == ManifestErrorKind::BadRequest), + "{error}" + ); + assert!( + error.to_string().contains( + "the saved plan is not accepted: order: the final sort orders by years desc with stored types" + ), + "{error}" + ); } #[tokio::test] @@ -2473,13 +2509,18 @@ async fn saved_temporal_params_keep_fresh_shape_refusals_on_empty_input() { .query_inspected("main", &source, "temporal", ¶ms) .await .unwrap(); + let envelope = run.replay_envelope(&source, "temporal"); assert_eq!( run.result.num_rows(), if predicate.ends_with("100") { 0 } else { 3 } ); - let restored = - serde_json::from_value(serde_json::to_value(&run.plan).unwrap()).unwrap(); - let replay = db.replay_bound_plan("main", restored).await.unwrap(); + let replay = db + .replay_bound_plan( + "main", + &with_plan(&envelope, serde_json::to_value(&run.plan).unwrap()), + ) + .await + .unwrap(); assert_eq!(rows_of(&replay.result), rows_of(&run.result)); let invalid = Literal::List(vec![Literal::Null]); let fresh = db @@ -2494,9 +2535,11 @@ async fn saved_temporal_params_keep_fresh_shape_refusals_on_empty_input() { .expect("fresh binding refuses the invalid temporal list"); let mut altered = run.plan; std::sync::Arc::make_mut(&mut altered.values.params).insert("values".into(), invalid); - let bound = serde_json::from_value(serde_json::to_value(altered).unwrap()).unwrap(); let replay = db - .replay_bound_plan("main", bound) + .replay_bound_plan( + "main", + &with_plan(&envelope, serde_json::to_value(altered).unwrap()), + ) .await .err() .expect("replay must preserve the fresh temporal parameter contract"); @@ -2526,11 +2569,11 @@ async fn saved_return_leaf_types_refuse_drift_when_limit_skips_input() { .query_inspected("main", source, "returned", ¶ms) .await .unwrap(); + let envelope = run.replay_envelope(source, "returned"); assert_eq!(run.result.num_rows(), 0); let encoded = serde_json::to_value(&run.plan).unwrap(); - let restored = serde_json::from_value(encoded.clone()).unwrap(); assert_eq!( - db.replay_bound_plan("main", restored) + db.replay_bound_plan("main", &with_plan(&envelope, encoded.clone())) .await .unwrap() .result @@ -2560,10 +2603,10 @@ async fn saved_return_leaf_types_refuse_drift_when_limit_skips_input() { projection["ty"], declared, "only the expression leaf changes" ); - match serde_json::from_value::(altered) { + match serde_json::from_value::(altered.clone()) { Err(_) => {} - Ok(bound) => { - db.replay_bound_plan("main", bound) + Ok(_) => { + db.replay_bound_plan("main", &with_plan(&envelope, altered)) .await .err() .unwrap_or_else(|| panic!("{column}: a skipped return accepted a leaf type different from its result type")); diff --git a/crates/omnigraph/tests/forbidden_apis.rs b/crates/omnigraph/tests/forbidden_apis.rs index bf3c6f7d0..7442e1bf7 100644 --- a/crates/omnigraph/tests/forbidden_apis.rs +++ b/crates/omnigraph/tests/forbidden_apis.rs @@ -755,7 +755,6 @@ gateway_surfaces! { "scan_proven_insert_delta_bounded", "include_proven_insert_blob_selection", "scan_stream", "scan_stream_bounded", "scan_stream_with", "scan_plan_with", "scan_plan_with_filter", "ordered_scan_error", "scan", "scan_with", - "fts_coverage", "count_rows", "dataset_version", "table_state", "scan_with_staged", "scan_with_pending", "scan_with_pending_materialized_blobs", "count_rows_with_staged", From 5963fadfb083c3d8002f2848a6b4dba0063f602b Mon Sep 17 00:00:00 2001 From: Ragnor Comerford Date: Sun, 11 Oct 2026 10:01:26 -0700 Subject: [PATCH 26/29] docs: record step 2a on the typed lowering RFC 0047 states the requirements from the typed lowering, two-sided predicate checks, traversal identity, the single acceptance gate, the narrow_key and hydrate_return_columns rules, full-text coverage from index facts and the replay versions, with a decision-log entry. The execution, testing and explain guides and the bm25 release note follow. --- .../bm25-scores-ignore-filters.fixed.md | 5 +- docs/dev/execution.md | 76 ++++++++++++------ docs/dev/testing.md | 2 +- docs/rfcs/0047-search-plan-truth.md | 80 +++++++++++++++++-- docs/user/queries/explain.md | 6 +- 5 files changed, 131 insertions(+), 38 deletions(-) diff --git a/changelog.d/bm25-scores-ignore-filters.fixed.md b/changelog.d/bm25-scores-ignore-filters.fixed.md index a0b73228d..3a4054db2 100644 --- a/changelog.d/bm25-scores-ignore-filters.fixed.md +++ b/changelog.d/bm25-scores-ignore-filters.fixed.md @@ -1,6 +1,7 @@ - A `bm25()` score no longer depends on the query's filters when rows were - written after the property's full-text index was built: such a scan now - applies its filter after scoring. See the [explain + written after the property's full-text index was built, or when the index + cannot confirm that it covers every row: such a scan now applies its filter + after scoring. See the [explain guide][bm25-eligibility-explain]. [bm25-eligibility-explain]: ../docs/user/queries/explain.md diff --git a/docs/dev/execution.md b/docs/dev/execution.md index eb74cf511..e9e6aabd6 100644 --- a/docs/dev/execution.md +++ b/docs/dev/execution.md @@ -697,37 +697,56 @@ run, `Session::explain_query`, the `explain` statement and the inspected run share the path, so explain validates exactly what a run validates and reports the outcome as `validation.scope`. -The requirements come from the declaration, its type context and the -catalog, not from the IR: binding identity, search identity and its declared -approximation, every eligibility conjunct (checked per `rrf` arm, since each -arm copies the match), correlated blocks, the projection and the origin of -every projected score, order and cut, and the policies the plan declares -for its search. Every node declares its count cut (`PhysicalNode::row_cut`, -a match over every variant: a `Limit`'s or `Page`'s rows, a `Sort`'s -`fetch`, a `RankFuse`'s limit, a ranked scan's candidate cap), and the +Acceptance first checks that the plan is well formed: aggregate +specifications against their typed signatures +(`aggregate::validate_aggregate_specs`), every node's output schema against +what its operator derives (`output::validate_output_schemas`), and a fusion's +row keys (`optimizer::validate_rank_fuse_row_tiebreaks`). Execution keeps +only the checks that need the snapshot or the bound values +(`validate_traversal_admission`, `typed_value::validate_plan_values`). + +The requirements come from the compiler's typed lowering of the checked +declaration (`AcceptInput.ir`), checked first against the declaration's +counts of traversals, blocks, return items, order keys and limit. Every +`QueryIR` expression carries its type and every conversion is an explicit +`Cast`, so the plan's expressions compare with the lowering's by equality, +types included; the declaration supplies only which return items and order +key are the written score calls. The checks: binding identity and traversal +identity (endpoints, edge selection, direction, endpoint types, hop bounds), +search identity and its declared approximation, eligibility in both +directions (every lowered conjunct is tested, checked per `rrf` arm since +each arm copies the match, and every tested conjunct is a lowered one, a +constant `true`, or the identity equality key narrowing derives through +`AcceptInput.keys`), correlated blocks, the exact projection and the origin +of every projected score, order and cut, late materialization +(`HydrateColumns`), and the policies the plan declares for its search. +Every node declares its count cut (`PhysicalNode::row_cut`, a match over +every variant: a `Limit`'s or `Page`'s rows, a `Sort`'s `fetch`, a ranked +scan's candidate cap; a fusion and a hydration cut nothing), and the validator justifies each one by its position: the root `Limit` and the final -`Sort` (below the root `Limit` and its projections) at the query's `limit`, a -fusion at that `limit`, and a ranked scan's cap as its declared policy sets -it. Any other cut, a capped sort below the final one or a cut in a -correlated block's tree, is refused as `row cut`. A matcher relates each written expression to the IR through -the compiler's lowering patterns (property leaves on physical columns, `in` -as a swapped `contains`, a bare search call as `= true`, a folded constant as -the value the engine's evaluator gives it). The order check recomputes the -root's order from the operators (`optimizer::derived_order`, typed -`OrderKey`s) and accepts the planner's tie-break omissions that are -comparator equivalences: a key already in the comparator, every key when the -query returns only order keys, and identities of bindings the query does not -name (anonymous endpoints, cycle temps) after every named one. +`Sort` (below the root `Limit`, its projections and a hydration) at the +query's `limit`, and a ranked scan's cap as its declared policy sets it. Any +other cut, a capped sort below the final one or a cut in a correlated +block's tree, is refused as `row cut`. The order check recomputes the root's +order from the operators (`optimizer::derived_order`, typed `OrderKey`s): +a fusion's sort leads with `._rrf` descending, a search-ordered +aggregate's with its remaining keys and then every group key it did not +write, and the planner's tie-break omissions that are comparator +equivalences are accepted: a key already in the comparator, every key when +the query returns only order keys or constants, and identities of bindings +the query does not name (anonymous endpoints, cycle temps) after every named +one. A query of one binding, Boolean/integer/String eligibility, an optional leading `bm25()` over a String property, direct property sort keys, a limit and a projection of properties and the selected score is a member of the exact fragment (`validate/subset.rs`). For a member the optimizer records the rules it applies (`AbsorbScanFilter`, `PruneScanColumns`, `Lower`, -`RankBm25Scan`) over arena references; the validator checks that the IR is -the declaration's lowering, rebuilds the canonical chain -`Limit(Sort(Project([Search](Filter(Scan)))))`, re-applies every rule -against its precondition and requires the result to equal the plan. The +`RankBm25Scan`, `HydrateReturnColumns`, `NarrowKey`) over arena references; +the validator rebuilds the canonical chain +`Limit(Sort(Project([Search](Filter(Scan)))))` from the typed lowering, +re-applies every rule against its precondition and requires the result to +equal the plan. The checker's arena keeps only each role's current node, releasing a replaced one before storing its successor, so its memory is linear in the query. A member is accepted as `exact_subset`; every other plan as `invariants_only`. @@ -742,8 +761,13 @@ derivation. It checks the byte limit and the envelope's versions before decoding the plan, refuses another schema and a dataset the plan did not pin as conflicts, re-establishes every full-text coverage fact the plan records from the pinned snapshot, recompiles the query, and accepts the plan again; a -recorded fact the snapshot contradicts, or a plan that fails a check, is -invalid evidence, a bad request. +recorded fact the snapshot contradicts, or a plan that fails a check +(well-formedness included), is invalid evidence, a bad request. + +Full-text coverage is derived from the index facts the planner reads for the +pinned dataset (`FullTextCoverage::of_facts`): `absent` without an inverted +index on the column, `full` only when every inverted segment there reports a +known coverage of every fragment, `partial` otherwise. ## Search and rank diff --git a/docs/dev/testing.md b/docs/dev/testing.md index ae659c4e8..ab6b08138 100644 --- a/docs/dev/testing.md +++ b/docs/dev/testing.md @@ -50,7 +50,7 @@ The engine integration suite is grouped by behavior, not implementation module: | Frozen engine v1, the reference | `crates/omnigraph-reference-engine/tests/frozen.rs` pins every source file's bytes; `forbidden_apis.rs` owns that `omnigraph-gqt` is the crate's only dependent and that its source names neither the engine nor the planner; the crate has no other tests, and a `.gqt` step's `--- expect same as v1` compares v2's rows with its answer | | V2 plan nodes' operators and the execution report | In-source `engine/report/tests.rs` owns which operator each node builds and what its row says (`ran`, `actual_rows`, `drained`); `crates/omnigraph-planner/tests/lower_walk.rs` owns the walk's order and refusals; the `ran` lines of `--- expect plan` read the report of the case's own run | | V2 plan sufficiency: the bound plan alone reproduces a run | `crates/omnigraph/tests/engine_v2_plan_replay.rs` (see [Plan replay](#plan-replay)): one query per node kind with a switch or a ladder, replayed twice through `Session::replay_bound_plan` from its serialized replay envelope, equal rows, the same trace and the same search decisions required; `crates/omnigraph-planner/tests/bound_plan.rs` owns the serialized form; in-source `engine/search.rs` tests own the declared ladder's stepping; `engine/plan_source.rs` tests own that the assumed memory limit sizes the plan, the lowering and the pool | -| V2 plan acceptance: every read plan checked against its checked query | In-source `crates/omnigraph-planner/src/validate/tests.rs` owns one broken plan per check (binding and search identity, predicate retention, row cut, order and tie-break, projection and score origin, declared policies, full-text prerequisites), exact-fragment membership over admitted and excluded forms, the derivation's rules and rejections, entry parity under an exhausted limit, and the `instrument:` derivation cost curve; `validate/replay.rs` and `validate/budget.rs` tests own decoding limits, version refusals and budget charges; `engine_v2.rs::every_read_door_shares_one_acceptance` owns that the run, both explain doors and the inspected run accept one plan and refuse alike; GQT cases claim the scope with `validation scope ` in `--- expect plan` | +| V2 plan acceptance: every read plan checked against its checked query | In-source `crates/omnigraph-planner/src/validate/tests.rs` owns one broken plan per check (binding, traversal and search identity, predicate retention and origin, row cut, order and tie-break including fused and grouped searches, projection and score origin, hydration, declared policies, full-text prerequisites), the key-narrowing conjunct admitted only with its re-derived id, exact-fragment membership over admitted and excluded forms, the derivation's rules and rejections (hydrated members included), entry parity under an exhausted limit, and the `instrument:` derivation cost curve; `engine/plan_source/key_tests.rs` owns the narrowed key on a real snapshot and its refusal when edited; `engine_v2_plan_replay.rs` owns that a saved plan failing well-formedness or a requirement is a bad request at the replay door; `validate/replay.rs` and `validate/budget.rs` tests own decoding limits, version refusals and budget charges; `engine_v2.rs::every_read_door_shares_one_acceptance` owns that the run, both explain doors and the inspected run accept one plan and refuse alike; GQT cases claim the scope with `validation scope ` in `--- expect plan` | | V2 scrubbed replay: no input beside the bound plan and the snapshot reaches execution | `engine_v2_scrubbed_replay.rs` owns the mechanism (the `std::env` grep over `crates/omnigraph/src/engine/`, the replay under an ambient memory limit and an ambient `OMNIGRAPH_EXPAND_INDEXED_MAX_FRONTIER` the plan did not capture, `ann_nprobes` as a plan field, the `profile` rows); `engine_v2_plan_replay.rs` owns the snapshot pins (an edge write or an insert after planning refuses the replay); the inventory cases `cases/v2/planner/input_*.gqt` own one input class each (`engine`, `ann_nprobes`, index build state), and the clock class is owned by `engine_v2_plan_replay.rs`; `engine_v2_memory.rs` owns `ran id_lookup` under a refused build | | V2 declared switches: the report names the side a run took | `cases/v2/planner/switch_ran_on_the_report.gqt` (the `ran` line of `--- expect plan`); in-source `engine/report/tests.rs` owns the switch's row and a scan whose plan declares no fallback | | Search and physical indexes | `search.rs`, `scalar_indexes.rs`, `lance_surface_guards.rs`, `rrf_prefilter_gate.rs` (the rrf plan gate's differential oracle and fences), `repro_issue_563.rs` (`#[ignore]`d overflow-scale symptom tier) | diff --git a/docs/rfcs/0047-search-plan-truth.md b/docs/rfcs/0047-search-plan-truth.md index fca0c5bd8..63fa6ff0e 100644 --- a/docs/rfcs/0047-search-plan-truth.md +++ b/docs/rfcs/0047-search-plan-truth.md @@ -7,7 +7,7 @@ implementation: in-progress authors: - Ragnor Comerford (@ragnorc) created: 2026-09-01 -updated: 2026-10-05 +updated: 2026-10-10 discussion: "https://github.com/ModernRelay/omnigraph/pull/791" supersedes: [] superseded_by: [] @@ -403,6 +403,18 @@ its aggregate, comparison and right-hand side. Its inner predicates are checked through the block's plan subtree, not derived from the declaration's inner clauses, whose variables the lowering may rename. +As built, requirements come from the compiler's typed lowering of the +checked declaration, which acceptance receives beside it. Since every +`QueryIR` expression carries its compiler type and every conversion is an +explicit `Cast`, the lowering keeps the meaning the paragraph above feared +it would lose, so requirements compare by structural equality, types +included: a conjunct, traversal, block, search argument, return item and +order key in the plan must equal the lowering's. The lowering is itself +checked against the declaration first; its traversal, block, return item, +order key and limit counts must be the declaration's. The declaration +supplies only what the lowering does not name: which return items and which +order key are the written score calls. + For a search-ordered aggregate, derive population requirements before grouping and order/cut requirements after grouping. With no remaining order key, group order stays unspecified as defined above. Full-text index readiness comes @@ -425,13 +437,33 @@ fact](#full-text-index-presence-as-a-planning-fact), not from query syntax. (`optimizer::derived_order` over typed `OrderKey`s) rather than reading a declared property. Three planner omissions in the identity tie-break are comparator equivalences and accepted: an identity already in the - comparator, every identity when the query returns only order keys (rows - that tie are indistinguishable), and the identities of bindings the query - does not name (anonymous endpoints, cycle temps), after every named one. + comparator, every identity when the query returns only order keys or + constants (rows that tie are indistinguishable), and the identities of + bindings the query does not name (anonymous endpoints, cycle temps), + after every named one. 3. Check required properties and recorded prerequisites before accepting the plan. Cost estimates select among accepted candidates and cannot discharge a missing requirement. +As built, predicate checks are two-sided. Retention requires every conjunct +of the lowering to be tested somewhere in the pipelines the rows come from; +origin requires every tested conjunct to be one the lowering requires, a +constant `true`, or the identity equality that key narrowing derives (see the +rule catalogue below). A plan that adds a predicate the query never wrote is +therefore refused as surely as one that drops a written predicate. Each +traversal keeps its identity: written endpoints, edge selection, direction, +endpoint types and hop bounds. + +Acceptance is the only gate. Before the requirement checks, it checks that +the plan is well formed: aggregate specifications agree with their typed +signatures, every node's output schema agrees with what its operator +derives, and a fusion's row keys are complete and canonical. Execution keeps +only the checks that need the pinned snapshot or the bound values (traversal +admission, literal and parameter conversion, stored-schema agreement). A +saved plan that fails any acceptance check is a bad request (400) at the +replay door, before execution; a fresh plan that fails one is a planner +defect. + Property checks do not by themselves prove row-selection equivalence. A plan with a dropped filter can have the same schema, ordering and row count. The stronger check below covers transformations within its stated subset. @@ -520,6 +552,16 @@ placing a projection) and `rank_bm25_scan` (the BM25 lowering row, with the eligibility placement below). The canonical sort already carries `fetch=n`, so no top-k rule applies, and the planner removes no Sort. +Rules version 2 adds two rules. `narrow_key` lets the scan of a binding with +a written equality on its type's single String key also test the identity +equality `@id = `, where `` is the canonical id the loader derives +from that key value; acceptance derives the id again through the catalog and +admits no other conjunct by this rule. `hydrate_return_columns` is late +materialization under a positive limit: the scan defers returned columns no +other node reads and carries `_rowaddr` in their place, and a +`HydrateColumns` root over the limit reads them from the same table by row +address. + `T` is a finite, versioned table from the admitted predicate's typed operators to typed scan-filter operators. Literal/parameter and property leaves retain their exact type, value and resolved identity; a comparison keeps its opcode @@ -667,8 +709,9 @@ schema identities and snapshot facts remain fixed through execution. As built, the replay envelope carries the query source and name, the scope, the digest of the accepted schema, the bound plan (whose values are the -captured parameters) and a member's derivation, under `replay_version`, -`rules_version` and `semantics_version` 1. A version mismatch, a different +captured parameters) and a member's derivation, under `replay_version` 1, +`rules_version` 2 and `semantics_version` 2 (the typed lowering and the +step 5 total order). A version mismatch, a different schema digest and a dataset the plan did not pin are conflicts (409) that ask for the query again; malformed evidence or a plan failing a check is a bad request (400); exhaustion is `ResourceLimitExceeded` (413). The recorded @@ -742,6 +785,13 @@ goes through the recording wrapper, so the fact is part of the plan's other planner input. Index coverage of newer rows is not consulted: an uncovered tail is not a refusal. +As built, the fact is derived from the planner's index facts for the pinned +dataset, the same facts key narrowing and index selection read: `absent` +when the property has no inverted index, `full` only when every inverted +segment on the column reports known coverage of every fragment, and +`partial` otherwise. Unknown coverage is never `full`. Replay derives it +again from the pinned dataset. + The `T27` rule is the catalog predicate index reconciliation already uses (a non-enum, single-column String `@index`), moved into the compiler so the type checker and reconciliation cannot drift. @@ -972,7 +1022,7 @@ and regressions. Step 7 can ship independently. | 2a | Shared query/explain validation and acceptance; exact-subset rules, adaptive-policy checks, bounded/versioned replay evidence, GQT scope assertions. The order checks of a fusion and of a search-ordered aggregate land with step 5, which implements their order | supports steps 3-6 | | 3 | `T27` and `FullTextIndexRequired` | #747 | | 4 | The ranked binding roots its component; the `rrf()` arms type error | #789 | -| 5 | One total order: fused score column, `Sort` over fusion and search-ordered aggregates, `T37` retired | #787, #788 | +| 5 | One total order: fused score column, `Sort` over fusion and search-ordered aggregates, `T37` retired, `T54` (PR #938, merged); its order checks land with step 2a | #787, #788 | | 6 | Read descriptors with producer ids and fusion-arm references | None | | 7 | Served read floor | #752 | | 8 | Engine v2 reverses a ranked component when selectivity and its mode-specific validation rule admit it | None (performance) | @@ -1048,3 +1098,19 @@ None. and acceptance justifies each declared cut by its position (root `Limit` and final sort `fetch` at the query's `limit`, the fusion's limit, a ranked scan's policy cap) and refuses every other. +- 2026-10-10: step 2a brought onto the typed lowering, index facts, key + narrowing, late materialization and step 5. Requirements now come from + the compiler's typed lowering and compare by equality, types included; + the declaration supplies only the written score calls. Predicate checks + became two-sided (an added conjunct is refused, not only a dropped one), + and traversals keep their hop bounds, direction and endpoint types. + Acceptance is the single gate: the plan well-formedness checks the typed + lowering added to execution run first in acceptance, and a saved plan + failing them is a bad request at the replay door. Two rules joined the + catalogue (`narrow_key`, `hydrate_return_columns`; rules version 2), and + order checks for a fusion and a search-ordered aggregate replaced their + step 2a exemptions (semantics version 2); a fusion no longer cuts rows, + so acceptance justifies no fusion cut. Full-text coverage is derived + from the index facts, with unknown coverage never full. A constant return + item joined the tie-break equivalences, and an `order` clause whose keys + all fold to constants still requires the sort by the identity tie-break. diff --git a/docs/user/queries/explain.md b/docs/user/queries/explain.md index 631170d49..8c278c465 100644 --- a/docs/user/queries/explain.md +++ b/docs/user/queries/explain.md @@ -132,8 +132,10 @@ value); `settings`, each setting it read with its spelling (`traversal`, resolved (`OMNIGRAPH_EXPAND_INDEXED_MAX_FRONTIER`, `OMNIGRAPH_EXPAND_INDEXED_MAX_HOPS`, when the query traverses); `gate_policy`, the `rrf_plan` mode and the prefilter gate's admission thresholds; -`full_text`, the full-text coverage (`full`, `partial` or `absent`) of every -property a `bm25` scan ranks, keyed `
.`; and +`full_text`, the full-text coverage of every property a `bm25` scan ranks, +keyed `
.` (`full` only when the index is known to cover every +fragment, `partial` when coverage is incomplete or unknown, `absent` without a +full-text index); and `memory_limit`, the query pool in bytes. Selector statements also capture `traversal_work_limit`, shared across the execution, and `has_wildcard_traversal`, retained through rewrites so historical replay can enforce the target rule. Query From 83c2b732b9d91b8d347cacb77b51d50f365d17a8 Mon Sep 17 00:00:00 2001 From: Ragnor Comerford Date: Sun, 11 Oct 2026 10:01:52 -0700 Subject: [PATCH 27/29] test(gqt): keep the connected-rrf P005 refusal in the corpus An rrf() whose arms rank two bindings one traversal connects is refused as P005 in both declaration orders; the case now asserts the public error for each order, beside the existing P001 step. --- ..._an_unranked_binding_and_a_destination.gqt | 23 +++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/crates/omnigraph-gqt/cases/v2/planner/search_order_on_an_unranked_binding_and_a_destination.gqt b/crates/omnigraph-gqt/cases/v2/planner/search_order_on_an_unranked_binding_and_a_destination.gqt index ddbec1fb0..4bbee622b 100644 --- a/crates/omnigraph-gqt/cases/v2/planner/search_order_on_an_unranked_binding_and_a_destination.gqt +++ b/crates/omnigraph-gqt/cases/v2/planner/search_order_on_an_unranked_binding_and_a_destination.gqt @@ -1,5 +1,6 @@ # issue: none # notes: A search-only order over a pipeline with a second, unranked binding sorts by the score and tie-breaks on every scope binding's id, which the planner reads for the scan; a search order on a traversal destination is refused at planning with the shape named and planner code P001. +# notes: An `rrf()` whose arms rank two bindings one traversal connects is refused as P005 in both declaration orders: no declaration order lets both arms start the component, so the refusal offers no reorder. # red_on: 2026-09-23, task 0226 tree before the one-pass tie-break: step 1 failed with `the planned tie-break column 'z.__id' is not in the sort input` (the tie-break was derived at lowering, after projection pushdown); step 2 failed with `column 'd._distance' not found for ordering` instead of a planning refusal. --- runner @@ -32,6 +33,28 @@ edge Knows: Person -> Doc {"edge":"Knows","from":"p1","to":"d003"} {"edge":"Knows","from":"p1","to":"d004"} +--- query +query connected_rrf_d_first($q: Vector(4)) { + match { $d: Doc $t: Doc $p: Person $p knows $d $p knows $t } + return { $d.slug, $t.slug } + order { rrf(nearest($d.embedding, $q), nearest($t.embedding, $q)) } + limit 1 +} +--- params +{"q": [0.0, 0.0, 0.0, 0.0]} +--- expect error: plan error: P005: + +--- query +query connected_rrf_t_first($q: Vector(4)) { + match { $t: Doc $d: Doc $p: Person $p knows $d $p knows $t } + return { $d.slug, $t.slug } + order { rrf(nearest($d.embedding, $q), nearest($t.embedding, $q)) } + limit 1 +} +--- params +{"q": [0.0, 0.0, 0.0, 0.0]} +--- expect error: plan error: P005: + --- query set engine = v2; query search_only_order_with_an_unranked_binding($q: Vector(4)) { From f7bbfc573ed1e92cedbe46612d87d2cc482d38a0 Mon Sep 17 00:00:00 2001 From: Ragnor Comerford Date: Mon, 12 Oct 2026 14:43:51 -0700 Subject: [PATCH 28/29] planner: check predicate origin above a fusion, not only inside its arms A fused plan's arms each copy the match, so retention stays per arm; but the nodes between the fusion and the root were checked by nothing, and a saved plan could filter fused rows there and pass acceptance. Every arm's pipeline now carries the nodes above the fusion for predicate origin, traversal identity and bindings, and a correlated block above the fusion is refused. `a_fusion_replays` forges a `Filter(false)` above the fusion and requires the replay door's refusal. --- .../src/validate/invariants.rs | 101 +++++++++++++----- .../integration/engine_v2_plan_replay.rs | 58 +++++++++- 2 files changed, 134 insertions(+), 25 deletions(-) diff --git a/crates/omnigraph-planner/src/validate/invariants.rs b/crates/omnigraph-planner/src/validate/invariants.rs index 76d0b4cb7..c0653f8c7 100644 --- a/crates/omnigraph-planner/src/validate/invariants.rs +++ b/crates/omnigraph-planner/src/validate/invariants.rs @@ -55,24 +55,57 @@ fn pipeline( Ok(nodes) } +/// One pipeline a query's eligibility constrains: `own`, the nodes that +/// produce its rows, and `after`, the nodes those rows pass through above a +/// fusion (empty without one). +struct Constrained { + own: Vec, + after: Vec, +} + +impl Constrained { + /// Every node the pipeline's rows reach. + fn reached(&self) -> Vec { + self.own.iter().chain(&self.after).copied().collect() + } +} + /// The pipelines whose rows a query's eligibility constrains: one per `rrf` -/// arm, each its own copy of the match, or the whole top level. +/// arm, each its own copy of the match, followed by the nodes above the +/// fusion; or the whole top level. fn constrained_pipelines( plan: &PhysicalPlan, budget: &mut Budget, -) -> Result>, ValidationError> { +) -> Result, ValidationError> { let top = pipeline(plan, plan.root(), budget)?; let fuse = top.iter().find_map(|id| match plan.node(*id) { - Some(PhysicalNode::RankFuse { arms, .. }) => Some(arms.clone()), + Some(PhysicalNode::RankFuse { arms, .. }) => Some((*id, arms.clone())), _ => None, }); - match fuse { - Some(arms) => arms - .iter() - .map(|arm| pipeline(plan, arm.input, budget)) - .collect(), - None => Ok(vec![top]), + let Some((fuse, arms)) = fuse else { + return Ok(vec![Constrained { + own: top, + after: Vec::new(), + }]); + }; + let mut below = HashSet::new(); + let mut arm_pipelines = Vec::new(); + for arm in &arms { + let own = pipeline(plan, arm.input, budget)?; + below.extend(own.iter().copied()); + arm_pipelines.push(own); } + let after: Vec = top + .into_iter() + .filter(|id| *id != fuse && !below.contains(id)) + .collect(); + Ok(arm_pipelines + .into_iter() + .map(|own| Constrained { + own, + after: after.clone(), + }) + .collect()) } /// Every conjunct a pipeline's nodes test, wherever placement put it, with @@ -154,11 +187,26 @@ impl Requirements { input: &AcceptInput<'_>, budget: &mut Budget, ) -> Result<(), ValidationError> { - let pipelines = constrained_pipelines(plan, budget)?; - for nodes in &pipelines { - self.check_bindings(plan, nodes)?; - self.check_eligibility(plan, nodes, input, budget)?; - self.check_blocks(plan, nodes)?; + // A fused query's eligibility belongs inside each arm (retention); + // what its rows pass through after the fusion is checked with the + // arm for origin, extra traversals and blocks. + for pipeline in &constrained_pipelines(plan, budget)? { + let reached = pipeline.reached(); + self.check_bindings(plan, &reached)?; + self.check_eligibility(plan, &pipeline.own, &reached, input, budget)?; + self.check_blocks(plan, &pipeline.own)?; + if let Some(id) = pipeline + .after + .iter() + .find(|id| matches!(plan.node(**id), Some(PhysicalNode::AntiJoin { .. }))) + { + return Err(ValidationError::violated( + "predicate origin", + format!( + "node {id} decides a correlated block above the fusion, outside every arm" + ), + )); + } } let top = pipeline(plan, plan.root(), budget)?; self.check_search(plan)?; @@ -269,28 +317,33 @@ impl Requirements { } /// The conjuncts a pipeline tests are exactly the ones the query - /// requires: every required conjunct is tested somewhere in it, and every - /// conjunct it tests is required, a constant `true` (which constrains - /// nothing) or the canonical identity of a written key equality on the - /// same scan. + /// requires: every required conjunct is tested among `retained` (an + /// `rrf` arm's own nodes), and every conjunct tested anywhere its rows + /// reach, after a fusion included, is required, a constant `true` (which + /// constrains nothing) or the canonical identity of a written key + /// equality on the same scan. fn check_eligibility( &self, plan: &PhysicalPlan, - nodes: &[NodeId], + retained: &[NodeId], + reached: &[NodeId], input: &AcceptInput<'_>, budget: &mut Budget, ) -> Result<(), ValidationError> { - let placed = conjuncts(plan, nodes); - budget.visit(u64::try_from(placed.len() + self.eligibility.len()).unwrap_or(u64::MAX))?; - let placed_set: HashSet<&IRExpr> = placed.iter().map(|(_, conjunct)| conjunct).collect(); + let kept = conjuncts(plan, retained); + let placed = conjuncts(plan, reached); + budget.visit( + u64::try_from(kept.len() + placed.len() + self.eligibility.len()).unwrap_or(u64::MAX), + )?; + let kept_set: HashSet<&IRExpr> = kept.iter().map(|(_, conjunct)| conjunct).collect(); if let Some(missing) = self .eligibility .iter() - .find(|required| !placed_set.contains(required)) + .find(|required| !kept_set.contains(required)) { // The same text under other stored types is a retyped conjunct. let text = missing.to_string(); - let detail = if placed + let detail = if kept .iter() .any(|(_, conjunct)| conjunct.to_string() == text) { diff --git a/crates/omnigraph/tests/integration/engine_v2_plan_replay.rs b/crates/omnigraph/tests/integration/engine_v2_plan_replay.rs index 0d2969ab7..1a213eaa3 100644 --- a/crates/omnigraph/tests/integration/engine_v2_plan_replay.rs +++ b/crates/omnigraph/tests/integration/engine_v2_plan_replay.rs @@ -1262,7 +1262,13 @@ async fn a_fusion_replays() { let db = docs(&dir).await; let mut params = ParamMap::from([("t".to_string(), Literal::String("needle".to_string()))]); params.insert("q".to_string(), Literal::List(vec![Literal::Float(0.0); 4])); - let Replayed { result, search, .. } = replayed(&db, DOC_QUERIES, "fused", ¶ms).await; + let Replayed { + result, + search, + plan, + envelope, + .. + } = replayed(&db, DOC_QUERIES, "fused", ¶ms).await; assert_eq!(result.len(), 3); let kinds: Vec<&str> = search .iter() @@ -1274,6 +1280,56 @@ async fn a_fusion_replays() { "the fusion's gate records one verdict, then its nearest arm its probe attempts: {search:#?}" ); + // A predicate the query never wrote, placed above the fusion rather than + // in an arm, is refused at the replay door instead of dropping rows. + let mut forged = plan.clone(); + let fuse = forged + .plan + .live() + .find(|(_, node)| matches!(node, PhysicalNode::RankFuse { .. })) + .map(|(id, _)| id) + .expect("the fused query plans a RankFuse"); + let parent = forged + .plan + .live() + .find(|(_, node)| node.inputs().contains(&fuse)) + .map(|(id, _)| id) + .expect("the fusion feeds a parent"); + let properties = forged.plan.properties(fuse).cloned().unwrap(); + let never = forged.plan.add(PhysicalNode::Filter { + input: fuse, + filters: vec![omnigraph_compiler::ir::IRExpr::Literal( + Literal::Bool(false), + omnigraph_compiler::ExprType::from_prop(&omnigraph_compiler::PropType::scalar( + omnigraph_compiler::ScalarType::Bool, + false, + )), + )], + }); + forged.plan.set_properties(never, properties); + match forged.plan.node_mut(parent) { + Some(PhysicalNode::Sort { input, .. }) => *input = never, + other => panic!("the fusion's parent is a sort: {other:?}"), + } + let error = match db + .replay_bound_plan( + "main", + &with_plan(&envelope, serde_json::to_value(&forged).unwrap()), + ) + .await + { + Err(error) => error, + Ok(replay) => panic!( + "a filter above the fusion replayed: {:?}", + rows_of(&replay.result) + ), + }; + assert!( + matches!(&error, OmniError::Manifest(error) if error.kind == ManifestErrorKind::BadRequest), + "{error}" + ); + assert!(error.to_string().contains("predicate origin"), "{error}"); + // Two nearest arms record their attempts apart, one decision per scan. let mut params = ParamMap::from([("q".to_string(), Literal::List(vec![Literal::Float(0.0); 4]))]); From b8d245048a8aca2df348dfdd24a79d74d3ca7fe3 Mon Sep 17 00:00:00 2001 From: Ragnor Comerford Date: Mon, 12 Oct 2026 17:55:14 -0700 Subject: [PATCH 29/29] test(replay): forge the post-fusion filter under the fusion's actual parent The fused plan feeds its RankFuse into a Projection, not a Sort; the forged plan now rewires whichever single-input node reads the fusion. --- .../omnigraph/tests/integration/engine_v2_plan_replay.rs | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/crates/omnigraph/tests/integration/engine_v2_plan_replay.rs b/crates/omnigraph/tests/integration/engine_v2_plan_replay.rs index 194cec6b1..a30a94ddd 100644 --- a/crates/omnigraph/tests/integration/engine_v2_plan_replay.rs +++ b/crates/omnigraph/tests/integration/engine_v2_plan_replay.rs @@ -1308,8 +1308,12 @@ async fn a_fusion_replays() { }); forged.plan.set_properties(never, properties); match forged.plan.node_mut(parent) { - Some(PhysicalNode::Sort { input, .. }) => *input = never, - other => panic!("the fusion's parent is a sort: {other:?}"), + Some( + PhysicalNode::Projection { input, .. } + | PhysicalNode::Sort { input, .. } + | PhysicalNode::Limit { input, .. }, + ) => *input = never, + other => panic!("the fusion feeds a single-input node: {other:?}"), } let error = match db .replay_bound_plan(