From 2c305ca41de66be39be4d3a2e648f74c109b2ee6 Mon Sep 17 00:00:00 2001 From: Ragnor Comerford Date: Thu, 8 Oct 2026 15:47:45 -0700 Subject: [PATCH 01/14] fix(writes): charge Blob payload apart from the row around it Every keyed write checked one 32 MiB ceiling on a batch's whole Arrow memory, which includes a managed Blob value's bytes, so a value of exactly 32 MiB never fit beside its row. KeyedBytes splits that count into payload (each managed value's logical length) and framing (the rest), each under its own 32 MiB ceiling, and every keyed check uses it: staging, keyed-write preparation, the update carry scan, branch merge buffering and proven-insert chunking, and the compatibility loader's pre-decode forecast. The halves sum to the old count. A payload refusal names its resource with 'Blob payload bytes' in place of 'bytes'. Two Lance guards pin what the Blob write plan relies on: a whole-row merge-update keeps the row's stable row id, and merge-insert refuses an external reference outside the dataset's bases. --- changelog.d/blob-payload-budget.changed.md | 13 + crates/omnigraph/src/exec/merge.rs | 68 ++--- crates/omnigraph/src/exec/staging.rs | 113 +++---- crates/omnigraph/src/loader/mod.rs | 183 +++++++---- crates/omnigraph/src/storage_layer.rs | 131 ++++++-- crates/omnigraph/src/table_store.rs | 285 +++++++++--------- .../omnigraph/src/table_store/staged_tests.rs | 106 ++++++- .../omnigraph/tests/lance_surface_guards.rs | 87 ++++++ crates/omnigraph/tests/writes.rs | 203 +++++++++++-- docs/dev/execution.md | 5 +- docs/dev/writes.md | 22 +- docs/user/blobs.md | 8 +- docs/user/cli/managed-data.md | 3 +- docs/user/mutations/index.md | 9 +- 14 files changed, 844 insertions(+), 392 deletions(-) create mode 100644 changelog.d/blob-payload-budget.changed.md diff --git a/changelog.d/blob-payload-budget.changed.md b/changelog.d/blob-payload-budget.changed.md new file mode 100644 index 000000000..47daaff0f --- /dev/null +++ b/changelog.d/blob-payload-budget.changed.md @@ -0,0 +1,13 @@ +- A managed Blob value of exactly 32 MiB now fits in a write. Incremental + writes (inserts, updates, `append` and `merge` loads and branch merges) + check managed Blob payloads and the rest of the row against separate + 32 MiB ceilings. The old single ceiling counted both, so a 32 MiB value + was always refused for the row around it. A payload refusal names its + resource with `Blob payload bytes` in place of `bytes`, for example + `keyed write Blob payload bytes for node:Document`. An update that carries + an oversized Blob cell now reports `retained keyed batch Blob payload bytes + per operation`. One operation may now hold up to 32 MiB of Blob payload + plus 32 MiB of other row data, where it held 32 MiB in all. Encoded limits + stay: `omnigraph load` and `/load/ndjson` cap each line at 32 MiB, and + HTTP request bodies keep their caps, so those paths carry about 24 MiB of + `base64:` data per value. diff --git a/crates/omnigraph/src/exec/merge.rs b/crates/omnigraph/src/exec/merge.rs index d5a158291..f0634287a 100644 --- a/crates/omnigraph/src/exec/merge.rs +++ b/crates/omnigraph/src/exec/merge.rs @@ -7,7 +7,8 @@ use crate::ordered_cursor::{ use crate::seams::{decide_seam, fail}; use crate::session::Session; use crate::storage_layer::{ - KEYED_WRITE_MAX_BYTES, KEYED_WRITE_MAX_ROWS, KeyedWriteSemantics, ProvenInsertChunk, + KEYED_BLOB_PAYLOAD_MAX_BYTES, KEYED_WRITE_MAX_BYTES, KEYED_WRITE_MAX_ROWS, KeyedBytes, + KeyedWriteSemantics, ProvenInsertChunk, }; use crate::table_store::certified_insert_absence_rows; use futures::StreamExt; @@ -608,7 +609,7 @@ struct StagedTableWriter { dir: TempDir, dataset: Option, buffered_rows: usize, - buffered_bytes: u64, + buffered_bytes: KeyedBytes, row_count: u64, chunk_rows: Vec, batches: Vec, @@ -627,7 +628,7 @@ impl StagedTableWriter { dir, dataset: None, buffered_rows: 0, - buffered_bytes: 0, + buffered_bytes: KeyedBytes::default(), row_count: 0, chunk_rows: Vec::new(), batches: Vec::new(), @@ -651,24 +652,16 @@ impl StagedTableWriter { &row.dataset, &input, external_preflight, - KEYED_WRITE_MAX_BYTES, + KEYED_BLOB_PAYLOAD_MAX_BYTES, )? } else { - u64::try_from(input.get_array_memory_size()).map_err(|_| { - OmniError::manifest_internal("branch merge row memory size exceeds u64") - })? - }; - if predicted_row_bytes > KEYED_WRITE_MAX_BYTES { - return Err(OmniError::resource_limit( - "branch-merge fenced entity bytes", - KEYED_WRITE_MAX_BYTES, - predicted_row_bytes, - )); + KeyedBytes::of(&input)? } - let predicted_overflow = self + .ensure_fits("branch-merge fenced entity bytes")?; + let predicted_overflow = !self .buffered_bytes .checked_add(predicted_row_bytes) - .is_none_or(|bytes| bytes > KEYED_WRITE_MAX_BYTES); + .is_ok_and(KeyedBytes::fits); if self.buffered_rows > 0 && (self.buffered_rows >= KEYED_WRITE_MAX_ROWS || predicted_overflow) { @@ -679,20 +672,11 @@ impl StagedTableWriter { let batch = self .row_batch(row, input, materializer, external_preflight) .await?; - let row_bytes = u64::try_from(batch.get_array_memory_size()).map_err(|_| { - OmniError::manifest_internal("branch merge row memory size exceeds u64") - })?; - if row_bytes > KEYED_WRITE_MAX_BYTES { - return Err(OmniError::resource_limit( - "branch-merge fenced entity bytes", - KEYED_WRITE_MAX_BYTES, - row_bytes, - )); - } - let would_exceed_bytes = self + let row_bytes = KeyedBytes::of(&batch)?.ensure_fits("branch-merge fenced entity bytes")?; + let would_exceed_bytes = !self .buffered_bytes .checked_add(row_bytes) - .is_none_or(|bytes| bytes > KEYED_WRITE_MAX_BYTES); + .is_ok_and(KeyedBytes::fits); if self.buffered_rows > 0 && (self.buffered_rows >= KEYED_WRITE_MAX_ROWS || would_exceed_bytes) { @@ -703,14 +687,9 @@ impl StagedTableWriter { .checked_add(1) .ok_or_else(|| OmniError::manifest_internal("branch merge row count overflow"))?; self.buffered_rows += 1; - self.buffered_bytes = self - .buffered_bytes - .checked_add(row_bytes) - .ok_or_else(|| OmniError::manifest_internal("branch merge byte count overflow"))?; + self.buffered_bytes = self.buffered_bytes.checked_add(row_bytes)?; self.batches.push(batch); - if self.buffered_rows >= KEYED_WRITE_MAX_ROWS - || self.buffered_bytes >= KEYED_WRITE_MAX_BYTES - { + if self.buffered_rows >= KEYED_WRITE_MAX_ROWS || self.buffered_bytes.is_full() { self.flush().await?; } Ok(()) @@ -728,7 +707,7 @@ impl StagedTableWriter { .materialize_blob_batch_bounded_with_preflight_cache( &row.dataset, batch, - KEYED_WRITE_MAX_BYTES, + KEYED_BLOB_PAYLOAD_MAX_BYTES, external_preflight, &mut self.external_payloads, ) @@ -779,7 +758,7 @@ impl StagedTableWriter { .map_err(OmniError::arrow_internal)? }; self.buffered_rows = 0; - self.buffered_bytes = 0; + self.buffered_bytes = KeyedBytes::default(); self.chunk_rows.push(batch.num_rows()); let chunk_count = u64::try_from(self.chunk_rows.len()) .map_err(|_| OmniError::manifest_internal("branch merge chunk count exceeds u64"))?; @@ -4319,12 +4298,13 @@ async fn next_exact_staged_chunk( } else { arrow_select::concat::concat_batches(schema, &slices).map_err(OmniError::arrow_internal)? }; - let chunk_bytes = u64::try_from(chunk.get_array_memory_size()) - .map_err(|_| OmniError::manifest_internal("branch merge chunk bytes exceed u64"))?; - if chunk.num_rows() > KEYED_WRITE_MAX_ROWS || chunk_bytes > KEYED_WRITE_MAX_BYTES { + let chunk_bytes = KeyedBytes::of(&chunk)?; + if chunk.num_rows() > KEYED_WRITE_MAX_ROWS || !chunk_bytes.fits() { return Err(OmniError::manifest_internal(format!( - "branch merge reconstructed a keyed chunk outside its planned bound: {} rows / {chunk_bytes} bytes", - chunk.num_rows() + "branch merge reconstructed a keyed chunk outside its planned bound: {} rows / {} framing and {} payload bytes", + chunk.num_rows(), + chunk_bytes.framing, + chunk_bytes.payload ))); } Ok(chunk) @@ -5169,10 +5149,10 @@ impl Omnigraph { .preflight_persisted_blob_selection(&blob_selection) .await?; let carried_blob_bytes = blob_selection.materialized_payload_bytes(&external_preflight)?; - if carried_blob_bytes > KEYED_WRITE_MAX_BYTES { + if carried_blob_bytes > KEYED_BLOB_PAYLOAD_MAX_BYTES { return Err(OmniError::resource_limit( "materialized blob payload bytes", - KEYED_WRITE_MAX_BYTES, + KEYED_BLOB_PAYLOAD_MAX_BYTES, carried_blob_bytes, )); } diff --git a/crates/omnigraph/src/exec/staging.rs b/crates/omnigraph/src/exec/staging.rs index f6ea5d1d0..cd1d960e8 100644 --- a/crates/omnigraph/src/exec/staging.rs +++ b/crates/omnigraph/src/exec/staging.rs @@ -27,8 +27,8 @@ use std::collections::{HashMap, HashSet}; use std::sync::Arc; use crate::storage_layer::{ - DeletedIdBudget, KEYED_WRITE_MAX_BYTES, KEYED_WRITE_MAX_ROWS, KeyedWriteSemantics, - SnapshotHandle, StagedHandle, retain_keyed_batch, retained_keyed_bytes, + DeletedIdBudget, KEYED_BLOB_PAYLOAD_MAX_BYTES, KEYED_WRITE_MAX_ROWS, KeyedBytes, + KeyedWriteSemantics, SnapshotHandle, StagedHandle, retain_keyed_batch, retained_keyed_bytes, }; use arrow_array::{Array, RecordBatch, StringArray, UInt32Array}; use arrow_schema::SchemaRef; @@ -76,15 +76,12 @@ impl PendingTable { self.batches.iter().map(|b| b.num_rows()).sum() } - fn total_bytes(&self) -> Result { - self.batches.iter().try_fold(0_u64, |bytes, batch| { - let batch_bytes = u64::try_from(batch.get_array_memory_size()).map_err(|_| { - OmniError::manifest_internal("pending keyed batch bytes exceed u64") - })?; - bytes - .checked_add(batch_bytes) - .ok_or_else(|| OmniError::manifest_internal("pending keyed byte count overflow")) - }) + fn total_bytes(&self) -> Result { + self.batches + .iter() + .try_fold(KeyedBytes::default(), |bytes, batch| { + bytes.checked_add(KeyedBytes::of(batch)?) + }) } } @@ -120,7 +117,7 @@ pub(crate) struct MutationStaging { /// In-memory accumulated batches per table (insert/update path). pub(crate) pending: HashMap, /// Monotonic retained-batch accounting, updated only by `append_batch`. - pending_bytes: u64, + pending_bytes: KeyedBytes, /// Per-table delete predicates from delete-touching ops. D₂ guarantees a /// table is write-XOR-delete within one query, so this never overlaps /// `pending`. Staged as one combined `stage_delete` per table at @@ -259,8 +256,7 @@ impl MutationStaging { ))); } } - let batch_bytes = u64::try_from(batch.get_array_memory_size()) - .map_err(|_| OmniError::manifest_internal("pending keyed batch bytes exceed u64"))?; + let batch_bytes = KeyedBytes::of(&batch)?; let pending_bytes = if matches!(mode, PendingMode::StrictInsert | PendingMode::Upsert) { let existing_rows = self .pending @@ -279,23 +275,14 @@ impl MutationStaging { } let existing_bytes = match self.pending.get(table_key) { Some(existing) => existing.total_bytes()?, - None => 0, + None => KeyedBytes::default(), }; - let bytes = existing_bytes - .checked_add(batch_bytes) - .ok_or_else(|| OmniError::manifest_internal("pending keyed byte count overflow"))?; - if bytes > KEYED_WRITE_MAX_BYTES { - return Err(OmniError::resource_limit( - format!("keyed entity bytes for {table_key}"), - KEYED_WRITE_MAX_BYTES, - bytes, - )); - } + existing_bytes + .checked_add(batch_bytes)? + .ensure_fits(&format!("keyed entity bytes for {table_key}"))?; retained_keyed_bytes(self.pending_bytes, batch_bytes)? } else { - self.pending_bytes - .checked_add(batch_bytes) - .ok_or_else(|| OmniError::manifest_internal("pending keyed byte count overflow"))? + self.pending_bytes.checked_add(batch_bytes)? }; let entry = self .pending @@ -402,7 +389,7 @@ impl MutationStaging { /// keyed-write fence, while bytes include every pending table so one graph /// mutation cannot multiply the 32 MiB retained-memory budget by touching /// several tables. - pub(crate) fn pending_resource_usage(&self, table_key: &str) -> Result<(u64, u64)> { + pub(crate) fn pending_resource_usage(&self, table_key: &str) -> Result<(u64, KeyedBytes)> { let rows = u64::try_from( self.pending .get(table_key) @@ -512,26 +499,32 @@ impl MutationStaging { .collect::>(); let copied_external_blob_bytes = external_blob_preflight.materialized_payload_bytes(&copied_external_blob_uris)?; - if copied_external_blob_bytes > KEYED_WRITE_MAX_BYTES { + if copied_external_blob_bytes > KEYED_BLOB_PAYLOAD_MAX_BYTES { return Err(OmniError::resource_limit( "materialized external blob payload bytes", - KEYED_WRITE_MAX_BYTES, + KEYED_BLOB_PAYLOAD_MAX_BYTES, copied_external_blob_bytes, )); } - let retained_batch_bytes = stage_inputs.iter().try_fold(0, |bytes, (_, table, _, _)| { - if table.mode == PendingMode::Overwrite { - Ok(bytes) - } else { - retain_keyed_batch(bytes, &table.batch) - } - })?; - retained_keyed_bytes(retained_batch_bytes, copied_external_blob_bytes)?; + let retained_batch_bytes = + stage_inputs + .iter() + .try_fold(KeyedBytes::default(), |bytes, (_, table, _, _)| { + if table.mode == PendingMode::Overwrite { + Ok(bytes) + } else { + retain_keyed_batch(bytes, &table.batch) + } + })?; + retained_keyed_bytes( + retained_batch_bytes, + KeyedBytes::payload(copied_external_blob_bytes), + )?; // Only after the complete operation has passed policy, source, and // aggregate-copy admission do we read payload bytes. Reuse remains // batch-bounded so this vector cannot retain an operation-sized cache. - let mut materialized_keyed_bytes = 0; + let mut materialized_keyed_bytes = KeyedBytes::default(); for (table_key, table, _, _) in &mut stage_inputs { table.batch = match table.mode { PendingMode::StrictInsert | PendingMode::Upsert => { @@ -662,26 +655,18 @@ fn prepare_pending_table( } if matches!(table.mode, PendingMode::StrictInsert | PendingMode::Upsert) { - let (rows, bytes) = - table - .batches - .iter() - .try_fold((0_u64, 0_u64), |(rows, bytes), batch| { - let batch_rows = u64::try_from(batch.num_rows()) - .map_err(|_| OmniError::manifest_internal("keyed batch rows exceed u64"))?; - let batch_bytes = - u64::try_from(batch.get_array_memory_size()).map_err(|_| { - OmniError::manifest_internal("keyed batch bytes exceed u64") - })?; - Ok::<_, OmniError>(( - rows.checked_add(batch_rows).ok_or_else(|| { - OmniError::manifest_internal("keyed row count overflow") - })?, - bytes.checked_add(batch_bytes).ok_or_else(|| { - OmniError::manifest_internal("keyed byte count overflow") - })?, - )) - })?; + let (rows, bytes) = table.batches.iter().try_fold( + (0_u64, KeyedBytes::default()), + |(rows, bytes), batch| { + let batch_rows = u64::try_from(batch.num_rows()) + .map_err(|_| OmniError::manifest_internal("keyed batch rows exceed u64"))?; + Ok::<_, OmniError>(( + rows.checked_add(batch_rows) + .ok_or_else(|| OmniError::manifest_internal("keyed row count overflow"))?, + bytes.checked_add(KeyedBytes::of(batch)?)?, + )) + }, + )?; if rows > KEYED_WRITE_MAX_ROWS as u64 { return Err(OmniError::resource_limit( format!("keyed entities for {table_key}"), @@ -689,13 +674,7 @@ fn prepare_pending_table( rows, )); } - if bytes > KEYED_WRITE_MAX_BYTES { - return Err(OmniError::resource_limit( - format!("keyed entity bytes for {table_key}"), - KEYED_WRITE_MAX_BYTES, - bytes, - )); - } + bytes.ensure_fits(&format!("keyed entity bytes for {table_key}"))?; } let mode = table.mode; diff --git a/crates/omnigraph/src/loader/mod.rs b/crates/omnigraph/src/loader/mod.rs index 06fcc2ce7..1a6f8734f 100644 --- a/crates/omnigraph/src/loader/mod.rs +++ b/crates/omnigraph/src/loader/mod.rs @@ -28,7 +28,10 @@ use crate::error::{OmniError, Result, missing_graph_type_at_snapshot}; use crate::exec::staging::{MutationStaging, PendingMode}; use crate::seams::{catalog, decide_seam, fail}; use crate::session::Session; -use crate::storage_layer::{DeletedIdBudget, KEYED_WRITE_MAX_BYTES, retain_keyed_batch}; +use crate::storage_layer::{ + DeletedIdBudget, KEYED_BLOB_PAYLOAD_MAX_BYTES, KEYED_WRITE_MAX_BYTES, KeyedBytes, + retain_keyed_batch, +}; /// Result of a load operation. #[derive(Debug, Clone, Default)] @@ -771,7 +774,7 @@ async fn load_jsonl_reader_once( // Phase 2a: build and validate every node batch up front. Cheap and // synchronous — surfaces validation errors before any S3 traffic. let mut node_id_remap = TypedNodeIdRemap::default(); - let mut prepared_keyed_bytes = 0; + let mut prepared_keyed_bytes = KeyedBytes::default(); let mut prepared_nodes: Vec<(String, String, Vec, usize)> = Vec::with_capacity(node_rows.len().saturating_add(strict_nodes.len())); let mut __dst_nr: Vec<_> = node_rows.into_iter().collect(); @@ -1433,18 +1436,20 @@ fn take_object_or_empty( #[derive(Default)] struct KeyedInputBudget { tables: HashMap, - bytes: u64, + bytes: KeyedBytes, } #[derive(Default)] struct KeyedTableInput { rows: usize, - bytes: u64, + bytes: KeyedBytes, } /// Charge a keyed JSON record before the parse spool retains it: a lower bound -/// on its Arrow payload per table and across tables, taken before base64 is -/// decoded. Not a JSON DOM bound; `MutationStaging::append_batch` is the authority. +/// on its Arrow bytes per table and across tables, taken before base64 is +/// decoded, split like the later batch check (`KeyedBytes`): a `base64:` +/// value's decoded length as payload, the rest as framing. Not a JSON DOM +/// bound; `MutationStaging::append_batch` is the authority. fn account_keyed_json_row( table_key: &str, data: &JsonValue, @@ -1463,58 +1468,56 @@ fn account_keyed_json_row( entry.rows as u64, )); } - let row_bytes = estimate_json_arrow_bytes(data)? - .checked_add( - u64::try_from(structural_string_bytes) - .map_err(|_| OmniError::manifest_internal("keyed string bytes exceed u64"))?, - ) - .ok_or_else(|| OmniError::manifest_internal("keyed input entity bytes overflow"))?; + let structural = u64::try_from(structural_string_bytes) + .map_err(|_| OmniError::manifest_internal("keyed string bytes exceed u64"))?; + let row_bytes = + estimate_json_keyed_bytes(data)?.checked_add(KeyedBytes::framing(structural))?; entry.bytes = entry .bytes - .checked_add(row_bytes) - .ok_or_else(|| OmniError::manifest_internal("keyed parsed byte count overflow"))?; - if entry.bytes > KEYED_WRITE_MAX_BYTES { - return Err(OmniError::resource_limit( - format!("keyed parsed entity bytes for {table_key}"), - KEYED_WRITE_MAX_BYTES, - entry.bytes, - )); - } - let total = budgets.bytes.checked_add(row_bytes).ok_or_else(|| { - OmniError::manifest_internal("keyed parsed operation byte count overflow") - })?; - if total > KEYED_WRITE_MAX_BYTES { - return Err(OmniError::resource_limit( - "keyed parsed entity bytes per operation", - KEYED_WRITE_MAX_BYTES, - total, - )); - } - budgets.bytes = total; + .checked_add(row_bytes)? + .ensure_fits(&format!("keyed parsed entity bytes for {table_key}"))?; + budgets.bytes = budgets + .bytes + .checked_add(row_bytes)? + .ensure_fits("keyed parsed entity bytes per operation")?; Ok(()) } fn estimate_json_arrow_bytes(value: &JsonValue) -> Result { + let bytes = estimate_json_keyed_bytes(value)?; + bytes + .framing + .checked_add(bytes.payload) + .ok_or_else(|| OmniError::manifest_internal("JSON value bytes overflow")) +} + +/// A lower bound on the Arrow bytes a JSON value becomes. A `base64:` string +/// counts its decoded length as payload; everything else is framing. +fn estimate_json_keyed_bytes(value: &JsonValue) -> Result { match value { - JsonValue::Null => Ok(0), - JsonValue::Bool(_) => Ok(1), + JsonValue::Null => Ok(KeyedBytes::default()), + JsonValue::Bool(_) => Ok(KeyedBytes::framing(1)), // Four bytes avoids rejecting valid Float32/Int32 input early. Wider // physical scalars are charged exactly by the later Arrow batch check. - JsonValue::Number(_) => Ok(4), + JsonValue::Number(_) => Ok(KeyedBytes::framing(4)), JsonValue::String(value) => { - let bytes = match value.strip_prefix("base64:") { - Some(encoded) => base64::decoded_len_estimate(encoded.len()).saturating_sub( - encoded - .as_bytes() - .iter() - .rev() - .take_while(|&&byte| byte == b'=') - .count(), - ), - None => value.len(), + let string_bytes = |bytes: usize| { + u64::try_from(bytes) + .map_err(|_| OmniError::manifest_internal("JSON string bytes exceed u64")) }; - u64::try_from(bytes) - .map_err(|_| OmniError::manifest_internal("JSON string bytes exceed u64")) + match value.strip_prefix("base64:") { + Some(encoded) => Ok(KeyedBytes::payload(string_bytes( + base64::decoded_len_estimate(encoded.len()).saturating_sub( + encoded + .as_bytes() + .iter() + .rev() + .take_while(|&&byte| byte == b'=') + .count(), + ), + )?)), + None => Ok(KeyedBytes::framing(string_bytes(value.len())?)), + } } JsonValue::Array(values) => { let offsets = u64::try_from(values.len()) @@ -1522,20 +1525,20 @@ fn estimate_json_arrow_bytes(value: &JsonValue) -> Result { .checked_add(1) .and_then(|count| count.checked_mul(4)) .ok_or_else(|| OmniError::manifest_internal("JSON array offset bytes overflow"))?; - values.iter().try_fold(offsets, |bytes, value| { - bytes - .checked_add(estimate_json_arrow_bytes(value)?) - .ok_or_else(|| OmniError::manifest_internal("JSON array bytes overflow")) - }) + values + .iter() + .try_fold(KeyedBytes::framing(offsets), |bytes, value| { + bytes.checked_add(estimate_json_keyed_bytes(value)?) + }) } // Property names are schema, not per-row Arrow payload. Count values // only so the early lower bound does not reject an otherwise-valid // wide schema; exact field buffers are charged after batch building. - JsonValue::Object(values) => values.values().try_fold(0_u64, |bytes, value| { - bytes - .checked_add(estimate_json_arrow_bytes(value)?) - .ok_or_else(|| OmniError::manifest_internal("JSON object bytes overflow")) - }), + JsonValue::Object(values) => values + .values() + .try_fold(KeyedBytes::default(), |bytes, value| { + bytes.checked_add(estimate_json_keyed_bytes(value)?) + }), } } @@ -2349,10 +2352,10 @@ fn preflight_blob_decode_budget<'a>( decoded_bytes = decoded_bytes.checked_add(estimate).ok_or_else(|| { OmniError::manifest_internal("decoded blob input byte count overflow") })?; - if decoded_bytes > KEYED_WRITE_MAX_BYTES { + if decoded_bytes > KEYED_BLOB_PAYLOAD_MAX_BYTES { return Err(OmniError::resource_limit( format!("decoded blob input bytes for {table_key}"), - KEYED_WRITE_MAX_BYTES, + KEYED_BLOB_PAYLOAD_MAX_BYTES, decoded_bytes, )); } @@ -2372,10 +2375,10 @@ pub(crate) fn append_blob_value(builder: &mut BlobArrayBuilder, value: &str) -> .take_while(|&&b| b == b'=') .count(), ); - if decoded_estimate as u64 > KEYED_WRITE_MAX_BYTES { + if decoded_estimate as u64 > KEYED_BLOB_PAYLOAD_MAX_BYTES { return Err(OmniError::resource_limit( "decoded blob input bytes", - KEYED_WRITE_MAX_BYTES, + KEYED_BLOB_PAYLOAD_MAX_BYTES, decoded_estimate as u64, )); } @@ -3945,6 +3948,47 @@ edge WorksAt: Person -> Company )); } + /// The pre-decode forecast charges a `base64:` value by its decoded length to + /// the payload ceiling and the rest of the row to the framing ceiling, like + /// the batch check after it, so an exact-limit value fits beside its row and + /// one more decoded byte is refused by the payload ceiling. + #[test] + fn parsed_forecast_charges_base64_payload_apart_from_framing() { + let limit = usize::try_from(KEYED_BLOB_PAYLOAD_MAX_BYTES).unwrap(); + let encode = |bytes: usize| { + format!( + "base64:{}", + base64::engine::general_purpose::STANDARD.encode(vec![7_u8; bytes]) + ) + }; + + let row = serde_json::json!({"title": "exact", "content": encode(limit)}); + let mut budget = KeyedInputBudget::default(); + account_keyed_json_row("node:Document", &row, 0, &mut budget) + .expect("an exact-limit payload fits beside its row"); + assert_eq!( + budget.bytes, + KeyedBytes { + framing: 5, + payload: KEYED_BLOB_PAYLOAD_MAX_BYTES, + } + ); + + let row = serde_json::json!({"title": "over", "content": encode(limit + 1)}); + let error = + account_keyed_json_row("node:Document", &row, 0, &mut KeyedInputBudget::default()) + .expect_err("one decoded byte over must be refused"); + assert!(matches!( + error, + OmniError::ResourceLimitExceeded { + ref resource, + limit: KEYED_BLOB_PAYLOAD_MAX_BYTES, + actual, + } if resource == "keyed parsed entity Blob payload bytes for node:Document" + && actual == KEYED_BLOB_PAYLOAD_MAX_BYTES + 1 + )); + } + #[test] fn operation_byte_allowances_span_graph_types_and_include_their_ceiling() { let row = serde_json::json!({"payload": "x".repeat(17 * 1024 * 1024)}); @@ -3958,11 +4002,22 @@ edge WorksAt: Person -> Company && actual > KEYED_WRITE_MAX_BYTES )); + use crate::storage_layer::{KeyedBytes, retained_keyed_bytes}; assert_eq!( - crate::storage_layer::retained_keyed_bytes(KEYED_WRITE_MAX_BYTES - 1, 1).unwrap(), - KEYED_WRITE_MAX_BYTES + retained_keyed_bytes( + KeyedBytes::framing(KEYED_WRITE_MAX_BYTES - 1), + KeyedBytes::framing(1) + ) + .unwrap(), + KeyedBytes::framing(KEYED_WRITE_MAX_BYTES) + ); + assert!( + retained_keyed_bytes( + KeyedBytes::framing(KEYED_WRITE_MAX_BYTES), + KeyedBytes::framing(1) + ) + .is_err() ); - assert!(crate::storage_layer::retained_keyed_bytes(KEYED_WRITE_MAX_BYTES, 1).is_err()); let id = &row["payload"].as_str().unwrap()[..16 * 1024 * 1024 - std::mem::size_of::()]; let mut removed = DeletedIdBudget::default(); diff --git a/crates/omnigraph/src/storage_layer.rs b/crates/omnigraph/src/storage_layer.rs index 9792d0445..f218fb85e 100644 --- a/crates/omnigraph/src/storage_layer.rs +++ b/crates/omnigraph/src/storage_layer.rs @@ -73,27 +73,114 @@ use crate::table_store::{ pub(crate) const KEYED_WRITE_MAX_ROWS: usize = 8192; pub(crate) const KEYED_WRITE_MAX_BYTES: u64 = 32 * 1024 * 1024; -/// The operation-wide sibling of the per-table keyed ceiling. This uses the +/// The ceiling on managed Blob payload bytes, by logical length, that one +/// keyed write retains: the payload half of [`KeyedBytes`]. A single managed +/// value of exactly this size is admitted. +pub(crate) const KEYED_BLOB_PAYLOAD_MAX_BYTES: u64 = 32 * 1024 * 1024; + +/// What a keyed batch retains, under two ceilings. `payload` is the logical +/// length of every managed Blob value the batch carries; `framing` is the rest +/// of its Arrow memory, the Blob columns' offsets, validity and URIs included, +/// with the existing conservative count of a slice's complete backing buffers. +/// The two always sum to the batch's Arrow memory, so splitting them moves no +/// byte out of the accounting. One combined ceiling could not admit a value of +/// exactly [`KEYED_BLOB_PAYLOAD_MAX_BYTES`]: the batch around it is never empty. +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +pub(crate) struct KeyedBytes { + pub(crate) framing: u64, + pub(crate) payload: u64, +} + +impl KeyedBytes { + pub(crate) fn of(batch: &RecordBatch) -> Result { + let total = u64::try_from(batch.get_array_memory_size()) + .map_err(|_| OmniError::manifest_internal("keyed batch bytes exceed u64"))?; + let payload = crate::table_store::logical_blob_payload_bytes(batch)?; + let framing = total.checked_sub(payload).ok_or_else(|| { + OmniError::manifest_internal("Blob payload bytes exceed their batch's Arrow memory") + })?; + Ok(Self { framing, payload }) + } + + pub(crate) fn framing(framing: u64) -> Self { + Self { + framing, + payload: 0, + } + } + + pub(crate) fn payload(payload: u64) -> Self { + Self { + framing: 0, + payload, + } + } + + pub(crate) fn checked_add(self, other: Self) -> Result { + Ok(Self { + framing: self + .framing + .checked_add(other.framing) + .ok_or_else(|| OmniError::manifest_internal("keyed framing byte count overflow"))?, + payload: self + .payload + .checked_add(other.payload) + .ok_or_else(|| OmniError::manifest_internal("keyed payload byte count overflow"))?, + }) + } + + pub(crate) fn fits(self) -> bool { + self.framing <= KEYED_WRITE_MAX_BYTES && self.payload <= KEYED_BLOB_PAYLOAD_MAX_BYTES + } + + /// Whether either half has reached its ceiling, so nothing more fits. + pub(crate) fn is_full(self) -> bool { + self.framing >= KEYED_WRITE_MAX_BYTES || self.payload >= KEYED_BLOB_PAYLOAD_MAX_BYTES + } + + /// Refuse a half above its ceiling. `resource` names the framing ceiling + /// ("... bytes ..."); the payload ceiling is the same name with "Blob + /// payload bytes" in place of its first "bytes". + pub(crate) fn ensure_fits(self, resource: &str) -> Result { + if self.framing > KEYED_WRITE_MAX_BYTES { + return Err(OmniError::resource_limit( + resource, + KEYED_WRITE_MAX_BYTES, + self.framing, + )); + } + if self.payload > KEYED_BLOB_PAYLOAD_MAX_BYTES { + return Err(OmniError::resource_limit( + blob_payload_resource(resource), + KEYED_BLOB_PAYLOAD_MAX_BYTES, + self.payload, + )); + } + Ok(self) + } +} + +/// The payload ceiling's name for a framing ceiling's name. +pub(crate) fn blob_payload_resource(resource: &str) -> String { + debug_assert!(resource.contains(" bytes"), "{resource}"); + resource.replacen(" bytes", " Blob payload bytes", 1) +} + +/// The operation-wide sibling of the per-table keyed ceilings. This uses the /// existing Arrow accounting (including its conservative shared-buffer count), -/// not a second allocator or a claim about native execution/RSS. -pub(crate) fn retained_keyed_bytes(current: u64, additional: u64) -> Result { - let actual = current - .checked_add(additional) - .ok_or_else(|| OmniError::manifest_internal("retained keyed batch byte count overflow"))?; - if actual > KEYED_WRITE_MAX_BYTES { - return Err(OmniError::resource_limit( - "retained keyed batch bytes per operation", - KEYED_WRITE_MAX_BYTES, - actual, - )); - } - Ok(actual) +/// split by [`KeyedBytes`], not a second allocator or a claim about native +/// execution/RSS. +pub(crate) fn retained_keyed_bytes( + current: KeyedBytes, + additional: KeyedBytes, +) -> Result { + current + .checked_add(additional)? + .ensure_fits("retained keyed batch bytes per operation") } -pub(crate) fn retain_keyed_batch(current: u64, batch: &RecordBatch) -> Result { - let bytes = u64::try_from(batch.get_array_memory_size()) - .map_err(|_| OmniError::manifest_internal("retained keyed batch bytes exceed u64"))?; - retained_keyed_bytes(current, bytes) +pub(crate) fn retain_keyed_batch(current: KeyedBytes, batch: &RecordBatch) -> Result { + retained_keyed_bytes(current, KeyedBytes::of(batch)?) } /// One allowance per mutation (all tables and cascades) or load (all replacement @@ -146,11 +233,15 @@ pub(crate) const BLOB_REBUILD_IO_BUFFER_BYTES: u64 = 8 * 1024 * 1024; pub(crate) struct PendingScanBudget { pub(crate) table_key: String, pub(crate) initial_rows: u64, - pub(crate) initial_bytes: u64, + pub(crate) initial_bytes: KeyedBytes, } impl PendingScanBudget { - pub(crate) fn new(table_key: impl Into, initial_rows: u64, initial_bytes: u64) -> Self { + pub(crate) fn new( + table_key: impl Into, + initial_rows: u64, + initial_bytes: KeyedBytes, + ) -> Self { Self { table_key: table_key.into(), initial_rows, diff --git a/crates/omnigraph/src/table_store.rs b/crates/omnigraph/src/table_store.rs index 1c4db6b20..a263f9325 100644 --- a/crates/omnigraph/src/table_store.rs +++ b/crates/omnigraph/src/table_store.rs @@ -74,8 +74,9 @@ use crate::db::{DatasetEntry, Snapshot}; use crate::error::{OmniError, Result}; use crate::seams::{decide_seam, skip}; use crate::storage_layer::{ - BLOB_REBUILD_IO_BUFFER_BYTES, IndexBuildSpec, KEYED_WRITE_MAX_BYTES, KEYED_WRITE_MAX_ROWS, - KeyedWriteSemantics, PendingScanBudget, ProvenInsertChunk, + BLOB_REBUILD_IO_BUFFER_BYTES, IndexBuildSpec, KEYED_BLOB_PAYLOAD_MAX_BYTES, + KEYED_WRITE_MAX_BYTES, KEYED_WRITE_MAX_ROWS, KeyedBytes, KeyedWriteSemantics, + PendingScanBudget, ProvenInsertChunk, blob_payload_resource, }; pub(crate) use crate::error::is_scratch_exhaustion; @@ -1931,17 +1932,18 @@ impl TableStore { None } - /// Conservative pre-read size for a persisted descriptor batch. The raw - /// one-row Arrow allocation already accounts every ordinary column and - /// descriptor buffer; adding logical payload bytes may overestimate the - /// smaller logical Blob descriptor, but cannot understate retained memory. + /// Conservative pre-read size for a persisted descriptor batch: its raw + /// Arrow allocation, every ordinary column and descriptor buffer, as + /// framing, and the logical payload its Blob cells will materialize. The + /// descriptor buffers may overestimate the smaller logical Blob framing, + /// but cannot understate retained memory. pub(crate) fn predicted_materialized_blob_batch_bytes( &self, ds: &Dataset, batch: &RecordBatch, external: &ExternalBlobPreflight, max_blob_bytes: u64, - ) -> Result { + ) -> Result { let payload = self.persisted_blob_payload_bytes(ds.schema(), batch, external)?; if payload > max_blob_bytes { return Err(OmniError::resource_limit( @@ -1950,12 +1952,7 @@ impl TableStore { payload, )); } - let retained = u64::try_from(batch.get_array_memory_size()).map_err(|_| { - OmniError::manifest_internal("persisted Blob batch memory size exceeds u64") - })?; - retained.checked_add(payload).ok_or_else(|| { - OmniError::manifest_internal("materialized Blob batch byte count overflow") - }) + KeyedBytes::of(batch)?.checked_add(KeyedBytes::payload(payload)) } async fn rebuild_blob_column( @@ -2787,8 +2784,6 @@ impl TableStore { "stage_keyed_write called with empty batch", )); } - let batch_bytes = u64::try_from(batch.get_array_memory_size()) - .map_err(|_| OmniError::manifest_internal("keyed write batch bytes exceed u64"))?; if batch.num_rows() > KEYED_WRITE_MAX_ROWS { return Err(OmniError::resource_limit( format!("keyed write entities for {type_key}"), @@ -2796,13 +2791,7 @@ impl TableStore { batch.num_rows() as u64, )); } - if batch_bytes > KEYED_WRITE_MAX_BYTES { - return Err(OmniError::resource_limit( - format!("keyed write bytes for {type_key}"), - KEYED_WRITE_MAX_BYTES, - batch_bytes, - )); - } + KeyedBytes::of(&batch)?.ensure_fits(&format!("keyed write bytes for {type_key}"))?; let id_field_id = exact_id_primary_key_field_id(&ds, system_columns, "stage_keyed_write")?; let expected_read_version = ds.version().version; let expected_schema_preorder_ids = schema_preorder_field_ids(&ds, "stage_keyed_write")?; @@ -2957,9 +2946,6 @@ impl TableStore { "{context} called with empty batch" ))); } - let batch_bytes = u64::try_from(batch.get_array_memory_size()).map_err(|_| { - OmniError::manifest_internal(format!("{context} batch bytes exceed u64")) - })?; if batch.num_rows() > KEYED_WRITE_MAX_ROWS { return Err(OmniError::resource_limit( format!("keyed write entities for {table_key}"), @@ -2967,13 +2953,7 @@ impl TableStore { batch.num_rows() as u64, )); } - if batch_bytes > KEYED_WRITE_MAX_BYTES { - return Err(OmniError::resource_limit( - format!("keyed write bytes for {table_key}"), - KEYED_WRITE_MAX_BYTES, - batch_bytes, - )); - } + KeyedBytes::of(&batch)?.ensure_fits(&format!("keyed write bytes for {table_key}"))?; let batch = self .prepare_keyed_write_batch(table_key, batch, system_columns) @@ -3108,38 +3088,21 @@ impl TableStore { self.validate_keyed_write_batch(table_key, &batch, system_columns)?; let external_uris = collect_external_blob_uris(&batch)?; let payload_bytes = preflight.materialized_payload_bytes(&external_uris)?; - if payload_bytes > KEYED_WRITE_MAX_BYTES { + if payload_bytes > KEYED_BLOB_PAYLOAD_MAX_BYTES { return Err(OmniError::resource_limit( "materialized external blob payload bytes", - KEYED_WRITE_MAX_BYTES, + KEYED_BLOB_PAYLOAD_MAX_BYTES, payload_bytes, )); } - let retained_bytes = u64::try_from(batch.get_array_memory_size()).map_err(|_| { - OmniError::manifest_internal("keyed write input batch bytes exceed u64") - })?; - let predicted_bytes = retained_bytes.checked_add(payload_bytes).ok_or_else(|| { - OmniError::manifest_internal("materialized keyed Blob byte count overflow") - })?; - if predicted_bytes > KEYED_WRITE_MAX_BYTES { - return Err(OmniError::resource_limit( - format!("keyed write bytes for {table_key}"), - KEYED_WRITE_MAX_BYTES, - predicted_bytes, - )); - } + let resource = format!("keyed write bytes for {table_key}"); + KeyedBytes::of(&batch)? + .checked_add(KeyedBytes::payload(payload_bytes))? + .ensure_fits(&resource)?; let batch = - materialize_external_blob_inputs(batch, KEYED_WRITE_MAX_BYTES, preflight).await?; - let materialized_bytes = u64::try_from(batch.get_array_memory_size()).map_err(|_| { - OmniError::manifest_internal("materialized keyed write batch bytes exceed u64") - })?; - if materialized_bytes > KEYED_WRITE_MAX_BYTES { - return Err(OmniError::resource_limit( - format!("keyed write bytes for {table_key}"), - KEYED_WRITE_MAX_BYTES, - materialized_bytes, - )); - } + materialize_external_blob_inputs(batch, KEYED_BLOB_PAYLOAD_MAX_BYTES, preflight) + .await?; + KeyedBytes::of(&batch)?.ensure_fits(&resource)?; Ok(batch) } @@ -4643,7 +4606,7 @@ impl TableStore { // columns now so an already-overwide match fails before // `take_rows` performs the second full-row read. let non_blob_bytes = non_blob_column_bytes(committed_ds, &batch)?; - let payload_budget = account.remaining_bytes_after(non_blob_bytes)?; + let payload_budget = account.payload_remaining_after(non_blob_bytes)?; if let Some(schema) = &carried_without_blobs { let carried_rows = predicate_rows_as(schema, &batch)?; @@ -4690,16 +4653,10 @@ impl TableStore { .unwrap_or(error)); } Err(OmniError::ResourceLimitExceeded { actual, .. }) => { - let actual = account - .bytes_with(non_blob_bytes)? - .checked_add(actual) - .ok_or_else(|| { - OmniError::manifest_internal("pending scan byte count overflow") - })?; return Err(OmniError::resource_limit( - "retained keyed batch bytes per operation", - KEYED_WRITE_MAX_BYTES, - actual, + blob_payload_resource("retained keyed batch bytes per operation"), + KEYED_BLOB_PAYLOAD_MAX_BYTES, + account.payload_with(actual)?, )); } Err(error) => return Err(error), @@ -4974,7 +4931,7 @@ fn assign_row_id_meta(fragments: &mut [Fragment], start_row_id: u64) -> Result<( struct PendingScanAccount { table_key: String, rows: u64, - bytes: u64, + bytes: KeyedBytes, } impl PendingScanAccount { @@ -4982,7 +4939,7 @@ impl PendingScanAccount { let mut account = Self { table_key: budget.table_key, rows: 0, - bytes: 0, + bytes: KeyedBytes::default(), }; account.add_usage(budget.initial_rows, budget.initial_bytes)?; Ok(account) @@ -5002,12 +4959,10 @@ impl PendingScanAccount { fn add_batch(&mut self, batch: &RecordBatch) -> Result<()> { let rows = u64::try_from(batch.num_rows()) .map_err(|_| OmniError::manifest_internal("pending scan row count exceeds u64"))?; - let bytes = u64::try_from(batch.get_array_memory_size()) - .map_err(|_| OmniError::manifest_internal("pending scan bytes exceed u64"))?; - self.add_usage(rows, bytes) + self.add_usage(rows, KeyedBytes::of(batch)?) } - fn add_usage(&mut self, rows: u64, bytes: u64) -> Result<()> { + fn add_usage(&mut self, rows: u64, bytes: KeyedBytes) -> Result<()> { let next_rows = self .rows .checked_add(rows) @@ -5021,15 +4976,8 @@ impl PendingScanAccount { } let next_bytes = self .bytes - .checked_add(bytes) - .ok_or_else(|| OmniError::manifest_internal("pending scan byte count overflow"))?; - if next_bytes > KEYED_WRITE_MAX_BYTES { - return Err(OmniError::resource_limit( - "retained keyed batch bytes per operation", - KEYED_WRITE_MAX_BYTES, - next_bytes, - )); - } + .checked_add(bytes)? + .ensure_fits("retained keyed batch bytes per operation")?; self.rows = next_rows; self.bytes = next_bytes; Ok(()) @@ -5052,22 +5000,21 @@ impl PendingScanAccount { Ok(()) } - fn bytes_with(&self, bytes: u64) -> Result { + fn payload_with(&self, payload: u64) -> Result { self.bytes - .checked_add(bytes) - .ok_or_else(|| OmniError::manifest_internal("pending scan byte count overflow")) + .payload + .checked_add(payload) + .ok_or_else(|| OmniError::manifest_internal("pending scan payload byte count overflow")) } - fn remaining_bytes_after(&self, bytes: u64) -> Result { - let actual = self.bytes_with(bytes)?; - if actual > KEYED_WRITE_MAX_BYTES { - return Err(OmniError::resource_limit( - "retained keyed batch bytes per operation", - KEYED_WRITE_MAX_BYTES, - actual, - )); - } - Ok(KEYED_WRITE_MAX_BYTES - actual) + /// Charge a match's non-Blob framing and return the payload its carried + /// Blob cells may still materialize. + fn payload_remaining_after(&self, framing: u64) -> Result { + let bytes = self + .bytes + .checked_add(KeyedBytes::framing(framing))? + .ensure_fits("retained keyed batch bytes per operation")?; + Ok(KEYED_BLOB_PAYLOAD_MAX_BYTES - bytes.payload) } fn next_scan_rows(&self) -> usize { @@ -5078,7 +5025,9 @@ impl PendingScanAccount { } fn next_scan_bytes(&self) -> u64 { - KEYED_WRITE_MAX_BYTES.saturating_sub(self.bytes).max(1) + KEYED_WRITE_MAX_BYTES + .saturating_sub(self.bytes.framing) + .max(1) } } @@ -5257,6 +5206,38 @@ struct LogicalBlobInput<'a> { uris: &'a StringArray, } +/// The logical length of every managed Blob value a batch carries in the +/// logical input shape (`{data, uri}`): the payload half of +/// [`crate::storage_layer::KeyedBytes`]. A Blob column in any other shape, such +/// as a persisted descriptor, holds no payload bytes and counts as framing. +pub(crate) fn logical_blob_payload_bytes(batch: &RecordBatch) -> Result { + let mut total = 0_u64; + for (field, column) in batch.schema().fields().iter().zip(batch.columns()) { + let is_blob = + lance::datatypes::Field::try_from(field.as_ref()).is_ok_and(|field| field.is_blob()); + if !is_blob { + continue; + } + let Some(descriptions) = column.as_any().downcast_ref::() else { + continue; + }; + let Ok(input) = logical_blob_input(descriptions, field.name()) else { + continue; + }; + for row in 0..descriptions.len() { + if descriptions.is_null(row) || input.data.is_null(row) { + continue; + } + let length = u64::try_from(input.data.value_length(row)) + .map_err(|_| OmniError::manifest_internal("negative Blob payload length"))?; + total = total + .checked_add(length) + .ok_or_else(|| OmniError::manifest_internal("Blob payload byte count overflow"))?; + } + } + Ok(total) +} + fn logical_blob_input<'a>( descriptions: &'a StructArray, field_name: &str, @@ -5973,7 +5954,7 @@ fn bounded_proven_insert_stream( None::<(RecordBatch, usize)>, Vec::::new(), 0_usize, - 0_u64, + KeyedBytes::default(), schema, table_key, ), @@ -5992,21 +5973,17 @@ fn bounded_proven_insert_stream( if let Some((batch, consumed_rows)) = pending.take() { let batch_rows = batch.num_rows(); let batch_bytes = - u64::try_from(batch.get_array_memory_size()).map_err(|_| { - OmniError::manifest_internal( - "proven insert delta batch bytes exceed u64", - ) - .into_datafusion_external() - })?; + KeyedBytes::of(&batch).map_err(OmniError::into_datafusion_external)?; + let next_bytes = accumulated_bytes + .checked_add(batch_bytes) + .map_err(OmniError::into_datafusion_external)?; let fits = accumulated_rows .checked_add(batch_rows) .is_some_and(|rows| rows <= KEYED_WRITE_MAX_ROWS) - && accumulated_bytes - .checked_add(batch_bytes) - .is_some_and(|bytes| bytes <= KEYED_WRITE_MAX_BYTES); + && next_bytes.fits(); if accumulated.is_empty() || fits { accumulated_rows += batch_rows; - accumulated_bytes += batch_bytes; + accumulated_bytes = next_bytes; accumulated.push(batch); current_offset += consumed_rows; if current @@ -6016,9 +5993,7 @@ fn bounded_proven_insert_stream( current = None; current_offset = 0; } - if accumulated_rows == KEYED_WRITE_MAX_ROWS - || accumulated_bytes == KEYED_WRITE_MAX_BYTES - { + if accumulated_rows == KEYED_WRITE_MAX_ROWS || accumulated_bytes.is_full() { let output = finish_proven_insert_batch( &schema, std::mem::take(&mut accumulated), @@ -6034,7 +6009,7 @@ fn bounded_proven_insert_stream( None, Vec::new(), 0, - 0, + KeyedBytes::default(), schema, table_key, ), @@ -6059,7 +6034,7 @@ fn bounded_proven_insert_stream( pending, Vec::new(), 0, - 0, + KeyedBytes::default(), schema, table_key, ), @@ -6102,7 +6077,7 @@ fn bounded_proven_insert_stream( pending, Vec::new(), 0, - 0, + KeyedBytes::default(), schema, table_key, ), @@ -6149,43 +6124,47 @@ fn next_proven_insert_batch( let (mut rows, mut logical_bytes) = largest_proven_insert_prefix(batch, offset, max_rows)?; loop { let is_whole_batch = offset == 0 && rows == batch.num_rows(); - let retained_bytes = u64::try_from(batch.get_array_memory_size()).unwrap_or(u64::MAX); + let retained = KeyedBytes::of(batch)?; + let retained_bytes = retained.framing.saturating_add(retained.payload); + let logical_total = logical_bytes.framing.saturating_add(logical_bytes.payload); // ArrayData's logical slice measure excludes the backing capacity held // by an Arrow slice. Preserve an already-compact whole scanner batch, // but copy any selected prefix or obvious retained-parent emission. let retained_parent_slop = 64_u64 * 1024; let keeps_only_logical_slice = - retained_bytes <= logical_bytes.saturating_add(retained_parent_slop); - let candidate = if is_whole_batch - && retained_bytes <= KEYED_WRITE_MAX_BYTES - && keeps_only_logical_slice - { + retained_bytes <= logical_total.saturating_add(retained_parent_slop); + let candidate = if is_whole_batch && retained.fits() && keeps_only_logical_slice { batch.clone() } else { copy_proven_insert_batch_range(batch, offset, rows)? }; - let bytes = u64::try_from(candidate.get_array_memory_size()).map_err(|_| { - OmniError::manifest_internal("proven insert delta batch bytes exceed u64") - })?; - if bytes <= KEYED_WRITE_MAX_BYTES { + let bytes = KeyedBytes::of(&candidate)?; + if bytes.fits() { validate_proven_insert_source_batch(&candidate, table_key)?; return Ok((candidate, rows)); } if rows == 1 { - return Err(OmniError::resource_limit( - format!("proven insert delta bytes for {table_key}"), - KEYED_WRITE_MAX_BYTES, - bytes, - )); + bytes.ensure_fits(&format!("proven insert delta bytes for {table_key}"))?; } drop(candidate); - rows = ((u128::try_from(rows).unwrap() * u128::from(KEYED_WRITE_MAX_BYTES)) - / u128::from(bytes)) + rows = shrunk_row_count(rows, bytes).clamp(1, rows - 1); + logical_bytes = proven_insert_slice_bytes(batch, offset, rows)?; + } +} + +/// Scale a row count down in proportion to the half of `bytes` that most +/// exceeds its ceiling. +fn shrunk_row_count(rows: usize, bytes: KeyedBytes) -> usize { + let scaled = |actual: u64, ceiling: u64| { + if actual <= ceiling { + return u128::MAX; + } + u128::try_from(rows).unwrap_or(u128::MAX) * u128::from(ceiling) / u128::from(actual) + }; + scaled(bytes.framing, KEYED_WRITE_MAX_BYTES) + .min(scaled(bytes.payload, KEYED_BLOB_PAYLOAD_MAX_BYTES)) .try_into() .unwrap_or(1_usize) - .clamp(1, rows - 1); - logical_bytes = proven_insert_slice_memory_size(batch, offset, rows)?; - } } /// Select the largest row prefix whose logical Arrow buffers fit the byte @@ -6196,14 +6175,14 @@ fn largest_proven_insert_prefix( batch: &RecordBatch, offset: usize, max_rows: usize, -) -> Result<(usize, u64)> { - let one_row_bytes = proven_insert_slice_memory_size(batch, offset, 1)?; - if max_rows == 1 || one_row_bytes > KEYED_WRITE_MAX_BYTES { +) -> Result<(usize, KeyedBytes)> { + let one_row_bytes = proven_insert_slice_bytes(batch, offset, 1)?; + if max_rows == 1 || !one_row_bytes.fits() { return Ok((1, one_row_bytes)); } - let max_bytes = proven_insert_slice_memory_size(batch, offset, max_rows)?; - if max_bytes <= KEYED_WRITE_MAX_BYTES { + let max_bytes = proven_insert_slice_bytes(batch, offset, max_rows)?; + if max_bytes.fits() { return Ok((max_rows, max_bytes)); } @@ -6211,13 +6190,27 @@ fn largest_proven_insert_prefix( let mut high = max_rows - 1; while low < high { let middle = low + (high - low).div_ceil(2); - if proven_insert_slice_memory_size(batch, offset, middle)? <= KEYED_WRITE_MAX_BYTES { + if proven_insert_slice_bytes(batch, offset, middle)?.fits() { low = middle; } else { high = middle - 1; } } - Ok((low, proven_insert_slice_memory_size(batch, offset, low)?)) + Ok((low, proven_insert_slice_bytes(batch, offset, low)?)) +} + +/// The selected rows' compact Arrow size, split like [`KeyedBytes::of`]. +fn proven_insert_slice_bytes( + batch: &RecordBatch, + offset: usize, + rows: usize, +) -> Result { + let total = proven_insert_slice_memory_size(batch, offset, rows)?; + let payload = logical_blob_payload_bytes(&batch.slice(offset, rows))?; + let framing = total.checked_sub(payload).ok_or_else(|| { + OmniError::manifest_internal("Blob payload bytes exceed their slice's Arrow memory") + })?; + Ok(KeyedBytes { framing, payload }) } fn proven_insert_slice_memory_size(batch: &RecordBatch, offset: usize, rows: usize) -> Result { @@ -6275,15 +6268,7 @@ fn validate_proven_insert_source_batch(batch: &RecordBatch, table_key: &str) -> batch.num_rows() as u64, )); } - let bytes = u64::try_from(batch.get_array_memory_size()) - .map_err(|_| OmniError::manifest_internal("proven insert delta batch bytes exceed u64"))?; - if bytes > KEYED_WRITE_MAX_BYTES { - return Err(OmniError::resource_limit( - format!("proven insert delta bytes for {table_key}"), - KEYED_WRITE_MAX_BYTES, - bytes, - )); - } + KeyedBytes::of(batch)?.ensure_fits(&format!("proven insert delta bytes for {table_key}"))?; Ok(()) } diff --git a/crates/omnigraph/src/table_store/staged_tests.rs b/crates/omnigraph/src/table_store/staged_tests.rs index b014a169a..5b1272a68 100644 --- a/crates/omnigraph/src/table_store/staged_tests.rs +++ b/crates/omnigraph/src/table_store/staged_tests.rs @@ -23,8 +23,8 @@ use crate::error::{OmniError, StorageFailureKind}; use crate::instrumentation::{MergeWriteProbes, with_merge_write_probes}; use crate::storage_layer::{ - IndexBuildSpec, KEYED_WRITE_MAX_BYTES, KEYED_WRITE_MAX_ROWS, KeyedWriteSemantics, - PendingScanBudget, ProvenInsertChunk, SnapshotHandle, + IndexBuildSpec, KEYED_BLOB_PAYLOAD_MAX_BYTES, KEYED_WRITE_MAX_BYTES, KEYED_WRITE_MAX_ROWS, + KeyedBytes, KeyedWriteSemantics, PendingScanBudget, ProvenInsertChunk, SnapshotHandle, }; use crate::table_store::{StagedWrite, TableStore}; use arrow_array::{Array, Int32Array, RecordBatch, StringArray, StructArray, UInt64Array}; @@ -253,19 +253,90 @@ fn collect_age_for_id(batches: &[RecordBatch], needle: &str) -> Option { None } +/// A keyed batch charges a managed Blob value's logical length to the payload +/// ceiling and the Arrow bytes around it to the framing ceiling, so a value of +/// exactly the payload ceiling fits, though the batch's whole Arrow size is over +/// it, and one more byte is refused by the payload ceiling. An external +/// reference's URI is framing: it carries no payload. +#[test] +fn keyed_bytes_split_blob_payload_from_framing() { + let limit = usize::try_from(KEYED_BLOB_PAYLOAD_MAX_BYTES).unwrap(); + let exact = blob_person_pk_batch("alice", &vec![7_u8; limit]); + let bytes = KeyedBytes::of(&exact).unwrap(); + assert_eq!(bytes.payload, KEYED_BLOB_PAYLOAD_MAX_BYTES); + assert!(bytes.framing > 0, "the batch around a value is never empty"); + assert_eq!( + bytes.framing + bytes.payload, + u64::try_from(exact.get_array_memory_size()).unwrap() + ); + assert!(bytes.framing + bytes.payload > KEYED_WRITE_MAX_BYTES); + assert_eq!( + bytes + .ensure_fits("keyed write bytes for node:Person") + .unwrap(), + bytes + ); + + let over = blob_person_pk_batch("alice", &vec![7_u8; limit + 1]); + match KeyedBytes::of(&over) + .unwrap() + .ensure_fits("keyed write bytes for node:Person") + { + Err(OmniError::ResourceLimitExceeded { + resource, + limit, + actual, + }) => { + assert_eq!(resource, "keyed write Blob payload bytes for node:Person"); + assert_eq!( + (limit, actual), + ( + KEYED_BLOB_PAYLOAD_MAX_BYTES, + KEYED_BLOB_PAYLOAD_MAX_BYTES + 1 + ) + ); + } + other => panic!("one payload byte over must be refused, got {other:?}"), + } + + let mut external = lance::blob::BlobArrayBuilder::new(1); + external.push_uri("s3://bucket/object.bin").unwrap(); + let schema = Arc::new(Schema::new(vec![ + Field::new("id", DataType::Utf8, false), + lance::blob::blob_field("content", true), + ])); + let reference = RecordBatch::try_new( + schema, + vec![ + Arc::new(StringArray::from(vec!["bob"])) as _, + external.finish().unwrap(), + ], + ) + .unwrap(); + let bytes = KeyedBytes::of(&reference).unwrap(); + assert_eq!(bytes.payload, 0); + assert_eq!( + bytes.framing, + u64::try_from(reference.get_array_memory_size()).unwrap() + ); +} + #[test] fn pending_scan_budget_caps_are_inclusive_and_one_over_is_typed() { PendingScanAccount::new(PendingScanBudget::new( "test:people", KEYED_WRITE_MAX_ROWS as u64, - KEYED_WRITE_MAX_BYTES, + KeyedBytes { + framing: KEYED_WRITE_MAX_BYTES, + payload: KEYED_BLOB_PAYLOAD_MAX_BYTES, + }, )) - .expect("the exact keyed row/byte limits are inclusive"); + .expect("the exact keyed row, framing and payload limits are inclusive"); let row_error = PendingScanAccount::new(PendingScanBudget::new( "test:people", KEYED_WRITE_MAX_ROWS as u64 + 1, - 0, + KeyedBytes::default(), )) .err() .expect("one row over must be rejected"); @@ -281,7 +352,7 @@ fn pending_scan_budget_caps_are_inclusive_and_one_over_is_typed() { let byte_error = PendingScanAccount::new(PendingScanBudget::new( "test:people", 0, - KEYED_WRITE_MAX_BYTES + 1, + KeyedBytes::framing(KEYED_WRITE_MAX_BYTES + 1), )) .err() .expect("one byte over must be rejected"); @@ -294,6 +365,23 @@ fn pending_scan_budget_caps_are_inclusive_and_one_over_is_typed() { } if resource == "retained keyed batch bytes per operation" && actual == KEYED_WRITE_MAX_BYTES + 1 )); + + let payload_error = PendingScanAccount::new(PendingScanBudget::new( + "test:people", + 0, + KeyedBytes::payload(KEYED_BLOB_PAYLOAD_MAX_BYTES + 1), + )) + .err() + .expect("one payload byte over must be rejected"); + assert!(matches!( + payload_error, + OmniError::ResourceLimitExceeded { + ref resource, + limit: KEYED_BLOB_PAYLOAD_MAX_BYTES, + actual, + } if resource == "retained keyed batch Blob payload bytes per operation" + && actual == KEYED_BLOB_PAYLOAD_MAX_BYTES + 1 + )); } #[tokio::test] @@ -1452,7 +1540,7 @@ async fn scan_with_pending_rejects_key_column_missing_from_projection() { Some(&["note"]), None, Some("id"), - PendingScanBudget::new("test:people", 0, 0), + PendingScanBudget::new("test:people", 0, KeyedBytes::default()), ) .await .expect_err("scan_with_pending must reject merge-shadow with missing key in projection"); @@ -1474,7 +1562,7 @@ async fn scan_with_pending_rejects_key_column_missing_from_projection() { Some(&["id", "note"]), None, Some("id"), - PendingScanBudget::new("test:people", 8190, 0), + PendingScanBudget::new("test:people", 8190, KeyedBytes::default()), ) .await .expect("projection containing key_column must succeed"); @@ -1495,7 +1583,7 @@ async fn scan_with_pending_rejects_key_column_missing_from_projection() { Some(&["id", "note"]), None, Some("id"), - PendingScanBudget::new("test:people", 8191, 0), + PendingScanBudget::new("test:people", 8191, KeyedBytes::default()), ) .await .expect_err("pending + unshadowed committed output must share the row budget"); diff --git a/crates/omnigraph/tests/lance_surface_guards.rs b/crates/omnigraph/tests/lance_surface_guards.rs index 0f840b66b..ba54b79a5 100644 --- a/crates/omnigraph/tests/lance_surface_guards.rs +++ b/crates/omnigraph/tests/lance_surface_guards.rs @@ -5054,6 +5054,9 @@ async fn camelcase_index_equality_routes_to_scalar_index() { // a silently-wrong batch) in release. Faithful transcription of lance#7444's // minimal repro: merge-seed → merge-update → delete → filter + with_row_id. // This guard turns red if a future Lance bump regresses the upstream fix. +// It also pins that the merge-update keeps the updated row's stable row id: +// a Blob write reads that id back from its own detached commit to form the +// ETag a later read at the published snapshot returns (RFC 0033 §4.3). #[tokio::test] async fn filtered_scan_tolerates_merge_update_row_id_overlap() { use futures::TryStreamExt; @@ -5108,11 +5111,29 @@ async fn filtered_scan_tolerates_merge_update_row_id_overlap() { (1..=40).map(|i| format!("r{i}")).collect(), ); let ds = merge(ds, seed, schema.clone()).await; + let row_id_of = |ds: Dataset, slug: &'static str| async move { + let mut scan = ds.scan(); + scan.with_row_id(); + scan.filter(&format!("slug = '{slug}'")).unwrap(); + let batch = scan.try_into_batch().await.unwrap(); + assert_eq!(batch.num_rows(), 1, "{slug}"); + batch + .column_by_name(ROW_ID) + .unwrap() + .as_primitive::() + .value(0) + }; + let seeded_row_id = row_id_of(ds.clone(), "t3").await; let updates = mk_batch( (1..=15).map(|i| format!("t{i}")).collect(), (1..=15).map(|i| format!("e{i}")).collect(), ); let ds = Arc::new(merge(ds, updates, schema.clone()).await); + assert_eq!( + row_id_of((*ds).clone(), "t3").await, + seeded_row_id, + "a whole-row merge-update keeps the updated row's stable row id" + ); // The delete's deletion vector makes the overlapping id region sparse. let staged = lance::dataset::DeleteBuilder::new(ds.clone(), "slug = 't20'") @@ -5144,6 +5165,72 @@ async fn filtered_scan_tolerates_merge_update_row_id_overlap() { } } +// --- Guard: merge-insert refuses an external Blob reference outside the bases --- +// +// Lance 11's merge-insert writer stages its files with `WriteParams::default()`, +// whose `allow_external_blob_outside_bases` is false, and a merge-insert takes +// no other write params. A keyed write therefore cannot carry a stored external +// reference as a reference: the engine reads it and stores managed bytes, or +// refuses it under a denying policy (RFC 0033 §4.3). When this goes red, +// merge-insert stores such a reference, and carrying a sibling's reference +// without reading it is open again (RFC 0033, Phase 4). +#[tokio::test] +async fn merge_insert_refuses_an_external_blob_outside_the_dataset_bases() { + let dir = tempfile::tempdir().unwrap(); + let external_path = dir.path().join("outside.bin"); + std::fs::write(&external_path, b"outside the dataset").unwrap(); + let external_uri = url::Url::from_file_path(&external_path) + .unwrap() + .to_string(); + let schema = Arc::new(Schema::new(vec![ + Field::new("id", DataType::Utf8, false), + lance::blob::blob_field("content", true), + ])); + let row = |content: arrow_array::ArrayRef| { + RecordBatch::try_new( + schema.clone(), + vec![Arc::new(StringArray::from(vec!["a"])), content], + ) + .unwrap() + }; + let mut managed = BlobArrayBuilder::new(1); + managed.push_bytes(b"managed").unwrap(); + let ds = Dataset::write( + RecordBatchIterator::new(vec![Ok(row(managed.finish().unwrap()))], schema.clone()), + dir.path().join("table.lance").to_str().unwrap(), + Some(WriteParams { + mode: WriteMode::Create, + enable_stable_row_ids: true, + data_storage_version: Some(LanceFileVersion::V2_2), + ..Default::default() + }), + ) + .await + .unwrap(); + + let mut external = BlobArrayBuilder::new(1); + external.push_uri(external_uri.as_str()).unwrap(); + let job = MergeInsertBuilder::try_new(Arc::new(ds), vec!["id".to_string()]) + .unwrap() + .when_matched(WhenMatched::UpdateAll) + .when_not_matched(WhenNotMatched::InsertAll) + .try_build() + .unwrap(); + let error = job + .execute_reader(RecordBatchIterator::new( + vec![Ok(row(external.finish().unwrap()))], + schema.clone(), + )) + .await + .expect_err("merge-insert must refuse a reference outside the dataset's bases"); + assert!( + error + .to_string() + .contains("is outside registered external bases"), + "{error}" + ); +} + // --- Guard 21: starts_with routes to the BTREE (LikePrefix) and stays literal -- // // The .gq `starts_with` predicate lowers to the DataFusion `starts_with` diff --git a/crates/omnigraph/tests/writes.rs b/crates/omnigraph/tests/writes.rs index 515d1c95e..4189a6121 100644 --- a/crates/omnigraph/tests/writes.rs +++ b/crates/omnigraph/tests/writes.rs @@ -672,6 +672,7 @@ async fn mutation_keyed_write_row_cap_accepts_limit_and_rejects_one_over_pre_eff const OPERATION_BYTES: u64 = 32 * 1024 * 1024; const KEYED_BATCH_BYTES: &str = "retained keyed batch bytes per operation"; +const KEYED_BATCH_PAYLOAD_BYTES: &str = "retained keyed batch Blob payload bytes per operation"; const REMOVED_ID_BYTES: &str = "retained removed-id bytes per operation"; fn operation_refusal(error: &OmniError, label: &str) -> bool { @@ -891,10 +892,15 @@ edge LinkZ: Zed -> Zed { n: I32? } ); } -/// 24 MiB of external payload plus 10 MiB of retained rows fit per type, not together. +/// External payload to be copied joins the operation's Blob payload allowance +/// with managed payload, before any payload read: 12 MiB copied plus 21 MiB +/// managed is refused although each fits. Retained rows have their own +/// allowance, so 24 MiB copied plus 10 MiB of rows fits. /// Rust, not GQT: MiB-scale inputs and the payload-read probe are outside the case format. #[tokio::test] -async fn external_blob_bytes_join_the_operation_allowance_before_any_payload_read() { +async fn external_blob_bytes_join_the_operation_payload_allowance_before_any_payload_read() { + use base64::Engine; + const SCHEMA: &str = "\ node Document { title: String @key content: Blob? note: String? } node Image { title: String @key content: Blob? note: String? } @@ -926,7 +932,10 @@ node Image { title: String @key content: Blob? note: String? } let before_manifest = snapshot_main(&db).await.unwrap().graph_manifest_version(); let files = files_under(&graph_path); - let note = "x".repeat(5 * 1024 * 1024); + let managed = format!( + "base64:{}", + base64::engine::general_purpose::STANDARD.encode(vec![7_u8; 21 * 1024 * 1024]) + ); let stage_probes = StageWriteProbes::rendezvous(1); let read_probes = omnigraph::instrumentation::MergeWriteProbes::default(); let error = with_stage_write_probes( @@ -935,19 +944,19 @@ node Image { title: String @key content: Blob? note: String? } read_probes.clone(), db.mutate( "main", - r#"query wide($uri: String, $note: String) { - insert Document { title: "one", content: $uri, note: $note } - insert Image { title: "two", content: $uri, note: $note } + r#"query wide($uri: String, $managed: Blob) { + insert Document { title: "one", content: $uri } + insert Image { title: "two", content: $managed } }"#, "wide", - ¶ms(&[("$uri", &external_uri), ("$note", ¬e)]), + ¶ms(&[("$uri", &external_uri), ("$managed", &managed)]), ), ), ) .await - .expect_err("external payloads and retained rows must share one operation allowance"); + .expect_err("copied and managed payloads must share one operation allowance"); assert!( - operation_refusal(&error, KEYED_BATCH_BYTES), + operation_refusal(&error, KEYED_BATCH_PAYLOAD_BYTES), "unexpected refusal: {error:?}" ); assert_eq!( @@ -962,27 +971,32 @@ node Image { title: String @key content: Blob? note: String? } snapshot_main(&db).await.unwrap().graph_manifest_version(), before_manifest ); + let note = "x".repeat(5 * 1024 * 1024); let accepted = db .mutate( "main", - r#"query small($uri: String) { - insert Document { title: "one", content: $uri } + r#"query wide($uri: String, $note: String) { + insert Document { title: "one", content: $uri, note: $note } + insert Image { title: "two", content: $uri, note: $note } }"#, - "small", - ¶ms(&[("$uri", &external_uri)]), + "wide", + ¶ms(&[("$uri", &external_uri), ("$note", ¬e)]), ) .await - .expect("one 12 MiB external payload fits after the refusal"); - assert_eq!(accepted.affected_nodes, 1); + .expect("24 MiB copied and 10 MiB of rows fit under separate allowances"); + assert_eq!(accepted.affected_nodes, 2); } -/// Three equal payloads leave a LargeBinary buffer at 4/3 of its content: predicted -/// 24.4 MiB, materialized 32.5 MiB. Rust, not GQT: MiB-scale inputs, builder capacity. +/// Three equal payloads leave a LargeBinary buffer at 4/3 of its content, and +/// the unused capacity, about 8.1 MiB over both types, is framing. With 13 MiB +/// of rows per type the predicted framing, 26 MiB, fits; the materialized +/// framing, about 34.1 MiB, does not. Rust, not GQT: MiB-scale inputs, builder +/// capacity. #[tokio::test] async fn materialized_blob_batches_are_rechecked_against_the_operation_allowance_before_staging() { const SCHEMA: &str = "\ -node Document { title: String @key content: Blob? } -node Image { title: String @key content: Blob? } +node Document { title: String @key content: Blob? note: String? } +node Image { title: String @key content: Blob? note: String? } "; let dir = tempfile::tempdir().unwrap(); @@ -1019,16 +1033,19 @@ node Image { title: String @key content: Blob? } read_probes.clone(), db.mutate( "main", - r#"query thrice($uri: String) { - insert Document { title: "a", content: $uri } + r#"query thrice($uri: String, $note: String) { + insert Document { title: "a", content: $uri, note: $note } insert Document { title: "b", content: $uri } insert Document { title: "c", content: $uri } - insert Image { title: "a", content: $uri } + insert Image { title: "a", content: $uri, note: $note } insert Image { title: "b", content: $uri } insert Image { title: "c", content: $uri } }"#, "thrice", - ¶ms(&[("$uri", &external_uri)]), + ¶ms(&[ + ("$uri", &external_uri), + ("$note", &"x".repeat(13 * 1024 * 1024)), + ]), ), ), ) @@ -1223,7 +1240,8 @@ query update_note($note: String) { ref resource, limit: LIMIT, actual, - } if resource == "retained keyed batch bytes per operation" && actual > LIMIT + } if resource == "retained keyed batch Blob payload bytes per operation" + && actual == LIMIT + 1 ), "oversized update blob must be rejected before payload read, got {error:?}" ); @@ -1286,6 +1304,143 @@ query replace_content($c: Blob) { ) .await; assert_eq!(&bytes[..], &[1, 2, 3]); + + // The payload ceiling is inclusive: an external cell of exactly the limit + // is carried beside its row, read once and stored as managed bytes. + let exact_path = dir.path().join("exact.blob"); + let file = std::fs::File::create(&exact_path).unwrap(); + file.set_len(LIMIT).unwrap(); + drop(file); + let exact_row = serde_json::json!({ + "type": "Document", + "data": { + "title": "wide", + "content": format!("file://{}", exact_path.display()), + } + }) + .to_string(); + db.load_jsonl(&exact_row, LoadMode::Overwrite) + .await + .unwrap(); + db.mutate( + "main", + UPDATE, + "update_note", + ¶ms(&[("$note", "carried")]), + ) + .await + .expect("an external cell of exactly the payload ceiling is carried"); + let bytes = read_managed_blob_bytes( + &db, + ReadTarget::branch("main"), + node_blob_cell("Document", "wide", "content"), + ) + .await; + assert_eq!(bytes.len(), usize::try_from(LIMIT).unwrap()); + assert!(bytes.iter().all(|&byte| byte == 0)); +} + +/// A managed Blob value of exactly the payload ceiling fits beside its row: +/// payload and framing have separate ceilings, and the compatibility loader's +/// pre-decode forecast splits like the batch check. One more byte is refused +/// before any graph or table effect. The strict NDJSON loader bounds its +/// encoded line instead, so the exact-limit value's 44.7 MB line is refused. +/// Rust, not GQT: 32 MiB payloads are outside the case format. +#[tokio::test] +async fn exact_limit_blob_payload_fits_beside_its_row_and_one_more_byte_is_refused() { + use base64::Engine; + + const LIMIT: usize = 32 * 1024 * 1024; + const SCHEMA: &str = "node Document { title: String @key content: Blob? }\n"; + const INSERT: &str = r#" +query put_doc($title: String, $c: Blob) { + insert Document { title: $title, content: $c } +} +"#; + let encode = |bytes: usize| { + format!( + "base64:{}", + base64::engine::general_purpose::STANDARD.encode(vec![7_u8; bytes]) + ) + }; + let row = |title: &str, content: &str| { + serde_json::json!({"type": "Document", "data": {"title": title, "content": content}}) + .to_string() + }; + let dir = tempfile::tempdir().unwrap(); + let db = helpers::session( + Omnigraph::init(dir.path().to_str().unwrap(), SCHEMA) + .await + .unwrap(), + ); + + let exact = encode(LIMIT); + db.mutate( + "main", + INSERT, + "put_doc", + ¶ms(&[("$title", "inserted"), ("$c", &exact)]), + ) + .await + .expect("an insert admits an exact-limit value"); + db.load_jsonl(&row("loaded", &exact), LoadMode::Merge) + .await + .expect("the compatibility loader admits an exact-limit value"); + for title in ["inserted", "loaded"] { + let bytes = read_managed_blob_bytes( + &db, + ReadTarget::branch("main"), + node_blob_cell("Document", title, "content"), + ) + .await; + assert_eq!(bytes.len(), LIMIT, "{title}"); + assert!(bytes.iter().all(|&byte| byte == 7), "{title}"); + } + + let before = snapshot_main(&db).await.unwrap().graph_manifest_version(); + let strict = db + .load_graph_batch("main", &row("strict", &exact), LoadMode::Merge) + .await + .expect_err("the strict loader bounds the encoded line"); + assert!( + matches!(&strict, OmniError::ResourceLimitExceeded { resource, .. } + if resource == "graph_batch_line_bytes"), + "{strict:?}" + ); + + let over = encode(LIMIT + 1); + let insert = db + .mutate( + "main", + INSERT, + "put_doc", + ¶ms(&[("$title", "over"), ("$c", &over)]), + ) + .await + .expect_err("one more byte is refused"); + let load = db + .load_jsonl(&row("over", &over), LoadMode::Merge) + .await + .expect_err("one more byte is refused"); + for (error, resource) in [ + (insert, "decoded blob input bytes"), + ( + load, + "keyed parsed entity Blob payload bytes for node:Document", + ), + ] { + assert!( + matches!(&error, OmniError::ResourceLimitExceeded { resource: actual_resource, limit, actual } + if actual_resource == resource + && *limit == LIMIT as u64 + && *actual == LIMIT as u64 + 1), + "{error:?}" + ); + } + assert_eq!( + snapshot_main(&db).await.unwrap().graph_manifest_version(), + before + ); } /// A predicate update carrying many rows' Blob cells reads the managed ones in diff --git a/docs/dev/execution.md b/docs/dev/execution.md index bdf7bb22c..f90fe2d8d 100644 --- a/docs/dev/execution.md +++ b/docs/dev/execution.md @@ -682,8 +682,9 @@ All load modes share the mutation publisher and recovery protocol: Mutation insert/update and keyed Load retain the per-table limits of 8,192 rows and 32 MiB, plus one 32 MiB sum of retained Arrow batches across touched tables. The sum uses `get_array_memory_size`, preserving conservative shared-buffer -counting. Keyed parsing separately caps its decoded-payload estimate across -types at 32 MiB before retaining each row. External Blob copy admission includes +counting, and managed Blob payloads are charged apart from it under their own +32 MiB ceilings (see [writes](writes.md)). Keyed parsing separately caps its +decoded-payload estimate across types at 32 MiB before retaining each row. External Blob copy admission includes the retained keyed batches plus copied payload estimates before payload reads; materialized batches are checked again before fragment staging. diff --git a/docs/dev/writes.md b/docs/dev/writes.md index da820d2ca..7a0ba34fe 100644 --- a/docs/dev/writes.md +++ b/docs/dev/writes.md @@ -342,12 +342,22 @@ Insert/update mutations and keyed Append/Merge loads also cap the sum of retained Arrow batches across tables at 32 MiB (`retained keyed batch bytes per operation`). Admission uses `get_array_memory_size` accounting; shared buffers may be conservatively counted -more than once. An update's pending-aware scan charges the same sum under the -same resource name. The keyed parse spool separately caps its decoded-payload -estimate across tables at 32 MiB (`keyed parsed entity bytes per operation`). -External Blob copy admission adds copied payload estimates, which are not yet -read, to the retained keyed batches under the same resource name before reading -payloads, then checks materialized batches before staging fragments. +more than once. Every keyed byte check splits that count with one function, +`KeyedBytes::of` in `storage_layer.rs`: the logical length of each managed Blob +value in the logical `{data, uri}` shape is payload, under its own 32 MiB +ceiling (`KEYED_BLOB_PAYLOAD_MAX_BYTES`, reported as `… Blob payload bytes …`), +and the rest is framing, under the existing ceilings. The halves sum to the +Arrow count, so the split moves no byte out of the accounting; one combined +ceiling could never admit a value of exactly 32 MiB beside its row. An update's +pending-aware scan charges the same sums under the same resource names, and +the payload remaining is the budget for reading its carried Blob cells. The +keyed parse spool separately caps its pre-decode estimate across tables at +32 MiB (`keyed parsed entity bytes per operation`), split the same way, a +`base64:` value counting its decoded length as payload. External Blob copy +admission adds copied payload estimates, which are not yet read, to the payload +half before reading payloads, then checks materialized batches before staging +fragments. Branch merge's row buffering and proven-insert chunking use the +same split. Delete mutations, cascades and Overwrite's removed-ID detection stream matches instead of collecting the full scan. One 32 MiB diff --git a/docs/user/blobs.md b/docs/user/blobs.md index e58f6ca0c..1f5d0e9fe 100644 --- a/docs/user/blobs.md +++ b/docs/user/blobs.md @@ -152,7 +152,8 @@ and retry. |---|---|---| | 32 MiB of decoded `base64:` bytes | Each node or edge type in one load, in every mode, including `overwrite` | `decoded blob input bytes for ` | | 32 MiB of decoded `base64:` bytes | One `base64:` value, in a load or in an insert or update mutation | `decoded blob input bytes` | -| 32 MiB per touched type, and 32 MiB across all touched types, Blob bytes included | Incremental writes: `append` and `merge` loads, inserts and updates. External bytes copied in and Blob values carried unchanged by an update count | `keyed write bytes for
`, `keyed entity bytes for
`, `retained keyed batch bytes per operation` | +| 32 MiB per touched type, and 32 MiB across all touched types, Blob payloads excluded | Incremental writes: `append` and `merge` loads, inserts, updates and branch merges. Every byte of the rows except managed Blob payloads counts, URIs and Blob framing included | `keyed write bytes for
`, `keyed entity bytes for
`, `retained keyed batch bytes per operation` | +| 32 MiB of managed Blob payload per touched type, and 32 MiB across all touched types, inclusive | The same writes. Each Blob value counts its length; external bytes copied in and Blob values carried unchanged by an update count. A single value of exactly 32 MiB fits beside its row | the same names with `Blob payload bytes` in place of `bytes`, for example `keyed entity Blob payload bytes for
` | | 32 MiB of external payload copied into managed storage | One incremental write operation across all its types, and each type within it: two types copying 20 MiB each exceed it although each fits its per-type limit | `materialized external blob payload bytes` | | 32 MiB of Blob payload | One branch merge that writes rows, across all types, managed and external bytes together | `materialized blob payload bytes` | | 8,192 external references | One write operation or merge | `external Blob reference cells` | @@ -165,7 +166,10 @@ and retry. The HTTP load request body is also capped at 32 MiB. That cap counts the encoded request, so one request carries about 24 MiB of decoded `base64:` -data. Every HTTP request other than a load (`/load` and `/load/ndjson`) is bounded by the default 1 MiB request body limit, so a `base64:` +data. The NDJSON loader that `omnigraph load` and `/load/ndjson` use caps each +encoded line at 32 MiB the same way. The embedded `load` API checks each row's +decoded size instead and has no line cap, so it admits a 32 MiB value; over +HTTP, `/load` keeps its 32 MiB body cap. Every HTTP request other than a load (`/load` and `/load/ndjson`) is bounded by the default 1 MiB request body limit, so a `base64:` literal in an HTTP mutation hits that limit first. Values larger than these limits stay readable. The CLI and the HTTP server diff --git a/docs/user/cli/managed-data.md b/docs/user/cli/managed-data.md index 2a974c333..778d3c0de 100644 --- a/docs/user/cli/managed-data.md +++ b/docs/user/cli/managed-data.md @@ -156,7 +156,8 @@ One managed load accepts at most **32 MiB (33,554,432 bytes)** of input. The CLI checks this before sending the request. Incremental `append` and `merge` also retain the engine's **8,192 rows and 32 MiB per keyed table** bounds, plus **32 MiB across retained keyed batches** and a separate parsed-payload estimate -across types. Strict loads check projected in-memory size too. The NDJSON byte +across types; managed Blob payloads have their own 32 MiB per table and across +tables. Strict loads check projected in-memory size too. The NDJSON byte bound does not prove that a batch fits those [engine limits](../mutations/index.md#limits-and-conflicts). Split prepared `append` and `merge` inputs into suitable batches, preserving diff --git a/docs/user/mutations/index.md b/docs/user/mutations/index.md index ead28817c..65c01b4ef 100644 --- a/docs/user/mutations/index.md +++ b/docs/user/mutations/index.md @@ -149,9 +149,12 @@ Loads preserve supplied embeddings and do not generate them. Results report Insert/update mutations and incremental keyed loads are bounded to 8,192 entities and 32 MiB per touched type, plus 32 MiB of retained Arrow batches -across all touched types in one operation. Keyed loads also have a separate -32 MiB parsed-payload estimate across types. External Blob payloads that require -copying count toward the aggregate allowance. Every strict load retains its +across all touched types in one operation. Managed Blob payloads are not part of +those bytes: they have their own 32 MiB per touched type and across all touched +types, so a single 32 MiB value fits beside its row. Keyed loads also have a +separate 32 MiB parsed-payload estimate across types, split the same way. +External Blob payloads that require copying count toward the payload +allowance. Every strict load retains its projected in-memory size check. Blob values have further limits; see [Blob limits](../blobs.md#limits). From 9425d9f5a66b1b10c581ab82ef99a24a8a2ab98a Mon Sep 17 00:00:00 2001 From: Ragnor Comerford Date: Thu, 8 Oct 2026 22:02:07 -0700 Subject: [PATCH 02/14] feat(blob): put and clear one Blob cell from an embedded Session Session::put_blob_at_as and clear_blob_at_as replace or clear one Blob value of an existing node or edge, addressed by exact id like a Blob read. A cell write is a Mutation-protocol write: it stages one upsert of the carried row and runs the Mutation tail, now split into commit_staged_mutation and publish_committed_mutation, which mutate uses too. The target's old payload is never read; the row's other cells are carried as an update carries them and share the 32 MiB payload allowance with the new value, which is bounded at 32 MiB inclusive before anything is captured. An optional precondition (an ETag set, or any existing value) is evaluated against the attempt's base and re-evaluated when a competing commit forces a re-prepare; a change of branch incarnation or accepted schema between attempts fails closed. The receipt's ETag is read from the detached commit before publication. --- changelog.d/blob-cell-put-clear.added.md | 12 + crates/omnigraph-core/src/error.rs | 14 + crates/omnigraph-server/src/blob_transport.rs | 1 + crates/omnigraph-server/src/lib.rs | 3 + crates/omnigraph/src/blob.rs | 234 +++++--- crates/omnigraph/src/exec/blob_write.rs | 501 ++++++++++++++++++ crates/omnigraph/src/exec/mod.rs | 1 + crates/omnigraph/src/exec/mutation.rs | 133 +++-- crates/omnigraph/src/exec/staging.rs | 8 + crates/omnigraph/src/lib.rs | 7 +- crates/omnigraph/src/loader/mod.rs | 1 + .../omnigraph/tests/detached_commit_matrix.rs | 89 +++- crates/omnigraph/tests/end_to_end.rs | 201 ++++++- crates/omnigraph/tests/failpoints.rs | 456 +++++++++++++++- crates/omnigraph/tests/forbidden_apis.rs | 2 + .../omnigraph/tests/policy_engine_chassis.rs | 43 ++ crates/omnigraph/tests/writes.rs | 316 ++++++++++- docs/dev/blob.md | 13 +- docs/dev/testing.md | 2 +- docs/user/blobs.md | 45 +- 20 files changed, 1943 insertions(+), 139 deletions(-) create mode 100644 changelog.d/blob-cell-put-clear.added.md create mode 100644 crates/omnigraph/src/exec/blob_write.rs diff --git a/changelog.d/blob-cell-put-clear.added.md b/changelog.d/blob-cell-put-clear.added.md new file mode 100644 index 000000000..c47eecd10 --- /dev/null +++ b/changelog.d/blob-cell-put-clear.added.md @@ -0,0 +1,12 @@ +- An embedded `Session` can replace or clear one Blob value of an existing + node or edge by exact id with `put_blob_at_as` and `clear_blob_at_as`. Each + write is one graph commit under the `change` action, never reads the old + value, and carries the row's other cells as an `update` does. An optional + precondition, `BlobPrecondition::Tags` (the current ETag is one of the given + tags) or `BlobPrecondition::AnyExisting` (the cell is not null), fails with + `BlobWritePreconditionFailed`, which names the current ETag, and changes + nothing. A put returns the value's length, its ETag and the commit; the ETag + is the one a read at that commit reports. One put holds at most 32 MiB, + inclusive; a larger one fails before any table is opened with resource + `Blob write payload bytes`. The CLI and HTTP server do not expose these + writes yet. diff --git a/crates/omnigraph-core/src/error.rs b/crates/omnigraph-core/src/error.rs index 0b84f1ecf..7a15a386c 100644 --- a/crates/omnigraph-core/src/error.rs +++ b/crates/omnigraph-core/src/error.rs @@ -312,6 +312,16 @@ pub enum OmniError { expected: String, actual: Option, }, + /// A Blob write's `If-Match` precondition did not hold at the write's + /// pinned base, so the write had no effect. Distinct from the graph-head + /// `PreconditionFailed`: it compares the cell's strong validator, and a + /// null or external cell has none. `current_etag` is the managed cell's + /// validator at that base, or `None` when the cell has none. + #[error( + "Blob write precondition failed; the cell's current validator is {}", + current_etag.as_deref().unwrap_or("") + )] + BlobWritePreconditionFailed { current_etag: Option }, /// Engine-layer policy enforcement (MR-722). Wraps either a policy /// denial ("you can't do that") or a policy-evaluation failure /// ("the policy engine itself blew up"). The HTTP layer maps @@ -863,6 +873,10 @@ impl OmniError { } } + pub fn blob_write_precondition_failed(current_etag: Option) -> Self { + Self::BlobWritePreconditionFailed { current_etag } + } + pub fn recovery_required(operation_id: impl Into, reason: impl Into) -> Self { Self::RecoveryRequired { operation_id: operation_id.into(), diff --git a/crates/omnigraph-server/src/blob_transport.rs b/crates/omnigraph-server/src/blob_transport.rs index 2a041342a..38fa4623f 100644 --- a/crates/omnigraph-server/src/blob_transport.rs +++ b/crates/omnigraph-server/src/blob_transport.rs @@ -80,6 +80,7 @@ impl RedactedCause { OmniError::BlobRangeNotSatisfiable { .. } => "BlobRangeNotSatisfiable", OmniError::RecoveryRequired { .. } => "RecoveryRequired", OmniError::PreconditionFailed { .. } => "PreconditionFailed", + OmniError::BlobWritePreconditionFailed { .. } => "BlobWritePreconditionFailed", OmniError::Policy(_) => "Policy", OmniError::AlreadyInitialized { .. } => "AlreadyInitialized", OmniError::InitializationCommitted { .. } => "InitializationCommitted", diff --git a/crates/omnigraph-server/src/lib.rs b/crates/omnigraph-server/src/lib.rs index 76a88b83a..d9c52a7f8 100644 --- a/crates/omnigraph-server/src/lib.rs +++ b/crates/omnigraph-server/src/lib.rs @@ -1471,6 +1471,9 @@ impl ApiError { ), api::PreconditionFailureOutput { expected, actual }, ), + err @ OmniError::BlobWritePreconditionFailed { .. } => { + Self::blob_precondition_failed(err.to_string()) + } err @ OmniError::ExternalBlobPolicy { .. } => Self::bad_request(err.to_string()), err @ OmniError::StoredExternalBlobDenied { .. } => Self::bad_request(err.to_string()), OmniError::ExternalBlobSource { uri, reason } => { diff --git a/crates/omnigraph/src/blob.rs b/crates/omnigraph/src/blob.rs index 45b481884..2978bb9e7 100644 --- a/crates/omnigraph/src/blob.rs +++ b/crates/omnigraph/src/blob.rs @@ -59,6 +59,13 @@ pub struct BlobCell { pub struct BlobEtag(String); impl BlobEtag { + /// A caller-supplied strong entity tag, for a [`BlobPrecondition::Tags`] + /// comparison. It is compared exactly; a tag the engine never issued + /// matches nothing. + pub fn from_tag(tag: impl Into) -> Self { + Self(tag.into()) + } + pub fn as_str(&self) -> &str { &self.0 } @@ -74,6 +81,34 @@ impl fmt::Display for BlobEtag { } } +/// The condition a Blob write requires of the cell at its pinned write base. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum BlobPrecondition { + /// A non-null value exists, managed or external. A null cell fails it. + AnyExisting, + /// The managed value's validator equals one of these. A null or external + /// cell has no validator and fails it. + Tags(Vec), +} + +/// The result of a Blob write: the cell's new state and the exact commit of +/// the publication that made it visible. +#[derive(Debug, Clone)] +pub enum BlobWriteOutcome { + /// The cell now holds managed bytes. `etag` equals what a read at + /// `commit` returns. + Managed { + length: u64, + etag: BlobEtag, + commit: crate::db::GraphCommit, + }, + /// The cell is now null. `commit` is `None` when it already was, and the + /// clear published nothing. + Null { + commit: Option, + }, +} + /// Descriptor-only external Blob reference. Reading the cell never probes or /// dereferences this URI. #[derive(Debug, Clone, PartialEq, Eq)] @@ -1200,11 +1235,109 @@ impl<'a> BlobDescriptorDecoder<'a> { } } -struct ResolvedBlobCell { - table_key: String, +pub(crate) struct ResolvedBlobCell { + pub(crate) table_key: String, + pub(crate) stable_table_id: u64, + pub(crate) table_incarnation_id: u64, + pub(crate) stable_property_id: u64, +} + +/// Find one entity's row in an opened table version and classify its Blob +/// cell from the descriptor alone, without opening any payload: the stable +/// row id and the descriptor, or `None` when no row has the id. A duplicate id +/// or a malformed descriptor is a Blob integrity failure. +pub(crate) async fn locate_blob_cell( + dataset: &lance::Dataset, + id_column: &str, + cell: &BlobCell, +) -> Result> { + let mut scanner = dataset.scan(); + scanner + .project(&[cell.property.as_str()]) + .map_err(OmniError::storage)?; + scanner.filter_expr(col(id_column).eq(lit(cell.id.clone()))); + scanner.blob_handling(BlobHandling::BlobsDescriptions); + scanner.with_row_id(); + scanner.limit(Some(2), None).map_err(OmniError::storage)?; + let batches = scanner + .try_into_stream() + .await + .map_err(OmniError::storage)? + .try_collect::>() + .await + .map_err(OmniError::storage)?; + + let mut selected = None; + for batch in &batches { + let descriptions = batch + .column_by_name(&cell.property) + .and_then(|column| column.as_any().downcast_ref::()) + .ok_or_else(|| { + OmniError::blob_integrity(format!( + "Blob property '{}.{}' did not scan as a Blob-v2 descriptor", + cell.type_name, cell.property + )) + })?; + let row_ids = batch + .column_by_name("_rowid") + .and_then(|column| column.as_any().downcast_ref::()) + .ok_or_else(|| { + OmniError::blob_integrity(format!( + "Blob lookup for '{}.{}' omitted its stable row id", + cell.type_name, cell.property + )) + })?; + if descriptions.len() != row_ids.len() { + return Err(OmniError::blob_integrity(format!( + "Blob descriptor and stable-row-id cardinalities differ for '{}.{}'", + cell.type_name, cell.property + ))); + } + let decoder = BlobDescriptorDecoder::try_new(descriptions)?; + for row in 0..batch.num_rows() { + if selected.is_some() { + return Err(OmniError::blob_integrity(format!( + "entity id '{}' appears more than once in {} type '{}'", + cell.id, + entity_label(cell.entity), + cell.type_name + ))); + } + if row_ids.is_null(row) { + return Err(OmniError::blob_integrity(format!( + "Blob lookup for '{}.{}' returned a null stable row id", + cell.type_name, cell.property + ))); + } + selected = Some((row_ids.value(row), decoder.classify(row)?)); + } + } + Ok(selected) +} + +/// The strong validator of a managed cell in this exact opened table version: +/// table and property identity, the opened version, the row's stable id and +/// the transaction file named by the version's immutable manifest. A write +/// computes it from its own detached commit, a read from the version it opened, +/// and the two agree for the same version and row. +pub(crate) fn managed_blob_etag( + dataset: &lance::Dataset, stable_table_id: u64, table_incarnation_id: u64, stable_property_id: u64, + stable_row_id: u64, + cell: &BlobCell, +) -> Result { + let transaction_file = + immutable_transaction_witness(dataset.manifest().transaction_file.as_deref(), cell)?; + Ok(blob_etag( + stable_table_id, + table_incarnation_id, + stable_property_id, + dataset.version().version, + stable_row_id, + transaction_file, + )) } decide_seam! { @@ -1442,77 +1575,17 @@ impl Omnigraph { None => {} } - let mut scanner = dataset.scan(); - scanner - .project(&[cell.property.as_str()]) - .map_err(OmniError::storage)?; - scanner.filter_expr(col(catalog.system_columns.id).eq(lit(cell.id.clone()))); - scanner.blob_handling(BlobHandling::BlobsDescriptions); - scanner.with_row_id(); - scanner.limit(Some(2), None).map_err(OmniError::storage)?; - let stream = scanner - .try_into_stream() - .await - .map_err(OmniError::storage)?; - let batches = stream - .try_collect::>() - .await - .map_err(OmniError::storage)?; - - let mut selected = None; - for batch in &batches { - let descriptions = batch - .column_by_name(&cell.property) - .and_then(|column| column.as_any().downcast_ref::()) - .ok_or_else(|| { - OmniError::blob_integrity(format!( - "Blob property '{}.{}' did not scan as a Blob-v2 descriptor", - cell.type_name, cell.property - )) - })?; - let row_ids = batch - .column_by_name("_rowid") - .and_then(|column| column.as_any().downcast_ref::()) + let (stable_row_id, descriptor) = + locate_blob_cell(&dataset, catalog.system_columns.id, &cell) + .await? .ok_or_else(|| { - OmniError::blob_integrity(format!( - "Blob lookup for '{}.{}' omitted its stable row id", - cell.type_name, cell.property + OmniError::manifest_not_found(format!( + "no {} '{}' with id '{}' found", + entity_label(cell.entity), + cell.type_name, + cell.id )) })?; - if descriptions.len() != row_ids.len() { - return Err(OmniError::blob_integrity(format!( - "Blob descriptor and stable-row-id cardinalities differ for '{}.{}'", - cell.type_name, cell.property - ))); - } - let decoder = BlobDescriptorDecoder::try_new(descriptions)?; - for row in 0..batch.num_rows() { - if selected.is_some() { - return Err(OmniError::blob_integrity(format!( - "entity id '{}' appears more than once in {} type '{}'", - cell.id, - entity_label(cell.entity), - cell.type_name - ))); - } - if row_ids.is_null(row) { - return Err(OmniError::blob_integrity(format!( - "Blob lookup for '{}.{}' returned a null stable row id", - cell.type_name, cell.property - ))); - } - selected = Some((row_ids.value(row), decoder.classify(row)?)); - } - } - - let (stable_row_id, descriptor) = selected.ok_or_else(|| { - OmniError::manifest_not_found(format!( - "no {} '{}' with id '{}' found", - entity_label(cell.entity), - cell.type_name, - cell.id - )) - })?; let content = match descriptor { BlobDescriptor::Null => { @@ -1536,8 +1609,12 @@ impl Omnigraph { // after branch delete/recreate. The transaction-file identity // is stored in this exact immutable manifest and carries the // UUID of the transaction that produced this table version. - let transaction_file = immutable_transaction_witness( - dataset.manifest().transaction_file.as_deref(), + let etag = managed_blob_etag( + &dataset, + stable_table_id, + table_incarnation_id, + resolved_cell.stable_property_id, + stable_row_id, &cell, )?; let mut files = dataset @@ -1569,14 +1646,7 @@ impl Omnigraph { } BlobContent::Managed { length, - etag: blob_etag( - stable_table_id, - table_incarnation_id, - resolved_cell.stable_property_id, - actual_table_version, - stable_row_id, - transaction_file, - ), + etag, reader: BlobReader { _dataset: Arc::clone(&dataset), file: Arc::new(file), @@ -1594,7 +1664,7 @@ impl Omnigraph { } } -fn resolve_blob_cell( +pub(crate) fn resolve_blob_cell( catalog: &omnigraph_compiler::catalog::Catalog, cell: &BlobCell, ) -> Result { @@ -1661,7 +1731,7 @@ fn resolve_blob_cell( }) } -fn entity_label(kind: EntityKind) -> &'static str { +pub(crate) fn entity_label(kind: EntityKind) -> &'static str { match kind { EntityKind::Node => "node", EntityKind::Edge => "edge", diff --git a/crates/omnigraph/src/exec/blob_write.rs b/crates/omnigraph/src/exec/blob_write.rs new file mode 100644 index 000000000..2021b73b1 --- /dev/null +++ b/crates/omnigraph/src/exec/blob_write.rs @@ -0,0 +1,501 @@ +//! Blob cell writes (RFC 0033 Phase 3): replace one cell with managed bytes, +//! or clear it, on an existing node or edge addressed by exact id. +//! +//! A cell write is not a new writer kind. It stages the row's replacement +//! through Mutation's staging and validation, commits it detached and publishes +//! it once through the Mutation tail. Lance has no single-cell Blob write, so +//! the whole row is replaced and every other Blob cell of the row is carried by +//! value, under the rule a `.gq` update follows. + +use std::sync::Arc; + +use arrow_array::{ArrayRef, LargeBinaryArray, RecordBatch, StringArray, StructArray}; +use arrow_schema::{DataType, SchemaRef}; +use bytes::Bytes; +use datafusion::arrow::buffer::{Buffer, OffsetBuffer, ScalarBuffer}; +use datafusion::prelude::{col, lit}; + +use super::mutation::{ + build_null_blob_array, concat_match_batches_to_schema, open_table_for_mutation, +}; +use super::staging::{MutationStaging, PendingMode}; +use crate::blob::{ + BlobCell, BlobDescriptor, BlobEtag, BlobPrecondition, BlobWriteOutcome, entity_label, + locate_blob_cell, managed_blob_etag, resolve_blob_cell, +}; +use crate::changes::EntityKind; +use crate::db::Omnigraph; +use crate::db::manifest::HistoryReleaseBytes; +use crate::error::{OmniError, Result}; +use crate::session::Session; +use crate::storage_layer::{KEYED_BLOB_PAYLOAD_MAX_BYTES, PendingScanBudget}; + +/// What one Blob cell write stores. +enum BlobCellValue { + Bytes(Bytes), + Null, +} + +/// The authority a write attempt captured that a re-prepared attempt must +/// still see: the native branch incarnation and the accepted schema. +#[derive(PartialEq, Eq)] +struct AttemptIdentity { + branch_identifier: lance::dataset::refs::BranchIdentifier, + schema_ir_hash: String, + schema_identity_domain: String, + schema_identity_version: u32, +} + +impl AttemptIdentity { + fn of(txn: &crate::db::WriteTxn) -> Self { + Self { + branch_identifier: txn.authority.branch_identifier.clone(), + schema_ir_hash: txn.authority.schema_ir_hash.clone(), + schema_identity_domain: txn.authority.schema_identity_domain.clone(), + schema_identity_version: txn.authority.schema_identity_version, + } + } +} + +impl Session { + /// Replace one Blob cell of an existing node or edge with managed bytes. + /// + /// The entity must exist; the write never inserts a row. `bytes` is at + /// most 32 MiB, inclusive, and is refused before any table is opened when + /// it is larger. The row's other Blob cells are carried by value, so they + /// share the operation's 32 MiB payload allowance, and a stored external + /// reference among them must be admitted by the graph's external Blob + /// policy (`StoredExternalBlobDenied` otherwise). A failed `precondition` + /// returns `BlobWritePreconditionFailed` without effect. The outcome's + /// ETag equals what a read at its commit returns. + pub async fn put_blob_at_as( + &self, + branch: &str, + cell: BlobCell, + bytes: Bytes, + precondition: Option, + actor_id: Option<&str>, + ) -> Result { + let length = u64::try_from(bytes.len()) + .map_err(|_| OmniError::manifest_internal("Blob write payload length exceeds u64"))?; + if length > KEYED_BLOB_PAYLOAD_MAX_BYTES { + return Err(OmniError::resource_limit( + "Blob write payload bytes", + KEYED_BLOB_PAYLOAD_MAX_BYTES, + length, + )); + } + self.write_blob_cell_as( + branch, + cell, + BlobCellValue::Bytes(bytes), + precondition, + actor_id, + ) + .await + } + + /// Set one nullable Blob cell of an existing node or edge to null. + /// + /// Clearing a non-nullable property is refused. A cell that is already + /// null, with no `precondition`, publishes nothing and returns + /// `Null { commit: None }`; with a precondition it fails, since a null + /// cell satisfies neither form. + pub async fn clear_blob_at_as( + &self, + branch: &str, + cell: BlobCell, + precondition: Option, + actor_id: Option<&str>, + ) -> Result { + self.write_blob_cell_as(branch, cell, BlobCellValue::Null, precondition, actor_id) + .await + } + + async fn write_blob_cell_as( + &self, + branch: &str, + cell: BlobCell, + value: BlobCellValue, + precondition: Option, + actor_id: Option<&str>, + ) -> Result { + let requested = Omnigraph::normalize_branch_name(branch)?; + self.enforce( + omnigraph_policy::PolicyAction::Change, + &omnigraph_policy::ResourceScope::Branch( + requested.as_deref().unwrap_or("main").to_string(), + ), + actor_id, + )?; + let settings = self.effective("")?; + self.write_blob_cell( + requested.as_deref(), + &cell, + &value, + precondition.as_ref(), + actor_id, + settings.stage_write_concurrency(), + HistoryReleaseBytes(settings.history_release_bytes()), + ) + .await + } +} + +impl Omnigraph { + /// Re-prepare after a pre-effect read-set change, within the bound an + /// insert-only mutation uses. Unlike a predicate update, whose + /// read-modify-write plan would rebase, the stored value is the caller's, + /// so a fresh attempt is the same write. Each attempt re-reads the cell, + /// re-evaluates the precondition and re-carries the row's other cells. + #[allow(clippy::too_many_arguments)] + async fn write_blob_cell( + &self, + requested: Option<&str>, + cell: &BlobCell, + value: &BlobCellValue, + precondition: Option<&BlobPrecondition>, + actor_id: Option<&str>, + stage_write_concurrency: usize, + history_release_bytes: HistoryReleaseBytes, + ) -> Result { + const MAX_PRE_EFFECT_REPREPARES: usize = 32; + + let mut first = None; + for attempt in 0..=MAX_PRE_EFFECT_REPREPARES { + match self + .write_blob_cell_attempt( + requested, + cell, + value, + precondition, + actor_id, + stage_write_concurrency, + history_release_bytes, + &mut first, + ) + .await + { + Err(error) + if error.is_read_set_changed() && attempt < MAX_PRE_EFFECT_REPREPARES => + { + tracing::debug!( + attempt = attempt + 1, + branch = requested.unwrap_or("main"), + "prepared Blob write authority changed before effects; repreparing" + ); + crate::instrumentation::record_mutation_reprepare(); + self.refresh_coordinator_only().await?; + } + result => return result, + } + } + unreachable!("bounded Blob write retry loop always returns") + } + + #[allow(clippy::too_many_arguments)] + async fn write_blob_cell_attempt( + &self, + requested: Option<&str>, + cell: &BlobCell, + value: &BlobCellValue, + precondition: Option<&BlobPrecondition>, + actor_id: Option<&str>, + stage_write_concurrency: usize, + history_release_bytes: HistoryReleaseBytes, + first: &mut Option, + ) -> Result { + let first_attempt = first.is_none(); + let txn = self.open_write_txn(requested).await.map_err(|error| { + if first_attempt { + error.before_effect() + } else { + error.without_pre_effect_evidence() + } + })?; + let identity = AttemptIdentity::of(&txn); + match first { + None => *first = Some(identity), + Some(first) if *first != identity => { + return Err(OmniError::manifest_conflict(format!( + "branch '{}' changed incarnation or accepted schema while a Blob write \ + re-prepared; the write was not applied", + requested.unwrap_or("main") + ))); + } + Some(_) => {} + } + + let catalog = Arc::clone(&txn.catalog); + let resolved = resolve_blob_cell(&catalog, cell)?; + let schema: SchemaRef = match cell.entity { + EntityKind::Node => Arc::clone(&catalog.node_types[&cell.type_name].arrow_schema), + EntityKind::Edge => Arc::clone(&catalog.edge_types[&cell.type_name].arrow_schema), + }; + let target = schema.index_of(&cell.property).map_err(|_| { + OmniError::manifest_internal(format!( + "Blob property '{}.{}' is missing from its type's schema", + cell.type_name, cell.property + )) + })?; + if matches!(value, BlobCellValue::Null) && !schema.field(target).is_nullable() { + return Err(OmniError::manifest(format!( + "Blob property '{}.{}' is not nullable, so it cannot be cleared", + cell.type_name, cell.property + ))); + } + let entry = txn.base.dataset(&resolved.table_key).ok_or_else(|| { + OmniError::manifest(format!( + "{} type '{}' is unavailable on the write branch", + entity_label(cell.entity), + cell.type_name + )) + })?; + if entry.identity.stable_table_id != resolved.stable_table_id + || entry.identity.table_incarnation_id != resolved.table_incarnation_id + { + return Err(OmniError::blob_integrity(format!( + "write base for {} type '{}' has identity {}, expected {:016x}:{:016x}", + entity_label(cell.entity), + cell.type_name, + entry.identity, + resolved.stable_table_id, + resolved.table_incarnation_id + ))); + } + + let mut staging = MutationStaging::default(); + let (handle, _full_path, _table_branch) = open_table_for_mutation( + self, + &mut staging, + requested, + &resolved.table_key, + crate::db::MutationOpKind::Update, + Some(&txn), + ) + .await?; + let base = handle.ok_or_else(|| { + OmniError::manifest_internal("a strict Blob write opened no table handle") + })?; + let id_column = catalog.system_columns.id; + let (base_row_id, base_cell) = locate_blob_cell(base.dataset(), id_column, cell) + .await? + .ok_or_else(|| { + OmniError::manifest_not_found(format!( + "no {} '{}' with id '{}' found", + entity_label(cell.entity), + cell.type_name, + cell.id + )) + })?; + let current_etag = match base_cell { + BlobDescriptor::Managed { .. } => Some(managed_blob_etag( + base.dataset(), + resolved.stable_table_id, + resolved.table_incarnation_id, + resolved.stable_property_id, + base_row_id, + cell, + )?), + BlobDescriptor::Null | BlobDescriptor::External { .. } => None, + }; + if let Some(precondition) = precondition { + let holds = match precondition { + BlobPrecondition::AnyExisting => !matches!(base_cell, BlobDescriptor::Null), + BlobPrecondition::Tags(tags) => current_etag + .as_ref() + .is_some_and(|current| tags.contains(current)), + }; + if !holds { + return Err(OmniError::blob_write_precondition_failed( + current_etag.map(BlobEtag::into_string), + )); + } + } + if matches!(value, BlobCellValue::Null) && matches!(base_cell, BlobDescriptor::Null) { + return Ok(BlobWriteOutcome::Null { commit: None }); + } + + // The row's other cells, carried by value, with the target column left + // out so its old payload is never read. + let (pending_rows, pending_bytes) = staging.pending_resource_usage(&resolved.table_key)?; + let budget = PendingScanBudget::new(&resolved.table_key, pending_rows, pending_bytes); + let carried = self + .storage() + .scan_with_pending_materialized_blobs( + &base, + &[], + None, + Some(col(id_column).eq(lit(cell.id.clone()))), + Some(id_column), + &[cell.property.as_str()], + budget, + ) + .await?; + let carried_indices = (0..schema.fields().len()) + .filter(|index| *index != target) + .collect::>(); + let carried_schema = Arc::new( + schema + .project(&carried_indices) + .map_err(OmniError::arrow_internal)?, + ); + let carried = concat_match_batches_to_schema(&carried_schema, carried)?; + if carried.num_rows() != 1 { + return Err(OmniError::blob_integrity(format!( + "{} '{}' id '{}' matched {} rows while carrying its row", + entity_label(cell.entity), + cell.type_name, + cell.id, + carried.num_rows() + ))); + } + let target_column = match value { + BlobCellValue::Bytes(bytes) => { + managed_blob_column(schema.field(target).data_type(), bytes)? + } + BlobCellValue::Null => build_null_blob_array(1)?, + }; + let mut columns = carried.columns().to_vec(); + columns.insert(target, target_column); + let row = RecordBatch::try_new(Arc::clone(&schema), columns) + .map_err(OmniError::arrow_internal)?; + staging.append_batch( + &resolved.table_key, + Arc::clone(&schema), + PendingMode::Upsert, + row, + )?; + + let committed = self + .commit_staged_mutation( + staging, + &txn, + actor_id, + stage_write_concurrency, + history_release_bytes, + ) + .await?; + // The validator evidence comes from the exact detached version the pin + // will name, read before publication; nothing after the CAS reads + // storage, so a later write cannot change this outcome. + let managed = match value { + BlobCellValue::Bytes(bytes) => { + let detached = committed + .committed + .detached + .iter() + .find(|(table_key, _)| *table_key == resolved.table_key) + .map(|(_, handle)| handle) + .ok_or_else(|| { + OmniError::manifest_internal(format!( + "Blob write committed no detached version of '{}'", + resolved.table_key + )) + })?; + let (row_id, written) = locate_blob_cell(detached.dataset(), id_column, cell) + .await? + .ok_or_else(|| { + OmniError::blob_integrity(format!( + "the detached commit of '{}' lost id '{}'", + resolved.table_key, cell.id + )) + })?; + let length = u64::try_from(bytes.len()).map_err(|_| { + OmniError::manifest_internal("Blob write payload length exceeds u64") + })?; + if written != (BlobDescriptor::Managed { length }) { + return Err(OmniError::blob_integrity(format!( + "the detached commit of '{}.{}' stores {written:?}, expected a managed value of {length} bytes", + cell.type_name, cell.property + ))); + } + Some(( + length, + managed_blob_etag( + detached.dataset(), + resolved.stable_table_id, + resolved.table_incarnation_id, + resolved.stable_property_id, + row_id, + cell, + )?, + )) + } + BlobCellValue::Null => None, + }; + let commit = self + .publish_committed_mutation(committed, &txn, actor_id) + .await?; + Ok(match managed { + Some((length, etag)) => BlobWriteOutcome::Managed { + length, + etag, + commit, + }, + None => BlobWriteOutcome::Null { + commit: Some(commit), + }, + }) + } +} + +/// One logical Blob value holding `bytes` as managed data, in the column's +/// logical `{data, uri}` shape. Arrow adopts the bytes' buffer, so the payload +/// is not copied here. +fn managed_blob_column(data_type: &DataType, bytes: &Bytes) -> Result { + let DataType::Struct(children) = data_type else { + return Err(OmniError::manifest_internal( + "a Blob column's catalog type is not a struct", + )); + }; + if children.len() != 2 || children[0].name() != "data" || children[1].name() != "uri" { + return Err(OmniError::manifest_internal( + "a Blob column's catalog type is not the logical {data, uri} shape", + )); + } + let length = i64::try_from(bytes.len()) + .map_err(|_| OmniError::manifest_internal("Blob write payload length exceeds i64"))?; + let data = LargeBinaryArray::try_new( + OffsetBuffer::new(ScalarBuffer::from(vec![0_i64, length])), + Buffer::from(bytes.clone()), + None, + ) + .map_err(OmniError::arrow_internal)?; + let uri = StringArray::new_null(1); + let column = StructArray::try_new( + children.clone(), + vec![Arc::new(data) as ArrayRef, Arc::new(uri) as ArrayRef], + None, + ) + .map_err(OmniError::arrow_internal)?; + Ok(Arc::new(column)) +} + +#[cfg(test)] +mod tests { + use arrow_array::Array; + + use super::*; + + #[test] + fn managed_blob_column_adopts_the_payload_buffer() { + let field = lance::blob::blob_field("content", true); + let bytes = Bytes::from(vec![7_u8; 1024]); + let column = managed_blob_column(field.data_type(), &bytes).unwrap(); + let column = column.as_any().downcast_ref::().unwrap(); + let data = column + .column(0) + .as_any() + .downcast_ref::() + .unwrap(); + assert_eq!(data.value(0), &bytes[..]); + assert_eq!( + data.values().as_ptr(), + bytes.as_ptr(), + "the payload is adopted, not copied" + ); + assert!(column.column(1).is_null(0)); + assert!(column.is_valid(0)); + } +} diff --git a/crates/omnigraph/src/exec/mod.rs b/crates/omnigraph/src/exec/mod.rs index 546c6c305..7f2ccc48c 100644 --- a/crates/omnigraph/src/exec/mod.rs +++ b/crates/omnigraph/src/exec/mod.rs @@ -33,6 +33,7 @@ use crate::error::{MergeConflict, MergeConflictKind, OmniError, Result}; use crate::storage_layer::SnapshotHandle; use tempfile::{Builder as TempDirBuilder, TempDir}; +mod blob_write; pub(crate) mod merge; pub(crate) mod mutation; mod query_doors; diff --git a/crates/omnigraph/src/exec/mutation.rs b/crates/omnigraph/src/exec/mutation.rs index 0e0736371..4a8e7ee39 100644 --- a/crates/omnigraph/src/exec/mutation.rs +++ b/crates/omnigraph/src/exec/mutation.rs @@ -349,7 +349,7 @@ fn build_blob_array_from_value(value: &str) -> Result { } /// Build a null blob array with `num_rows` elements. -fn build_null_blob_array(num_rows: usize) -> Result { +pub(super) fn build_null_blob_array(num_rows: usize) -> Result { let mut builder = BlobArrayBuilder::new(num_rows); for _ in 0..num_rows { builder.push_null().map_err(OmniError::lance_internal)?; @@ -503,6 +503,13 @@ fn apply_assignments( use super::staging::{MutationStaging, PendingMode}; +/// Every table effect of a Mutation committed detached, with the gates and the +/// lineage intent its one publication needs. +pub(super) struct CommittedStagedMutation { + pub(super) committed: super::staging::CommittedMutation, + lineage_intent: crate::db::manifest::LineageIntent, +} + /// Open a sub-table dataset for read or staged write within the current /// mutation query, capturing pre-write metadata in `staging` on first touch. /// The captured table version is the physical staging baseline. The publisher's @@ -525,7 +532,7 @@ use super::staging::{MutationStaging, PendingMode}; /// touch records another predicate (`record_delete`), and `stage_all` combines /// them into one staged delete — there is no post-inline-commit reopen to /// special-case anymore. -async fn open_table_for_mutation( +pub(super) async fn open_table_for_mutation( db: &Omnigraph, staging: &mut MutationStaging, branch: Option<&str>, @@ -1007,49 +1014,18 @@ impl Omnigraph { }) } Ok(total) => { - self.validate_staged_mutation(&staging, &txn).await?; - let staged = staging - .stage_all_with_concurrency(self, requested.as_deref(), stage_write_concurrency) - .await?; - fail(&MUTATION_POST_STAGE_PRE_EFFECT_GATE)?; - let lineage_intent = self - .new_lineage_intent_for_branch( - requested.as_deref(), + let committed = self + .commit_staged_mutation( + staging, + &txn, actor_id, + stage_write_concurrency, history_release_bytes, ) .await?; - // `_held_gates` holds the shared schema permit, branch - // effect gate, and sorted table gates acquired by `commit_all`. - // They remain held through manifest publication, covering the - // complete same-process effect lifetime. They are a local - // serialization aid; the exact publisher precondition remains - // the correctness authority. - let super::staging::CommittedMutation { - updates, - expected_versions, - gates: _held_gates, - } = staged.commit_all(self, requested.as_deref(), &txn).await?; - // Failpoint for the detached-effects → publisher boundary: - // every table effect is committed detached but nothing is - // graph-visible. A failure here leaves the graph unchanged and - // the detached versions as reclaimable garbage. See - // `tests/failpoints.rs::finalize_publisher_residual_does_not_drift_untouched_tables`. - fail(&MUTATION_POST_FINALIZE_PRE_PUBLISHER)?; - let publish_result = self - .commit_updates_on_branch_with_expected( - requested.as_deref(), - &updates, - &expected_versions, - actor_id, - &txn, - lineage_intent, - ) - .await; - // RFC 0067: every effect is a detached commit of its pinned base, - // so a publish failure leaves the graph unchanged; the error - // is returned as is (a moved head is `ReadSetChanged`). - let commit = publish_result?; + let commit = self + .publish_committed_mutation(committed, &txn, actor_id) + .await?; Ok(crate::MutationReceipt { result: total, commit: Some(commit), @@ -1058,6 +1034,79 @@ impl Omnigraph { } } + /// The first half of the Mutation protocol's tail, shared by `.gq` + /// mutations and Blob cell writes: validate the staged change set, stage + /// every touched table, and commit each effect detached. Nothing is + /// graph-visible yet; the held gates and the lineage intent travel to + /// [`Self::publish_committed_mutation`]. + pub(super) async fn commit_staged_mutation( + &self, + staging: MutationStaging, + txn: &crate::db::WriteTxn, + actor_id: Option<&str>, + stage_write_concurrency: usize, + history_release_bytes: HistoryReleaseBytes, + ) -> Result { + let requested = txn.branch.as_deref(); + self.validate_staged_mutation(&staging, txn).await?; + let staged = staging + .stage_all_with_concurrency(self, requested, stage_write_concurrency) + .await?; + fail(&MUTATION_POST_STAGE_PRE_EFFECT_GATE)?; + let lineage_intent = self + .new_lineage_intent_for_branch(requested, actor_id, history_release_bytes) + .await?; + let committed = staged.commit_all(self, requested, txn).await?; + Ok(CommittedStagedMutation { + committed, + lineage_intent, + }) + } + + /// The second half of the Mutation protocol's tail: publish every detached + /// effect in one manifest CAS. This is the protocol's one publisher call. + pub(super) async fn publish_committed_mutation( + &self, + committed: CommittedStagedMutation, + txn: &crate::db::WriteTxn, + actor_id: Option<&str>, + ) -> Result { + let CommittedStagedMutation { + committed: + super::staging::CommittedMutation { + updates, + expected_versions, + detached: _, + gates: _held_gates, + }, + lineage_intent, + } = committed; + // `_held_gates` holds the shared schema permit, branch effect gate, + // and sorted table gates acquired by `commit_all`. They remain held + // through manifest publication, covering the complete same-process + // effect lifetime. They are a local serialization aid; the exact + // publisher precondition remains the correctness authority. + // + // Failpoint for the detached-effects → publisher boundary: every + // table effect is committed detached but nothing is graph-visible. A + // failure here leaves the graph unchanged and the detached versions as + // reclaimable garbage. See + // `tests/failpoints.rs::finalize_publisher_residual_does_not_drift_untouched_tables`. + fail(&MUTATION_POST_FINALIZE_PRE_PUBLISHER)?; + // RFC 0067: every effect is a detached commit of its pinned base, so a + // publish failure leaves the graph unchanged; the error is returned as + // is (a moved head is `ReadSetChanged`). + self.commit_updates_on_branch_with_expected( + txn.branch.as_deref(), + &updates, + &expected_versions, + actor_id, + txn, + lineage_intent, + ) + .await + } + /// Lower + validate a named mutation query into its IR. /// /// Hoisted out of [`Self::execute_named_mutation`] so the caller can @@ -1723,7 +1772,7 @@ async fn scan_deleted_ids( /// order; both should share a schema if pending was produced through /// `apply_assignments` with full-schema scan input. If schemas drift, /// surface the internal contract failure at the mutation boundary. -fn concat_match_batches_to_schema( +pub(super) fn concat_match_batches_to_schema( schema: &SchemaRef, batches: Vec, ) -> Result { diff --git a/crates/omnigraph/src/exec/staging.rs b/crates/omnigraph/src/exec/staging.rs index cd1d960e8..8b3663185 100644 --- a/crates/omnigraph/src/exec/staging.rs +++ b/crates/omnigraph/src/exec/staging.rs @@ -823,6 +823,10 @@ pub(crate) struct CommittedMutation { /// publisher checks them together with native branch identity, exact graph /// head, and schema identity as one authority precondition. pub(crate) expected_versions: crate::db::manifest::ExpectedTableVersions, + /// Each table's detached commit, keyed by table: the exact version its + /// pin will name, before publication. A Blob write reads its validator + /// evidence from here, never from a branch head. + pub(crate) detached: Vec<(String, crate::storage_layer::SnapshotHandle)>, /// The write envelope: shared schema permit, coarse branch gate, and /// sorted `(table, branch)` guards. The caller MUST hold the complete /// set across manifest publish (see `commit_all`) so no same-process @@ -931,6 +935,7 @@ impl StagedMutation { return Ok(CommittedMutation { updates: Vec::new(), expected_versions, + detached: Vec::new(), gates, }); } @@ -951,6 +956,7 @@ impl StagedMutation { let witness = txn.authority.staging_witness()?; let mut updates: Vec = Vec::with_capacity(staged.len()); + let mut detached_tables = Vec::with_capacity(staged.len()); for entry in staged { let StagedTableEntry { table_key, @@ -976,6 +982,7 @@ impl StagedMutation { .with_table_fork_owner(table_fork_owner) .with_staged(state.version, identity.uuid.clone()) .with_last_linear_version(path.entry.version_metadata.last_linear_version()); + detached_tables.push((table_key.clone(), detached)); updates.push(DatasetUpdate { identity: path.identity, type_key: table_key, @@ -990,6 +997,7 @@ impl StagedMutation { Ok(CommittedMutation { updates, expected_versions, + detached: detached_tables, gates, }) } diff --git a/crates/omnigraph/src/lib.rs b/crates/omnigraph/src/lib.rs index e8ec4f033..9ed1bb5d9 100644 --- a/crates/omnigraph/src/lib.rs +++ b/crates/omnigraph/src/lib.rs @@ -47,9 +47,10 @@ pub(crate) mod table_store; pub(crate) mod validate; pub use blob::{ - BLOB_READ_RANGE_MAX_BYTES, BlobCell, BlobContent, BlobEtag, BlobRead, BlobReader, - EXTERNAL_BLOB_URI_MAX_BYTES, ExternalBlobBase, ExternalBlobExecutionScope, ExternalBlobPolicy, - ExternalBlobRef, RangedExternalBlob, StorageRootConflict, + BLOB_READ_RANGE_MAX_BYTES, BlobCell, BlobContent, BlobEtag, BlobPrecondition, BlobRead, + BlobReader, BlobWriteOutcome, EXTERNAL_BLOB_URI_MAX_BYTES, ExternalBlobBase, + ExternalBlobExecutionScope, ExternalBlobPolicy, ExternalBlobRef, RangedExternalBlob, + StorageRootConflict, }; pub use changes::EntityKind; pub use omnigraph_compiler::settings; diff --git a/crates/omnigraph/src/loader/mod.rs b/crates/omnigraph/src/loader/mod.rs index 1a6f8734f..5d93b3eec 100644 --- a/crates/omnigraph/src/loader/mod.rs +++ b/crates/omnigraph/src/loader/mod.rs @@ -949,6 +949,7 @@ async fn load_jsonl_reader_once( let crate::exec::staging::CommittedMutation { updates, expected_versions, + detached: _, gates: _held_gates, } = staged.commit_all(db, branch, &txn).await?; // Same detached-effects → publisher boundary as mutations: every table diff --git a/crates/omnigraph/tests/detached_commit_matrix.rs b/crates/omnigraph/tests/detached_commit_matrix.rs index c0dbebaf8..03becaf6d 100644 --- a/crates/omnigraph/tests/detached_commit_matrix.rs +++ b/crates/omnigraph/tests/detached_commit_matrix.rs @@ -25,15 +25,18 @@ mod helpers; use std::sync::Arc; use std::sync::atomic::{AtomicUsize, Ordering}; +use bytes::Bytes; use helpers::recovery::sidecar_operation_ids; use helpers::{ - MUTATION_QUERIES, Session, collect_column_strings, count_rows, init_and_load, mixed_params, - mutate_main, read_table, + MUTATION_QUERIES, Session, collect_column_strings, count_rows, init_and_load, + init_and_load_with_schema, mixed_params, mutate_main, read_table, }; +use omnigraph::BlobCell; use omnigraph::db::{ CleanupPolicyOptions, Omnigraph, PreparedSchemaApply, ReadTarget, SystemColumnUpgradeOptions, SystemColumnUpgradeOutcome, }; +use omnigraph::error::{ManifestErrorKind, OmniError}; use omnigraph::loader::LoadMode; use omnigraph::seams::FailScenario; use omnigraph::seams::catalog; @@ -51,7 +54,8 @@ const PARK_HIT_ENV: &str = "OMNIGRAPH_RFC0067_PARK_HIT"; const PREPARED_SCHEMA_FILE: &str = "matrix-prepared-schema.json"; /// What the child runs: one of the `Writer::child_op` strings (`insert`, /// `insert_and_friend`, `cleanup`, `ensure_indices`, `merge`, `schema_apply`, -/// `prepared_schema_apply`, `optimize`, `load`, `fts_rebuild`, `system_column_upgrade`). +/// `prepared_schema_apply`, `optimize`, `load`, `fts_rebuild`, `system_column_upgrade`, +/// `blob_put`, `blob_clear`). const OP_ENV: &str = "OMNIGRAPH_RFC0067_OP"; #[derive(Clone, Copy, Debug, PartialEq, Eq)] @@ -86,6 +90,26 @@ enum Writer { /// spellings: one rename-only detached Project per table, staged and /// published through the schema-apply seams. SystemColumnUpgrade, + /// A Blob put replacing Alice's `photo` by exact id: one detached upsert + /// of the carried row through the mutation seams. + BlobPut, + /// A Blob clear of Alice's seeded `photo`: the same single-row upsert + /// carrying a null value. + BlobClear, +} + +fn photo_schema() -> String { + helpers::TEST_SCHEMA.replace("age: I32?", "age: I32?\n photo: Blob?") +} + +/// The Blob cell the Blob writers replace. +fn alice_photo() -> BlobCell { + helpers::node_blob_cell("Person", "Alice", "photo") +} + +/// The bytes a Blob put by `name` stores. +fn photo_bytes(name: &str) -> Bytes { + Bytes::from(format!("{name}-photo").into_bytes()) } fn city_schema() -> String { @@ -136,6 +160,7 @@ impl Writer { // The upgrade stages one detached commit per table; park after // the first and the second to leave a half-staged tail. Writer::SystemColumnUpgrade => vec![PostDetached(1), PostDetached(2), PrePublish], + Writer::BlobPut | Writer::BlobClear => vec![PostDetached(1), PrePublish], } } @@ -152,6 +177,8 @@ impl Writer { Writer::Load => "load", Writer::FtsRebuild => "fts_rebuild", Writer::SystemColumnUpgrade => "system_column_upgrade", + Writer::BlobPut => "blob_put", + Writer::BlobClear => "blob_clear", } } } @@ -280,6 +307,34 @@ async fn insert_and_friend(db: &Session, name: &str) -> omnigraph::error::Result .map(|_| ()) } +/// The BlobPut writer's op: replace Alice's photo with `name`'s bytes. +async fn put_photo(db: &Session, name: &str) -> omnigraph::error::Result<()> { + db.put_blob_at_as("main", alice_photo(), photo_bytes(name), None, None) + .await + .map(|_| ()) +} + +/// The BlobClear writer's op. +async fn clear_photo(db: &Session) -> omnigraph::error::Result<()> { + db.clear_blob_at_as("main", alice_photo(), None, None) + .await + .map(|_| ()) +} + +/// Alice's photo, `None` when the cell is null. +async fn observe_photo(db: &Omnigraph) -> Option> { + match db + .read_blob_at(ReadTarget::branch("main"), alice_photo()) + .await + { + Ok(_) => Some( + helpers::read_managed_blob_bytes(db, ReadTarget::branch("main"), alice_photo()).await, + ), + Err(OmniError::Manifest(error)) if error.kind == ManifestErrorKind::NotFound => None, + Err(error) => panic!("reading Alice's photo failed: {error}"), + } +} + /// The Load writer's op: two fresh rows through the loader's staging. async fn load_two(db: &Session, name: &str) -> omnigraph::error::Result<()> { let payload = format!( @@ -377,6 +432,8 @@ fn rfc0067_matrix_child_process() { .upgrade_system_columns(upgrade_execute()) .await .map(|_| ()), + "blob_put" => put_photo(&db, &name).await, + "blob_clear" => clear_photo(&db).await, _ => insert(&db, &name).await, }; if let Err(error) = outcome { @@ -574,6 +631,8 @@ async fn run_cell( .await .unwrap(); db + } else if matches!(writer, Writer::BlobPut | Writer::BlobClear) { + init_and_load_with_schema(&dir, &photo_schema()).await } else { init_and_load(&dir).await }; @@ -627,6 +686,11 @@ async fn run_cell( .unwrap(); db.ensure_indices().await.unwrap(); } + // A clear cell clears a photo a put stored first. + if writer == Writer::BlobClear { + put_photo(&db, &format!("m{index}_seed")).await.unwrap(); + } + let photo_before = observe_photo(&db).await; let (mut model, _) = observe_model(&db).await; let head_before = linear_head(&person_uri).await; let knows_head_before = linear_head(&knows_uri).await; @@ -673,6 +737,8 @@ async fn run_cell( .upgrade_system_columns(upgrade_execute()) .await .map(|_| ()), + Writer::BlobPut => put_photo(&db, &write_name).await, + Writer::BlobClear => clear_photo(&db).await, }; acknowledged = outcome.is_ok(); if let Err(error) = outcome { @@ -832,6 +898,21 @@ async fn run_cell( same_handle, model, "{cell}: the writer's handle disagrees with a fresh one" ); + if matches!(writer, Writer::BlobPut | Writer::BlobClear) { + // The cell holds the written value exactly when the write was + // acknowledged, on a fresh handle and on the writer's own. + let expected = match writer { + Writer::BlobPut if visible => Some(photo_bytes(&write_name).to_vec()), + Writer::BlobClear if visible => None, + _ => photo_before.clone(), + }; + assert_eq!(observe_photo(&fresh).await, expected, "{cell}: photo"); + assert_eq!( + observe_photo(&db).await, + expected, + "{cell}: the writer's handle reads a different photo" + ); + } let person_pin = table_pin(&fresh, "node:Person").await; let knows_pin = table_pin(&fresh, "edge:Knows").await; let person_head = linear_head(&person_uri).await; @@ -997,6 +1078,8 @@ async fn run_matrix() { Writer::Load, Writer::FtsRebuild, Writer::SystemColumnUpgrade, + Writer::BlobPut, + Writer::BlobClear, ]; let faults = [Fault::Return, Fault::Kill, Fault::Race]; let only: Option> = std::env::var("OMNIGRAPH_MATRIX_WRITERS").ok().map(|list| { diff --git a/crates/omnigraph/tests/end_to_end.rs b/crates/omnigraph/tests/end_to_end.rs index f53841753..2b254763a 100644 --- a/crates/omnigraph/tests/end_to_end.rs +++ b/crates/omnigraph/tests/end_to_end.rs @@ -4,13 +4,15 @@ use arrow_array::{Array, Int32Array, RecordBatch, StringArray}; use base64::Engine as _; use futures::TryStreamExt; +use bytes::Bytes; +use omnigraph::db::SnapshotId; use omnigraph::db::{Omnigraph, ReadTarget}; use omnigraph::error::{ManifestErrorKind, OmniError}; use omnigraph::instrumentation::{MergeWriteProbes, with_merge_write_probes}; use omnigraph::loader::LoadMode; use omnigraph::{ - BLOB_READ_RANGE_MAX_BYTES, BlobCell, BlobContent, EntityKind, ExternalBlobBase, - ExternalBlobExecutionScope, ExternalBlobPolicy, + BLOB_READ_RANGE_MAX_BYTES, BlobCell, BlobContent, BlobEtag, BlobPrecondition, BlobWriteOutcome, + EntityKind, ExternalBlobBase, ExternalBlobExecutionScope, ExternalBlobPolicy, }; use omnigraph_compiler::ir::ParamMap; @@ -2082,6 +2084,201 @@ async fn blob_update_null_round_trip() { ); } +/// Replacing and clearing one Blob cell by exact id, on a node and on an edge: +/// the write never inserts a row, its receipt is the publication's exact +/// commit, its ETag equals a read at that commit, a stale or `*` precondition +/// is judged at the write base, and clearing an already-null cell publishes +/// nothing. Rust, not GQT: a `.gq` statement cannot carry raw bytes or an +/// `If-Match`. +#[tokio::test] +async fn blob_put_and_clear_replace_one_cell_by_exact_id() { + let dir = tempfile::tempdir().unwrap(); + let db = helpers::session( + Omnigraph::init(dir.path().to_str().unwrap(), BLOB_SCHEMA) + .await + .unwrap(), + ); + let data = r#"{"type":"Document","data":{"title":"readme","content":"base64:SGVsbG8="}} +{"type":"Document","data":{"title":"peer"}} +{"edge":"Attachment","id":"attachment-1","from":"readme","to":"peer","data":{"payload":"base64:RWRnZQ=="}}"#; + db.load_jsonl(data, LoadMode::Overwrite).await.unwrap(); + + let managed_etag = |read: omnigraph::BlobRead| match read.content { + BlobContent::Managed { etag, .. } => etag, + BlobContent::External(external) => panic!("expected managed, got {external:?}"), + }; + for cell in [ + node_blob_cell("Document", "readme", "content"), + edge_blob_cell("Attachment", "attachment-1", "payload"), + ] { + let before = managed_etag( + db.read_blob_at(ReadTarget::branch("main"), cell.clone()) + .await + .unwrap(), + ); + let head = snapshot_main(&db).await.unwrap().graph_manifest_version(); + + let stale = db + .put_blob_at_as( + "main", + cell.clone(), + Bytes::from_static(b"never"), + Some(BlobPrecondition::Tags(vec![BlobEtag::from_tag( + "\"stale\"", + )])), + None, + ) + .await + .unwrap_err(); + assert!( + matches!(&stale, OmniError::BlobWritePreconditionFailed { current_etag: Some(current) } + if current == before.as_str()), + "{stale:?}" + ); + assert_eq!( + snapshot_main(&db).await.unwrap().graph_manifest_version(), + head, + "a failed precondition has no effect" + ); + + let BlobWriteOutcome::Managed { + length, + etag, + commit, + } = db + .put_blob_at_as( + "main", + cell.clone(), + Bytes::from_static(b"Replaced"), + Some(BlobPrecondition::Tags(vec![before.clone()])), + None, + ) + .await + .unwrap() + else { + panic!("a put stores managed bytes"); + }; + assert_eq!(length, 8); + assert_ne!(etag, before); + let at_commit = db + .read_blob_at( + ReadTarget::snapshot(SnapshotId::new(commit.graph_commit_id.clone())), + cell.clone(), + ) + .await + .unwrap(); + assert_eq!( + managed_etag(at_commit), + etag, + "the ETag equals a read at the commit" + ); + assert_eq!( + read_managed_blob_bytes(&db, ReadTarget::branch("main"), cell.clone()).await, + b"Replaced" + ); + + let BlobWriteOutcome::Managed { etag: again, .. } = db + .put_blob_at_as( + "main", + cell.clone(), + Bytes::from_static(b""), + Some(BlobPrecondition::AnyExisting), + None, + ) + .await + .unwrap() + else { + panic!("a put stores managed bytes"); + }; + assert_eq!( + read_managed_blob_bytes(&db, ReadTarget::branch("main"), cell.clone()).await, + b"", + "a valid empty value is managed, not null" + ); + + let BlobWriteOutcome::Null { commit: Some(_) } = db + .clear_blob_at_as( + "main", + cell.clone(), + Some(BlobPrecondition::Tags(vec![again])), + None, + ) + .await + .unwrap() + else { + panic!("clearing a value publishes a commit"); + }; + let cleared = db + .read_blob_at(ReadTarget::branch("main"), cell.clone()) + .await + .unwrap_err(); + assert!( + matches!(&cleared, OmniError::Manifest(error) if error.kind == ManifestErrorKind::NotFound), + "{cleared:?}" + ); + + let head = snapshot_main(&db).await.unwrap().graph_manifest_version(); + assert!(matches!( + db.clear_blob_at_as("main", cell.clone(), None, None) + .await + .unwrap(), + BlobWriteOutcome::Null { commit: None } + )); + for refused in [ + db.clear_blob_at_as( + "main", + cell.clone(), + Some(BlobPrecondition::AnyExisting), + None, + ) + .await + .unwrap_err(), + db.put_blob_at_as( + "main", + cell.clone(), + Bytes::from_static(b"x"), + Some(BlobPrecondition::AnyExisting), + None, + ) + .await + .unwrap_err(), + ] { + assert!( + matches!( + refused, + OmniError::BlobWritePreconditionFailed { current_etag: None } + ), + "a null cell satisfies neither precondition form" + ); + } + assert_eq!( + snapshot_main(&db).await.unwrap().graph_manifest_version(), + head, + "an already-null clear and failed preconditions publish nothing" + ); + } + + let missing = db + .put_blob_at_as( + "main", + node_blob_cell("Document", "absent", "content"), + Bytes::from_static(b"x"), + None, + None, + ) + .await + .unwrap_err(); + assert!( + matches!(&missing, OmniError::Manifest(error) if error.kind == ManifestErrorKind::NotFound), + "{missing:?}" + ); + assert_eq!( + count_rows(&db, "node:Document").await, + 2, + "a put never inserts a row" + ); +} + /// The control, the same update with a value, opens the table, so the zero /// open count belongs to the refusal and not to a skipped scan. #[tokio::test] diff --git a/crates/omnigraph/tests/failpoints.rs b/crates/omnigraph/tests/failpoints.rs index 55f8dae97..57d932fcc 100644 --- a/crates/omnigraph/tests/failpoints.rs +++ b/crates/omnigraph/tests/failpoints.rs @@ -8,15 +8,18 @@ use std::sync::Arc; use arrow_array::{Int32Array, RecordBatch, StringArray}; use arrow_schema::Schema; +use bytes::Bytes; use lance::Dataset; use omnigraph::db::{ - GraphCreateReconciliation, Omnigraph, PreparedGraphCreate, ReadTarget, StagingVerdict, + GraphCreateReconciliation, Omnigraph, PreparedGraphCreate, ReadTarget, SnapshotId, + StagingVerdict, }; use omnigraph::error::{CompletionEvidence, ManifestErrorKind, OmniError}; use omnigraph::instrumentation::{MergeWriteProbes, with_merge_write_probes}; use omnigraph::loader::LoadMode; use omnigraph::seams::FailScenario; use omnigraph::seams::catalog; +use omnigraph::{BlobCell, BlobContent, BlobEtag, BlobPrecondition, BlobWriteOutcome}; use serial_test::serial; use helpers::collector::{ @@ -46,6 +49,64 @@ const RFC023_EXTERNAL_MODE_ENV: &str = "OMNIGRAPH_RFC023_EXTERNAL_MODE"; const RFC023_EXTERNAL_PAYLOAD_ENV: &str = "OMNIGRAPH_RFC023_EXTERNAL_PAYLOAD"; const RFC023_EXTERNAL_ACTION_ENV: &str = "OMNIGRAPH_RFC023_EXTERNAL_ACTION"; +const BLOB_WRITE_SCHEMA: &str = r#" +node Document { + title: String @key + content: Blob? + preview: Blob? + note: String? +} +"#; + +/// `readme` carries content `Hello`, preview `P0` and note `n0`. +const BLOB_WRITE_SEED: &str = r#"{"type":"Document","data":{"title":"readme","content":"base64:SGVsbG8=","preview":"base64:UDA=","note":"n0"}}"#; + +const BLOB_WRITE_MUTATIONS: &str = r#" +query set_note($title: String, $note: String) { + update Document set { note: $note } where title = $title +} +"#; + +async fn seed_blob_write_graph(uri: &str) -> Session { + let db = helpers::session(Omnigraph::init(uri, BLOB_WRITE_SCHEMA).await.unwrap()); + db.load_jsonl(BLOB_WRITE_SEED, LoadMode::Overwrite) + .await + .unwrap(); + db +} + +fn readme(property: &str) -> BlobCell { + node_blob_cell("Document", "readme", property) +} + +/// The current validator of a managed cell on `branch`. +async fn blob_etag(db: &Omnigraph, branch: &str, cell: BlobCell) -> BlobEtag { + match db + .read_blob_at(ReadTarget::branch(branch), cell) + .await + .unwrap() + .content + { + BlobContent::Managed { etag, .. } => etag, + BlobContent::External(external) => panic!("expected a managed value, got {external:?}"), + } +} + +async fn set_readme_note(db: &Session, note: &str) { + db.mutate( + "main", + BLOB_WRITE_MUTATIONS, + "set_note", + ¶ms(&[("$title", "readme"), ("$note", note)]), + ) + .await + .unwrap(); +} + +async fn main_commit_count(db: &Omnigraph) -> usize { + db.list_commits(Some("main")).await.unwrap().len() +} + const OCC_UNIQUE_SCHEMA: &str = r#" node User { name: String @key @@ -1290,6 +1351,251 @@ async fn mutation_revalidates_unique_after_pre_effect_authority_change() { assert_eq!(collect_column_strings(&users, "name"), vec!["winner"]); } +/// A Blob write re-prepared after a competing write evaluates its +/// precondition against the fresh base, and an unconditional write carries +/// the row's other cells from that fresh base, never from the stale one. +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +#[serial] +async fn blob_put_reevaluates_its_precondition_and_recarries_the_row_after_a_competing_write() { + let _scenario = FailScenario::setup(); + let dir = tempfile::tempdir().unwrap(); + let db = Arc::new(seed_blob_write_graph(dir.path().to_str().unwrap()).await); + + // The competing write changes only the note, yet it moves the content + // cell's validator: the parked put's If-Match no longer holds. + let stale = blob_etag(&db, "main", readme("content")).await; + let rendezvous = + helpers::failpoint::Rendezvous::park_first(&catalog::MUTATION_POST_STAGE_PRE_EFFECT_GATE); + let writer = { + let db = Arc::clone(&db); + let stale = stale.clone(); + tokio::spawn(async move { + db.put_blob_at_as( + "main", + readme("content"), + Bytes::from_static(b"stale"), + Some(BlobPrecondition::Tags(vec![stale])), + None, + ) + .await + }) + }; + rendezvous.wait_until_reached().await; + set_readme_note(&db, "n1").await; + let competing_commits = main_commit_count(&db).await; + rendezvous.release(); + let error = writer + .await + .unwrap() + .expect_err("the stale If-Match fails against the fresh base"); + let fresh = blob_etag(&db, "main", readme("content")).await; + assert_ne!(fresh, stale, "the competing write moved the validator"); + assert!( + matches!(&error, OmniError::BlobWritePreconditionFailed { current_etag: Some(current) } + if current == fresh.as_str()), + "the refusal names the fresh validator: {error:?}" + ); + assert_eq!( + main_commit_count(&db).await, + competing_commits, + "the refused put published nothing" + ); + assert_eq!( + read_managed_blob_bytes(&db, ReadTarget::branch("main"), readme("content")).await, + b"Hello" + ); + drop(rendezvous); + + // An unconditional put parked while another put replaces the preview and + // an update replaces the note: its retry carries both. + let rendezvous = + helpers::failpoint::Rendezvous::park_first(&catalog::MUTATION_POST_STAGE_PRE_EFFECT_GATE); + let writer = { + let db = Arc::clone(&db); + tokio::spawn(async move { + db.put_blob_at_as( + "main", + readme("content"), + Bytes::from_static(b"fresh"), + None, + None, + ) + .await + }) + }; + rendezvous.wait_until_reached().await; + db.put_blob_at_as( + "main", + readme("preview"), + Bytes::from_static(b"P1"), + None, + None, + ) + .await + .unwrap(); + set_readme_note(&db, "n2").await; + rendezvous.release(); + let BlobWriteOutcome::Managed { etag, .. } = writer.await.unwrap().unwrap() else { + panic!("a put stores a managed value"); + }; + assert_eq!(etag, blob_etag(&db, "main", readme("content")).await); + assert_eq!( + read_managed_blob_bytes(&db, ReadTarget::branch("main"), readme("content")).await, + b"fresh" + ); + assert_eq!( + read_managed_blob_bytes(&db, ReadTarget::branch("main"), readme("preview")).await, + b"P1", + "the retry carried the competing preview, not the stale one" + ); + let documents = read_table(&db, "node:Document").await; + assert_eq!(collect_column_strings(&documents, "note"), ["n2"]); +} + +/// A Blob write that re-prepares across an accepted-schema change or a +/// delete-and-recreate of its branch fails closed, without effect. +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +#[serial] +async fn blob_put_fails_closed_when_schema_or_branch_incarnation_changes_between_attempts() { + let _scenario = FailScenario::setup(); + let dir = tempfile::tempdir().unwrap(); + let db = Arc::new(seed_blob_write_graph(dir.path().to_str().unwrap()).await); + let refused = |error: &OmniError| { + matches!(error, OmniError::Manifest(manifest) + if manifest.kind == ManifestErrorKind::Conflict + && manifest.message.contains("changed incarnation or accepted schema")) + }; + let parked_put = |branch: &'static str| { + let db = Arc::clone(&db); + tokio::spawn(async move { + db.put_blob_at_as( + branch, + readme("content"), + Bytes::from_static(b"lost"), + None, + None, + ) + .await + }) + }; + + let rendezvous = + helpers::failpoint::Rendezvous::park_first(&catalog::MUTATION_POST_STAGE_PRE_EFFECT_GATE); + let writer = parked_put("main"); + rendezvous.wait_until_reached().await; + db.apply_schema(&BLOB_WRITE_SCHEMA.replace("note: String?", "note: String?\n tag: String?")) + .await + .unwrap(); + let schema_commits = main_commit_count(&db).await; + rendezvous.release(); + let error = writer + .await + .unwrap() + .expect_err("a schema change refuses the retry"); + assert!(refused(&error), "{error:?}"); + assert_eq!(main_commit_count(&db).await, schema_commits); + assert_eq!( + read_managed_blob_bytes(&db, ReadTarget::branch("main"), readme("content")).await, + b"Hello" + ); + drop(rendezvous); + + db.branch_create("feature").await.unwrap(); + let rendezvous = + helpers::failpoint::Rendezvous::park_first(&catalog::MUTATION_POST_STAGE_PRE_EFFECT_GATE); + let writer = parked_put("feature"); + rendezvous.wait_until_reached().await; + db.branch_delete("feature").await.unwrap(); + db.branch_create("feature").await.unwrap(); + rendezvous.release(); + let error = writer + .await + .unwrap() + .expect_err("a recreated branch refuses the retry"); + assert!(refused(&error), "{error:?}"); + for branch in ["feature", "main"] { + assert_eq!( + read_managed_blob_bytes(&db, ReadTarget::branch(branch), readme("content")).await, + b"Hello" + ); + } +} + +/// The receipt of a Blob put is evidence from its own detached commit: a +/// write by another process that lands between the put's manifest commit and +/// its return does not change the ETag the put reports. +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +#[serial] +async fn a_write_after_publication_cannot_change_a_blob_put_receipt() { + let _scenario = FailScenario::setup(); + let dir = tempfile::tempdir().unwrap(); + let uri = dir.path().to_str().unwrap().to_string(); + let db = Arc::new(seed_blob_write_graph(&uri).await); + + let rendezvous = + helpers::failpoint::Rendezvous::park_first(&catalog::GRAPH_PUBLISH_AFTER_MANIFEST_COMMIT); + let writer = { + let db = Arc::clone(&db); + tokio::spawn(async move { + db.put_blob_at_as( + "main", + readme("content"), + Bytes::from_static(b"mine"), + None, + None, + ) + .await + }) + }; + rendezvous.wait_until_reached().await; + // "Zm9yZWlnbg==" is "foreign". + let foreign = r#"{"type":"Document","data":{"title":"readme","content":"base64:Zm9yZWlnbg==","preview":"base64:UDA=","note":"n0"}}"#; + let external_uri = uri.clone(); + tokio::task::spawn_blocking(move || { + run_rfc023_external_writer(external_uri, LoadMode::Merge, foreign.to_string()) + }) + .await + .unwrap() + .expect("the foreign write publishes after the parked put's manifest commit"); + rendezvous.release(); + let BlobWriteOutcome::Managed { + length, + etag, + commit, + } = writer.await.unwrap().unwrap() + else { + panic!("a put stores a managed value"); + }; + assert_eq!(length, 4); + + let fresh = helpers::session(Omnigraph::open(&uri).await.unwrap()); + let at_commit = ReadTarget::snapshot(SnapshotId::new(commit.graph_commit_id.clone())); + let BlobContent::Managed { + etag: committed, .. + } = fresh + .read_blob_at(at_commit.clone(), readme("content")) + .await + .unwrap() + .content + else { + panic!("the put's commit holds a managed value"); + }; + assert_eq!(etag, committed, "the receipt names the put's own commit"); + assert_eq!( + read_managed_blob_bytes(&fresh, at_commit, readme("content")).await, + b"mine" + ); + assert_ne!( + etag, + blob_etag(&fresh, "main", readme("content")).await, + "the head now holds the foreign write" + ); + assert_eq!( + read_managed_blob_bytes(&fresh, ReadTarget::branch("main"), readme("content")).await, + b"foreign" + ); +} + /// The coarse token is branch-wide: a commit to a table that the prepared /// mutation does not write can still invalidate schema/cardinality/RI inputs. /// A strict update therefore reports `ReadSetChanged` before moving its Person @@ -6420,6 +6726,154 @@ async fn assert_optimize_detached_pin_survives_next_write(uri: &str) { assert_person_pin_detached(&db, "main", head_before).await; } +/// A Blob put and a Blob clear whose publication acknowledgement is lost are +/// published exactly once: the caller's retry with its old validator is +/// refused, the cell reads the same from the writer's handle and a fresh +/// one, and the same handle writes again once the fault stops. +#[tokio::test] +#[serial] +async fn blob_put_and_clear_publish_once_through_a_lost_acknowledgement() { + let _scenario = FailScenario::setup(); + let dir = tempfile::tempdir().unwrap(); + let uri = dir.path().to_str().unwrap().to_string(); + let db = seed_blob_write_graph(&uri).await; + let original = blob_etag(&db, "main", readme("content")).await; + let before = main_commit_count(&db).await; + + { + let _failpoint = catalog::GRAPH_PUBLISH_AFTER_MANIFEST_COMMIT.fire_once_at(1); + let error = db + .put_blob_at_as( + "main", + readme("content"), + Bytes::from_static(b"Published"), + Some(BlobPrecondition::Tags(vec![original.clone()])), + None, + ) + .await + .expect_err("the acknowledgement is lost after publication"); + assert!( + error + .to_string() + .contains("graph_publish.after_manifest_commit"), + "the publication boundary must be reached: {error}" + ); + } + assert_eq!( + main_commit_count(&db).await, + before + 1, + "published exactly once" + ); + let published = blob_etag(&db, "main", readme("content")).await; + let fresh = helpers::session(Omnigraph::open(&uri).await.unwrap()); + assert_eq!( + blob_etag(&fresh, "main", readme("content")).await, + published + ); + for handle in [&db, &fresh] { + assert_eq!( + read_managed_blob_bytes(handle, ReadTarget::branch("main"), readme("content")).await, + b"Published" + ); + } + let retry = db + .put_blob_at_as( + "main", + readme("content"), + Bytes::from_static(b"Published"), + Some(BlobPrecondition::Tags(vec![original])), + None, + ) + .await + .expect_err("a retry with the old validator finds the published value"); + assert!( + matches!(&retry, OmniError::BlobWritePreconditionFailed { current_etag: Some(current) } + if current == published.as_str()), + "{retry:?}" + ); + assert_eq!( + main_commit_count(&db).await, + before + 1, + "the retry published nothing" + ); + + { + let _failpoint = catalog::GRAPH_PUBLISH_AFTER_MANIFEST_COMMIT.fire_once_at(1); + db.clear_blob_at_as( + "main", + readme("content"), + Some(BlobPrecondition::Tags(vec![published.clone()])), + None, + ) + .await + .expect_err("the clear's acknowledgement is lost after publication"); + } + assert_eq!( + main_commit_count(&db).await, + before + 2, + "the clear published exactly once" + ); + let fresh = helpers::session(Omnigraph::open(&uri).await.unwrap()); + for handle in [&db, &fresh] { + let error = handle + .read_blob_at(ReadTarget::branch("main"), readme("content")) + .await + .expect_err("a cleared cell has no value and no validator"); + assert!( + matches!(&error, OmniError::Manifest(manifest) if manifest.kind == ManifestErrorKind::NotFound), + "{error:?}" + ); + } + let retry = db + .put_blob_at_as( + "main", + readme("content"), + Bytes::from_static(b"again"), + Some(BlobPrecondition::Tags(vec![published])), + None, + ) + .await + .expect_err("the cleared cell rejects its old validator"); + assert!( + matches!( + &retry, + OmniError::BlobWritePreconditionFailed { current_etag: None } + ), + "{retry:?}" + ); + assert!(matches!( + db.clear_blob_at_as("main", readme("content"), None, None) + .await + .unwrap(), + BlobWriteOutcome::Null { commit: None } + )); + assert_eq!( + main_commit_count(&db).await, + before + 2, + "no duplicate clear" + ); + + let BlobWriteOutcome::Managed { etag, .. } = db + .put_blob_at_as( + "main", + readme("content"), + Bytes::from_static(b"Next"), + None, + None, + ) + .await + .expect("the same handle writes again once the fault stops") + else { + panic!("a put stores a managed value"); + }; + assert_eq!(etag, blob_etag(&fresh, "main", readme("content")).await); + assert_eq!( + read_managed_blob_bytes(&fresh, ReadTarget::branch("main"), readme("preview")).await, + b"P0", + "every write carried the preview" + ); +} + /// Exercise both shared backend contracts locally even without configured cloud fixtures. #[tokio::test] #[serial] diff --git a/crates/omnigraph/tests/forbidden_apis.rs b/crates/omnigraph/tests/forbidden_apis.rs index ffe188316..be8d5098e 100644 --- a/crates/omnigraph/tests/forbidden_apis.rs +++ b/crates/omnigraph/tests/forbidden_apis.rs @@ -246,6 +246,7 @@ write_surfaces! { "db/omnigraph/prepared_create.rs" => WriteProtocol::Exact("quiesced birth settlement: exact init claim, bounded original empty tables, no published manifest or refs") => ["settle_prepared_graph_create_after_quiescence"], "db/omnigraph.rs" => WriteProtocol::Composed("read-write admission + owned local-root create-if-absent capability probe") => ["open", "open_with_storage"], "exec/mutation.rs" => MUTATION_V9 => ["mutate", "mutate_with_receipt", "mutate_as", "mutate_as_with_receipt", "mutate_as_with_expected_head", "mutate_as_with_expected_head_receipt"], + "exec/blob_write.rs" => MUTATION_V9 => ["put_blob_at_as", "clear_blob_at_as"], "loader/mod.rs" => LOAD_V9 => ["load_jsonl", "load_jsonl_file", "load", "load_with_receipt", "load_file", "load_graph_batch"], "loader/mod.rs" => WriteProtocol::Composed("optional branch create, then Load v9") => ["load_as", "load_as_with_receipt", "load_file_as", "load_file_as_with_receipt", "load_graph_batch_as", "load_graph_batch_as_with_receipt"], "loader/mod.rs" => WriteProtocol::Composed("branch create when absent, then Load v9 alias") => ["ingest", "ingest_as", "ingest_file", "ingest_file_as"], @@ -959,6 +960,7 @@ durable_calls! { // Blob live-branch recheck: lists the table's refs to prove a vanished // fork before the incarnation refusal; read-only access to the handle. ("blob.rs", ".dataset()", 1, WriteProtocol::ReadOnlyAccess), + ("exec/blob_write.rs", ".dataset()", 4, WriteProtocol::ReadOnlyAccess), // Commit-change enumeration: pinned parent/child handles for the ordered // merge's typed row comparison. Read-only by construction — the enumerator // stages no transaction and publishes nothing. diff --git a/crates/omnigraph/tests/policy_engine_chassis.rs b/crates/omnigraph/tests/policy_engine_chassis.rs index 81f529c5f..20d271692 100644 --- a/crates/omnigraph/tests/policy_engine_chassis.rs +++ b/crates/omnigraph/tests/policy_engine_chassis.rs @@ -371,6 +371,49 @@ async fn mutate_as_allows_when_policy_permits_actor() { .expect("act-allowed should be able to Change on main"); } +/// A Blob put or clear enforces `change` on its branch for the supplied actor +/// before it opens anything, like every other Mutation-protocol writer. +#[tokio::test] +async fn blob_put_and_clear_enforce_change_for_the_actor() { + const SCHEMA: &str = "node Document { title: String @key content: Blob? }\n"; + let dir = tempfile::tempdir().unwrap(); + let uri = dir.path().to_str().unwrap(); + { + let db = helpers::session(Omnigraph::init(uri, SCHEMA).await.unwrap()); + db.load_jsonl( + r#"{"type":"Document","data":{"title":"doc"}}"#, + LoadMode::Overwrite, + ) + .await + .unwrap(); + } + let (db, _engine) = install_policy(Omnigraph::open(uri).await.unwrap(), dir.path()); + let cell = || node_blob_cell("Document", "doc", "content"); + let bytes = || bytes::Bytes::from_static(b"payload"); + + assert_denied( + db.put_blob_at_as("main", cell(), bytes(), None, Some("act-denied")) + .await, + "put_blob_at_as", + ); + assert_denied( + db.clear_blob_at_as("main", cell(), None, Some("act-denied")) + .await, + "clear_blob_at_as", + ); + let no_actor = db.put_blob_at_as("main", cell(), bytes(), None, None).await; + assert!( + matches!(no_actor, Err(OmniError::Policy(_))), + "an installed policy refuses a write without an actor: {no_actor:?}" + ); + db.put_blob_at_as("main", cell(), bytes(), None, Some("act-allowed")) + .await + .expect("act-allowed may Change on main"); + db.clear_blob_at_as("main", cell(), None, Some("act-allowed")) + .await + .expect("act-allowed may Change on main"); +} + #[tokio::test] async fn load_as_denies_when_policy_rejects_actor() { let dir = tempfile::tempdir().unwrap(); diff --git a/crates/omnigraph/tests/writes.rs b/crates/omnigraph/tests/writes.rs index 4189a6121..f789e825e 100644 --- a/crates/omnigraph/tests/writes.rs +++ b/crates/omnigraph/tests/writes.rs @@ -25,7 +25,7 @@ use omnigraph::db::{Omnigraph, ReadTarget}; use omnigraph::error::OmniError; use omnigraph::instrumentation::{StageWriteProbes, with_stage_write_probes}; use omnigraph::loader::LoadMode; -use omnigraph::{ExternalBlobBase, ExternalBlobExecutionScope, ExternalBlobPolicy}; +use omnigraph::{BlobCell, ExternalBlobBase, ExternalBlobExecutionScope, ExternalBlobPolicy}; use helpers::*; @@ -1585,6 +1585,320 @@ query update_shelf($note: String) { ); } +/// A Blob put admits a value of exactly 32 MiB and refuses one more byte +/// before opening a table. The row's other Blob cells are carried by value and +/// share the operation's payload allowance, so a put whose target and carried +/// sibling together exceed it is refused before any table effect. +/// Rust, not GQT: MiB-scale payloads and raw bytes are outside the case format. +#[tokio::test] +async fn blob_put_payload_bound_is_inclusive_and_counts_carried_siblings() { + use base64::Engine; + + const LIMIT: usize = 32 * 1024 * 1024; + const SCHEMA: &str = "node Document { title: String @key content: Blob? preview: Blob? }\n"; + let dir = tempfile::tempdir().unwrap(); + let db = helpers::session( + Omnigraph::init(dir.path().to_str().unwrap(), SCHEMA) + .await + .unwrap(), + ); + let preview = format!( + "base64:{}", + base64::engine::general_purpose::STANDARD.encode(vec![5_u8; 20 * 1024 * 1024]) + ); + let rows = format!( + "{}\n{}", + serde_json::json!({"type": "Document", "data": {"title": "lone"}}), + serde_json::json!({"type": "Document", "data": {"title": "paired", "preview": preview}}), + ); + db.load_jsonl(&rows, LoadMode::Overwrite).await.unwrap(); + let cell = |title: &str, property: &str| node_blob_cell("Document", title, property); + + db.put_blob_at_as( + "main", + cell("lone", "content"), + bytes::Bytes::from(vec![7_u8; LIMIT]), + None, + None, + ) + .await + .expect("a value of exactly the limit is admitted"); + let stored = + read_managed_blob_bytes(&db, ReadTarget::branch("main"), cell("lone", "content")).await; + assert_eq!(stored.len(), LIMIT); + + let head = snapshot_main(&db).await.unwrap().graph_manifest_version(); + let over = db + .put_blob_at_as( + "main", + cell("lone", "content"), + bytes::Bytes::from(vec![7_u8; LIMIT + 1]), + None, + None, + ) + .await + .unwrap_err(); + assert!( + matches!(&over, OmniError::ResourceLimitExceeded { resource, limit, actual } + if resource == "Blob write payload bytes" + && *limit == LIMIT as u64 + && *actual == LIMIT as u64 + 1), + "{over:?}" + ); + let carried = db + .put_blob_at_as( + "main", + cell("paired", "content"), + bytes::Bytes::from(vec![7_u8; 20 * 1024 * 1024]), + None, + None, + ) + .await + .unwrap_err(); + assert!( + matches!(&carried, OmniError::ResourceLimitExceeded { resource, limit, actual } + if resource == "keyed entity Blob payload bytes for node:Document" + && *limit == LIMIT as u64 + && *actual == 40 * 1024 * 1024), + "the carried sibling shares the payload allowance: {carried:?}" + ); + assert_eq!( + snapshot_main(&db).await.unwrap().graph_manifest_version(), + head, + "both refusals precede any graph effect" + ); +} + +/// A put never reads the old value of the cell it replaces, so it succeeds +/// when that value is an external reference whose object is gone. Under an +/// admitting policy it carries the row's other cells by value: a managed +/// sibling stays byte-identical, and an external sibling is read and stored as +/// managed bytes, because Lance's merge-insert cannot carry the reference. +#[tokio::test] +async fn blob_put_carries_siblings_by_value_and_never_reads_its_target() { + const SCHEMA: &str = "node Document { title: String @key content: Blob? preview: Blob? }\n"; + + let root = tempfile::tempdir().unwrap(); + let source_dir = root.path().join("sources"); + std::fs::create_dir(&source_dir).unwrap(); + let gone = source_dir.join("gone.bin"); + std::fs::write(&gone, b"about to vanish").unwrap(); + let sibling = source_dir.join("sibling.bin"); + std::fs::write(&sibling, b"external sibling").unwrap(); + let file_uri = |path: &std::path::Path| url::Url::from_file_path(path).unwrap().to_string(); + let allow = ExternalBlobPolicy::allow(vec![ + ExternalBlobBase::new( + url::Url::from_directory_path(&source_dir).unwrap(), + ExternalBlobExecutionScope::EmbeddedOnly, + ) + .unwrap(), + ]) + .unwrap(); + let graph_path = root.path().join("graph"); + let db = helpers::session( + Omnigraph::init(graph_path.to_str().unwrap(), SCHEMA) + .await + .unwrap() + .with_external_blob_policy(allow) + .unwrap(), + ); + let rows = format!( + "{}\n{}", + serde_json::json!({"type": "Document", "data": { + "title": "stale-target", "content": file_uri(&gone), "preview": "base64:a2VwdA=="}}), + serde_json::json!({"type": "Document", "data": { + "title": "external-sibling", "content": "base64:b2xk", "preview": file_uri(&sibling)}}), + ); + db.load_jsonl(&rows, LoadMode::Overwrite).await.unwrap(); + std::fs::remove_file(&gone).unwrap(); + + let probes = omnigraph::instrumentation::MergeWriteProbes::default(); + omnigraph::instrumentation::with_merge_write_probes( + probes.clone(), + db.put_blob_at_as( + "main", + node_blob_cell("Document", "stale-target", "content"), + bytes::Bytes::from_static(b"replacement"), + None, + None, + ), + ) + .await + .expect("a put never reads the value it replaces"); + assert_eq!(probes.external_blob_probe_calls(), 0); + assert_eq!(probes.external_blob_payload_read_calls(), 0); + for (property, expected) in [("content", &b"replacement"[..]), ("preview", &b"kept"[..])] { + assert_eq!( + read_managed_blob_bytes( + &db, + ReadTarget::branch("main"), + node_blob_cell("Document", "stale-target", property), + ) + .await, + expected, + "{property}" + ); + } + + db.put_blob_at_as( + "main", + node_blob_cell("Document", "external-sibling", "content"), + bytes::Bytes::from_static(b"new"), + None, + None, + ) + .await + .unwrap(); + assert_eq!( + read_managed_blob_bytes( + &db, + ReadTarget::branch("main"), + node_blob_cell("Document", "external-sibling", "preview"), + ) + .await, + b"external sibling", + "the carried external sibling is stored as managed bytes" + ); +} + +/// Under a denying policy a put or clear that must carry a stored external +/// reference fails with `StoredExternalBlobDenied` naming that sibling, on a +/// node and on an edge, without effect. A row holding two such cells is +/// recovered by a `merge` load of the whole row, which never reads the stored +/// cells, for a node and an edge; a node can also be recovered by a `.gq` +/// update assigning both cells. An edge has no `.gq` update. +#[tokio::test] +async fn blob_write_under_deny_names_the_carried_reference_and_whole_row_writes_recover() { + const SCHEMA: &str = r#" +node Document { + title: String @key + content: Blob? + preview: Blob? +} +edge Attachment: Document -> Document { + payload: Blob? + thumb: Blob? +} +"#; + const CLEAR_BOTH: &str = r#" +query clear_both($content: Blob?, $preview: Blob?) { + update Document set { content: $content, preview: $preview } where title = "by-update" +} +"#; + + let root = tempfile::tempdir().unwrap(); + let source_dir = root.path().join("sources"); + std::fs::create_dir(&source_dir).unwrap(); + let source = source_dir.join("stored.bin"); + std::fs::write(&source, b"stored external bytes").unwrap(); + let source_uri = url::Url::from_file_path(&source).unwrap().to_string(); + let allow = ExternalBlobPolicy::allow(vec![ + ExternalBlobBase::new( + url::Url::from_directory_path(&source_dir).unwrap(), + ExternalBlobExecutionScope::EmbeddedOnly, + ) + .unwrap(), + ]) + .unwrap(); + let graph_path = root.path().join("graph"); + let uri = graph_path.to_str().unwrap().to_string(); + { + let seeding = helpers::session( + Omnigraph::init(&uri, SCHEMA) + .await + .unwrap() + .with_external_blob_policy(allow) + .unwrap(), + ); + let mut rows = Vec::new(); + for title in ["by-load", "by-update"] { + rows.push(serde_json::json!({"type": "Document", "data": { + "title": title, "content": source_uri, "preview": source_uri}})); + } + rows.push(serde_json::json!({"edge": "Attachment", "id": "attached", + "from": "by-load", "to": "by-update", + "data": {"payload": source_uri, "thumb": source_uri}})); + let rows = rows + .iter() + .map(serde_json::Value::to_string) + .collect::>() + .join("\n"); + seeding + .load_jsonl(&rows, LoadMode::Overwrite) + .await + .unwrap(); + } + + let db = helpers::session(Omnigraph::open(&uri).await.unwrap()); + let head = snapshot_main(&db).await.unwrap().graph_manifest_version(); + let node = |property: &str| node_blob_cell("Document", "by-load", property); + let edge = |property: &str| BlobCell { + entity: omnigraph::EntityKind::Edge, + type_name: "Attachment".into(), + id: "attached".into(), + property: property.into(), + }; + for (cell, carried, type_key) in [ + (node("content"), "preview", "node:Document"), + (edge("payload"), "thumb", "edge:Attachment"), + ] { + let put = db + .put_blob_at_as( + "main", + cell.clone(), + bytes::Bytes::from_static(b"x"), + None, + None, + ) + .await + .unwrap_err(); + let clear = db + .clear_blob_at_as("main", cell.clone(), None, None) + .await + .unwrap_err(); + for error in [put, clear] { + assert!( + matches!(&error, OmniError::StoredExternalBlobDenied { type_key: key, property, .. } + if key == type_key && property == carried), + "{error:?}" + ); + } + } + assert_eq!( + snapshot_main(&db).await.unwrap().graph_manifest_version(), + head, + "a denied carry has no effect" + ); + + let whole_rows = format!( + "{}\n{}", + serde_json::json!({"type": "Document", "data": { + "title": "by-load", "content": "base64:bmV3", "preview": null}}), + serde_json::json!({"edge": "Attachment", "id": "attached", + "from": "by-load", "to": "by-update", + "data": {"payload": "base64:bmV3", "thumb": null}}), + ); + db.load_jsonl(&whole_rows, LoadMode::Merge) + .await + .expect("a merge load of the whole row never reads the stored cells"); + for cell in [node("content"), edge("payload")] { + assert_eq!( + read_managed_blob_bytes(&db, ReadTarget::branch("main"), cell).await, + b"new" + ); + } + let mut nulls = params(&[]); + for name in ["content", "preview"] { + nulls.insert( + name.to_string(), + omnigraph_compiler::query::ast::Literal::Null, + ); + } + db.mutate("main", CLEAR_BOTH, "clear_both", &nulls) + .await + .expect("an update assigning both cells reads neither"); +} + /// Carrying an unassigned stored external reference needs the graph's policy /// to admit its source; under Deny the refusal names the row and property, and /// assigning the property (a new value or null) replaces it without reading it. diff --git a/docs/dev/blob.md b/docs/dev/blob.md index 4231cc41b..80b62867a 100644 --- a/docs/dev/blob.md +++ b/docs/dev/blob.md @@ -2,7 +2,7 @@ OmniGraph treats a Blob as one property cell on an existing node or edge. Lance owns the Blob-v2 physical placement; OmniGraph owns logical identity, snapshot selection, authorization, external-source admission, bounded delivery, and graph-level publication. -This page describes the implemented read and ingestion behavior. Upload and clear commands proposed by [RFC 0033](../rfcs/0033-blob-management.md) are not implemented yet. +This page describes the implemented read, ingestion and embedded cell-write behavior. The HTTP and CLI put and clear surfaces proposed by [RFC 0033](../rfcs/0033-blob-management.md) are not implemented yet. ## Logical states @@ -68,6 +68,14 @@ A materializing rewrite (a carried update or merge cell, the bounded rewrite str A whole-object surface cannot carry a ranged external descriptor: `ExternalBlobRef::whole_object_uri` returns a `RangedExternalBlob` (whose display never includes the URI) for it, and the HTTP redirect, CLI delivery, schema rewrite and export all refuse it rather than widen it. Surfaces that are not reloaded describe it instead: `export_blob_values` takes `RangedExternalBlobs::{Refuse, Describe}` and returns `LogicalBlobValue::RangedExternal` under `Describe`, which `describe_ranged_blob_cells` writes as `{"uri", "offset", "length"}` with a positive length (the decoder below guarantees one; a missing length there is an internal error) into the row images of `logical_row_image` (change-feed images, entity reads) and the snapshot rows of the change-feed baseline, and change-row comparison compares exactly, so a ranged row never wedges a feed cursor or refuses a baseline. The baseline shares export's table walk and line format and passes `Describe` where export passes `Refuse`: it is the exact state its consumer starts from, and a refusal there would leave the graph with no baseline. The descriptor decoder refuses an external descriptor with an offset but size 0 as a Blob integrity error: Lance reads size 0 as the object's size but keeps the position, which would read past the object's end. +## Engine cell writes + +`Session::put_blob_at_as` and `Session::clear_blob_at_as` (`exec/blob_write.rs`) replace one cell of an existing row. They are Mutation-protocol writes, not a writer kind of their own: an attempt captures a `WriteTxn`, stages one `PendingMode::Upsert` row through `MutationStaging`, and runs the Mutation tail, `commit_staged_mutation` (validation, staging, `commit_all`'s detached commit) then `publish_committed_mutation` (the protocol's one publisher call). `forbidden_apis.rs` registers the file under `MUTATION_V9`. + +Lance has no single-cell Blob write (`UpdateBuilder` and `RewriteColumns` refuse Blob-v2 columns), so the row is rebuilt: the target column is the new value, and every other column comes from one `scan_with_pending_materialized_blobs` of the exact id with the target omitted, so the target's old payload is never read or charged and the other Blob cells are carried under the update rule above. The new value is a logical `{data, uri}` struct whose `LargeBinary` child adopts the caller's `Bytes` buffer without a copy. A put over `KEYED_BLOB_PAYLOAD_MAX_BYTES` is refused before the write captures anything; the carried cells and the new value then share the keyed payload allowance. + +The precondition is evaluated against the attempt's base: the ETag is computed from the base table version the attempt opened, exactly as `read_blob_at` computes it. A pre-effect `ReadSetChanged` re-prepares the whole attempt within the insert-only bound of 32 (re-reading the cell, re-evaluating the precondition, re-carrying the row), because the stored value is the caller's rather than a read-modify-write plan. A re-prepared attempt that sees a different native branch identifier or accepted schema than the first one fails with a conflict. The receipt's ETag is read from the detached version `commit_all` returned, before the manifest CAS, so nothing after publication reads storage and a later write cannot change it. + ## HTTP and CLI delivery `GET` and `HEAD /graphs/{graph_id}/blob` select one logical cell. Managed delivery supports one bounded range, strong conditional requests, and constant-memory backpressure. `HEAD` does not read payload bytes. @@ -76,7 +84,7 @@ An external value produces a `302` redirect with the stored URI; the server neve The CLI exposes `blob get` and `blob stat` for embedded and remote graphs. `get` streams managed bytes to stdout or `--out`; `stat` returns kind, resolved-view metadata, size/ETag for managed data, or the descriptor for external data. The CLI refuses to follow external references. -There is currently no HTTP or CLI Blob put/clear surface. +There is currently no HTTP or CLI Blob put/clear surface; the embedded writes above are the only cell writes. ## Maintenance and export @@ -89,6 +97,7 @@ Export emits managed values as base64 and whole-object external values as URI de ## Test owners - Engine logical reads and ingestion: `crates/omnigraph/tests/end_to_end.rs`, `branching.rs`, and in-source Blob/table-store tests. +- Engine cell writes: `end_to_end.rs::blob_put_and_clear_replace_one_cell_by_exact_id` (node and edge, preconditions, the receipt), `writes.rs` (`blob_put_*` and `blob_write_*`: the inclusive bound, carried siblings, a denied carried reference), `policy_engine_chassis.rs`, the `BlobPut` and `BlobClear` writers of `detached_commit_matrix.rs`, and `failpoints.rs` (a competing write re-evaluates the precondition and re-carries the row, a schema or branch change refuses the retry, a lost acknowledgement publishes once, a write after publication leaves the receipt unchanged). - Base and storage-root disjointness: in-source `blob.rs` tests (root forms and S3/local overlap), `end_to_end.rs::external_blob_policy_refuses_base_overlapping_graph_root`, and the `omnigraph-cluster` tests `external_blob_config_rejects_bases_overlapping_storage_root`, `external_blob_base_overlapping_storage_root_refuses_apply_over_existing_state`, `external_blob_config_reports_uncomparable_storage_root_under_its_own_code`, `serving_quarantines_applied_policies_overlapping_storage_root` (the classifier), `serving_snapshot_quarantines_graph_whose_applied_base_overlaps_storage_root` (the snapshot reader: quarantine with a healthy sibling, refusal when none is left, ledger unchanged), and `serving_snapshot_refuses_overlapping_policy_its_digest_does_not_bind`. - Lance compatibility: `crates/omnigraph/tests/lance_surface_guards.rs`. - Cluster policy persistence and serving projection: `omnigraph-cluster` in-source tests. diff --git a/docs/dev/testing.md b/docs/dev/testing.md index 4867fc8a0..0c8ff18eb 100644 --- a/docs/dev/testing.md +++ b/docs/dev/testing.md @@ -55,7 +55,7 @@ The engine integration suite is grouped by behavior, not implementation module: | Search and physical indexes | `search.rs`, `scalar_indexes.rs`, `lance_surface_guards.rs`, `rrf_prefilter_gate.rs` (the rrf plan gate's differential oracle and fences), `repro_issue_563.rs` (`#[ignore]`d overflow-scale symptom tier) | | Writes, validation, schema, and policy | `writes.rs`, `validators.rs`, `schema_apply.rs`, `policy_engine_chassis.rs` | | Branches, snapshots, diffs, and merges | `branching.rs`, `point_in_time.rs`, `changes.rs`, `merge_truth_table.rs`, `merge_fast_forward.rs` | -| Recovery and crash windows | `recovery.rs`, `failpoints.rs` (including the `live_handle_*` liveness owners: a live handle writes again once faults stop, without reopening), `detached_commit_matrix.rs` (the RFC 0067 writer × window × fault × recovery-actor matrix over the insert, multi-table, load, cleanup, ensure-indices, full-text-rebuild, merge, schema-apply, optimize and system-column-upgrade writers; the same-handle liveness actor runs by default and `OMNIGRAPH_MATRIX=full` adds the other-process and cleanup actors), in-source manifest/recovery tests | +| Recovery and crash windows | `recovery.rs`, `failpoints.rs` (including the `live_handle_*` liveness owners: a live handle writes again once faults stop, without reopening), `detached_commit_matrix.rs` (the RFC 0067 writer × window × fault × recovery-actor matrix over the insert, multi-table, load, cleanup, ensure-indices, full-text-rebuild, merge, schema-apply, optimize, system-column-upgrade and Blob put and clear writers; the same-handle liveness actor runs by default and `OMNIGRAPH_MATRIX=full` adds the other-process and cleanup actors), in-source manifest/recovery tests | | Maintenance and substrate fences | `maintenance.rs`, `lance_surface_guards.rs`, `lance_version_columns.rs`, `forbidden_apis.rs` | | Export and lineage | `export.rs`, `lineage_projection.rs` | | Legacy-vintage graphs (`id`/`src`/`dst` spellings, born at the current stamp) | `legacy_columns.rs` — load, query, export round trip, evolution; needs `--features failpoints` | diff --git a/docs/user/blobs.md b/docs/user/blobs.md index 1f5d0e9fe..bf5c19b91 100644 --- a/docs/user/blobs.md +++ b/docs/user/blobs.md @@ -60,6 +60,44 @@ but no length is refused as a Blob integrity error. OmniGraph never deletes the object named by an external reference. +### Replacing one Blob value + +An embedded `Session` replaces or clears one Blob cell of an existing node or +edge, addressed by exact id like a Blob read: + +- `Session::put_blob_at_as` stores the given bytes as a managed value and + returns its length, its ETag and the commit; +- `Session::clear_blob_at_as` sets a nullable cell to null. Clearing a cell that + is already null publishes nothing and returns no commit. + +Each call that changes the cell is one graph commit and needs the `change` +action on the branch. Neither call inserts a row: a missing entity is +`NotFound`. Clearing a property that is not nullable is refused. The write never +reads the cell's old value. Lance replaces whole rows, so the row's other cells +are carried as an `update` carries them: a stored external reference among +them needs the external Blob policy to admit its source, and their managed +payloads count toward the operation's 32 MiB Blob payload limit together with +the new value. + +A precondition makes the write conditional on the cell's current value. +`BlobPrecondition::Tags` holds when the current ETag is one of the given tags. +`BlobPrecondition::AnyExisting` holds when the cell is not null. When it fails, +the write returns `BlobWritePreconditionFailed`, which carries the cell's +current ETag when it holds a managed value, and changes nothing. A commit that +lands while the write is in flight makes it start again from the new head and +check again, so a stale ETag fails instead of overwriting the newer value; a +write without a precondition replaces whatever is current. A schema apply or a +delete and recreate of the branch landing in that window refuses the write +with a conflict instead. + +The returned ETag is the one a read at the returned commit reports. When a call +fails after its commit became durable, for example because its acknowledgement +was lost, the value is published once; read the cell for its current ETag +before retrying with a precondition. + +The CLI and the HTTP server do not offer these writes yet. Through them, change +a Blob value with a load or a mutation. + ## Query behavior Blob properties are not ordinary `.gq` read values. They cannot be projected, @@ -67,8 +105,10 @@ filtered, ordered, or aggregated. Write them through load or mutation assignment, then read an individual Blob value through the dedicated CLI or HTTP surface. -There are no `blob put` or `blob clear` commands. Use the normal graph write -path so Blob changes remain part of an atomic graph commit. +There are no `blob put` or `blob clear` CLI commands yet. Use the normal graph +write path, or the embedded writes in +[Replacing one Blob value](#replacing-one-blob-value), so Blob changes remain +part of an atomic graph commit. ## CLI reads @@ -154,6 +194,7 @@ and retry. | 32 MiB of decoded `base64:` bytes | One `base64:` value, in a load or in an insert or update mutation | `decoded blob input bytes` | | 32 MiB per touched type, and 32 MiB across all touched types, Blob payloads excluded | Incremental writes: `append` and `merge` loads, inserts, updates and branch merges. Every byte of the rows except managed Blob payloads counts, URIs and Blob framing included | `keyed write bytes for
`, `keyed entity bytes for
`, `retained keyed batch bytes per operation` | | 32 MiB of managed Blob payload per touched type, and 32 MiB across all touched types, inclusive | The same writes. Each Blob value counts its length; external bytes copied in and Blob values carried unchanged by an update count. A single value of exactly 32 MiB fits beside its row | the same names with `Blob payload bytes` in place of `bytes`, for example `keyed entity Blob payload bytes for
` | +| 32 MiB, inclusive | The bytes of one embedded Blob put, checked before the write opens a table | `Blob write payload bytes` | | 32 MiB of external payload copied into managed storage | One incremental write operation across all its types, and each type within it: two types copying 20 MiB each exceed it although each fits its per-type limit | `materialized external blob payload bytes` | | 32 MiB of Blob payload | One branch merge that writes rows, across all types, managed and external bytes together | `materialized blob payload bytes` | | 8,192 external references | One write operation or merge | `external Blob reference cells` | From 13572c18d30fb6d3b8de36d7ab1923c6acb4e881 Mon Sep 17 00:00:00 2001 From: Ragnor Comerford Date: Thu, 8 Oct 2026 23:06:17 -0700 Subject: [PATCH 03/14] test(matrix): read the photo baseline only for the Blob writers --- crates/omnigraph/tests/detached_commit_matrix.rs | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/crates/omnigraph/tests/detached_commit_matrix.rs b/crates/omnigraph/tests/detached_commit_matrix.rs index 03becaf6d..d20d8152d 100644 --- a/crates/omnigraph/tests/detached_commit_matrix.rs +++ b/crates/omnigraph/tests/detached_commit_matrix.rs @@ -690,7 +690,12 @@ async fn run_cell( if writer == Writer::BlobClear { put_photo(&db, &format!("m{index}_seed")).await.unwrap(); } - let photo_before = observe_photo(&db).await; + let blob_writer = matches!(writer, Writer::BlobPut | Writer::BlobClear); + let photo_before = if blob_writer { + observe_photo(&db).await + } else { + None + }; let (mut model, _) = observe_model(&db).await; let head_before = linear_head(&person_uri).await; let knows_head_before = linear_head(&knows_uri).await; @@ -898,7 +903,7 @@ async fn run_cell( same_handle, model, "{cell}: the writer's handle disagrees with a fresh one" ); - if matches!(writer, Writer::BlobPut | Writer::BlobClear) { + if blob_writer { // The cell holds the written value exactly when the write was // acknowledged, on a fresh handle and on the writer's own. let expected = match writer { From 04b9e6675bad6fa8d04f6f137e3f1e3efb6f2c47 Mon Sep 17 00:00:00 2001 From: Ragnor Comerford Date: Fri, 9 Oct 2026 08:33:48 -0700 Subject: [PATCH 04/14] feat(server): PUT and DELETE one Blob cell over HTTP PUT and DELETE /graphs/{graph_id}/blob replace or clear one Blob value of an existing node or edge through the engine cell writes, each as one graph commit under `change`. PUT is a raw ingress route like /load/ndjson: it authorizes, checks the media type (415) and a declared length (413) before polling the body, then collects at most 32 MiB under the body deadline (408) into one buffer the engine adopts, and shrinks its ingress reservation to the received size. Both verbs take a branch, refuse snapshot and unknown parameters, run as owned writes, and answer the exact receipt. If-Match is parsed by a shared parser in the API types; a failed one is a 412 with the new blob_precondition_failure detail and the current ETag header. --- changelog.d/blob-cell-put-clear.added.md | 4 +- changelog.d/blob-http-put-delete.added.md | 12 + crates/omnigraph-api-types/src/lib.rs | 262 ++++++ crates/omnigraph-server/src/handlers.rs | 297 +++++++ crates/omnigraph-server/src/ingress.rs | 12 +- crates/omnigraph-server/src/lib.rs | 94 ++- crates/omnigraph-server/tests/auth_policy.rs | 53 +- crates/omnigraph-server/tests/data_routes.rs | 471 ++++++++++- crates/omnigraph-server/tests/openapi.rs | 91 ++ crates/omnigraph/src/blob.rs | 8 + crates/omnigraph/src/exec/blob_write.rs | 13 +- crates/omnigraph/src/lib.rs | 8 +- docs/dev/blob.md | 6 +- docs/user/blobs.md | 80 +- docs/user/operations/server.md | 9 +- docs/user/operations/troubleshooting.md | 2 +- openapi.json | 845 +++++++++++++++++++ 17 files changed, 2196 insertions(+), 71 deletions(-) create mode 100644 changelog.d/blob-http-put-delete.added.md diff --git a/changelog.d/blob-cell-put-clear.added.md b/changelog.d/blob-cell-put-clear.added.md index c47eecd10..9e065980f 100644 --- a/changelog.d/blob-cell-put-clear.added.md +++ b/changelog.d/blob-cell-put-clear.added.md @@ -8,5 +8,5 @@ nothing. A put returns the value's length, its ETag and the commit; the ETag is the one a read at that commit reports. One put holds at most 32 MiB, inclusive; a larger one fails before any table is opened with resource - `Blob write payload bytes`. The CLI and HTTP server do not expose these - writes yet. + `Blob write payload bytes`. The HTTP server exposes them as `PUT` and + `DELETE /blob`; the CLI does not expose them yet. diff --git a/changelog.d/blob-http-put-delete.added.md b/changelog.d/blob-http-put-delete.added.md new file mode 100644 index 000000000..ff8a3a55a --- /dev/null +++ b/changelog.d/blob-http-put-delete.added.md @@ -0,0 +1,12 @@ +- `PUT` and `DELETE /graphs/{graph_id}/blob` replace or clear one Blob value + of an existing node or edge, each as one graph commit under the `change` + action. `PUT` takes a raw `application/octet-stream` body of at most 32 MiB, + inclusive (`415` for another media type, `413` over the limit, refused + before reading when `Content-Length` declares it, `408` past the body + deadline). Both take `branch`, default `main`, refuse `snapshot`, and accept + `If-Match`: `*` or a list of entity tags. A failed `If-Match` is a `412` + with the new `blob_precondition_failure.current_etag` detail and an `ETag` + header naming the cell's current value. Success returns a receipt with the + selector, branch, `kind` (`managed` or `null`), `size` and `etag`, the actor + and the exact `commit`, which is `null` only when clearing a cell that was + already null. A `PUT` also returns the `ETag` header. diff --git a/crates/omnigraph-api-types/src/lib.rs b/crates/omnigraph-api-types/src/lib.rs index 2ba5d0151..c4c322894 100644 --- a/crates/omnigraph-api-types/src/lib.rs +++ b/crates/omnigraph-api-types/src/lib.rs @@ -1027,6 +1027,185 @@ impl BlobStatOutput { } } +/// Query parameters of `PUT` and `DELETE /graphs/{graph_id}/blob`. +/// +/// A write names a branch and never a snapshot; an unknown parameter, +/// `snapshot` included, is refused. +#[derive(Debug, Clone, Serialize, Deserialize, IntoParams)] +#[into_params(parameter_in = Query)] +#[serde(deny_unknown_fields)] +pub struct BlobWriteQuery { + /// Select a logical node or edge cell. + pub entity: BlobEntityKind, + /// Accepted-schema node or edge type name. + pub r#type: String, + /// Logical id of an existing entity within the selected type. + pub id: String, + /// Accepted-schema Blob property name. + pub property: String, + /// Branch to write. Defaults to `main`. + pub branch: Option, +} + +impl From<&BlobWriteQuery> for BlobSelectorOutput { + fn from(query: &BlobWriteQuery) -> Self { + Self { + entity: query.entity, + r#type: query.r#type.clone(), + id: query.id.clone(), + property: query.property.clone(), + } + } +} + +/// The state a Blob write left its cell in. Separate from +/// [`BlobContentKindOutput`]: a write can leave the cell null, a stat never +/// describes one. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, ToSchema)] +#[serde(rename_all = "snake_case")] +pub enum BlobWriteStateOutput { + Managed, + Null, +} + +/// Receipt of a Blob put or clear. +/// +/// A put leaves managed bytes, with their `size` and `etag`; a clear leaves +/// null, with neither. The `etag` is the one a read at `commit` reports. +#[derive(Debug, Clone, Serialize, Deserialize, ToSchema)] +pub struct BlobWriteOutput { + pub selector: BlobSelectorOutput, + /// The branch the write published to. + pub branch: String, + pub kind: BlobWriteStateOutput, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub size: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub etag: Option, + pub actor_id: Option, + /// The write's own publication. Always present on the wire; `null` only + /// for a clear of a cell that was already null, which publishes nothing. + #[serde(deserialize_with = "Option::deserialize")] + #[schema(required = true)] + pub commit: Option, +} + +/// Map an engine Blob write outcome to its receipt. The server and the CLI's +/// embedded writes share it, so both report the same receipt. +pub fn blob_write_output( + selector: BlobSelectorOutput, + branch: impl Into, + outcome: &omnigraph::BlobWriteOutcome, + actor_id: Option, +) -> BlobWriteOutput { + let branch = branch.into(); + match outcome { + omnigraph::BlobWriteOutcome::Managed { + length, + etag, + commit, + } => BlobWriteOutput { + selector, + branch, + kind: BlobWriteStateOutput::Managed, + size: Some(*length), + etag: Some(etag.as_str().to_string()), + actor_id, + commit: Some(commit_output(commit)), + }, + omnigraph::BlobWriteOutcome::Null { commit } => BlobWriteOutput { + selector, + branch, + kind: BlobWriteStateOutput::Null, + size: None, + etag: None, + actor_id, + commit: commit.as_ref().map(commit_output), + }, + } +} + +/// Parse the `If-Match` field lines of a Blob write into its precondition. +/// +/// Returns `Ok(None)` when no line is present. A field of exactly `*` is +/// [`omnigraph::BlobPrecondition::AnyExisting`]; otherwise the comma-separated +/// entity tags of every line form [`omnigraph::BlobPrecondition::Tags`]. Weak +/// tags never match a strong comparison, so they are dropped: a list of only +/// weak tags matches nothing. A malformed field, an empty one, or `*` beside +/// another element is refused, so a write never runs under a precondition it +/// misread. The server and the CLI share this parser. +pub fn parse_blob_if_match<'a>( + lines: impl IntoIterator, +) -> Result, String> { + fn skip_ows(mut rest: &[u8]) -> &[u8] { + while let [b' ' | b'\t', tail @ ..] = rest { + rest = tail; + } + rest + } + let malformed = || "If-Match must be `*` or a list of quoted entity tags".to_string(); + + let mut present = false; + let mut any = false; + let mut elements = 0_usize; + let mut tags = Vec::new(); + for line in lines { + present = true; + let mut rest = skip_ows(line); + while !rest.is_empty() { + if let [b',', tail @ ..] = rest { + rest = skip_ows(tail); + continue; + } + if let [b'*', tail @ ..] = rest { + any = true; + rest = tail; + } else { + let (weak, quoted) = match rest.strip_prefix(b"W/") { + Some(quoted) => (true, quoted), + None => (false, rest), + }; + let [b'"', body @ ..] = quoted else { + return Err(malformed()); + }; + let close = body + .iter() + .position(|byte| *byte == b'"') + .ok_or_else(malformed)?; + let opaque = &body[..close]; + if !opaque + .iter() + .all(|byte| matches!(byte, 0x21 | 0x23..=0x7e | 0x80..=0xff)) + { + return Err(malformed()); + } + // A tag that is not UTF-8 can never equal an engine-issued + // ETag, so leaving it out changes no outcome. + if !weak && let Ok(opaque) = std::str::from_utf8(opaque) { + tags.push(omnigraph::BlobEtag::from_tag(format!("\"{opaque}\""))); + } + rest = &body[close + 1..]; + } + elements += 1; + rest = skip_ows(rest); + match rest { + [] => {} + [b',', tail @ ..] => rest = skip_ows(tail), + _ => return Err(malformed()), + } + } + } + if !present { + return Ok(None); + } + match (any, elements) { + (_, 0) => Err(malformed()), + (true, 1) => Ok(Some(omnigraph::BlobPrecondition::AnyExisting)), + (true, _) => Err(malformed()), + (false, _) => Ok(Some(omnigraph::BlobPrecondition::Tags(tags))), + } +} + #[derive(Debug, Clone, Serialize, Deserialize, ToSchema)] #[serde(deny_unknown_fields)] pub struct ChangeRequest { @@ -1435,6 +1614,17 @@ pub struct RecoveryRequiredOutput { pub operation_id: String, } +/// A Blob write's `If-Match` did not hold for the cell (HTTP 412). The write +/// had no effect. `current_etag` is the cell's validator when it holds a +/// managed value; it is absent for a null or external value, which no tag +/// matches. Distinct from [`PreconditionFailureOutput`], the graph-commit +/// precondition. +#[derive(Debug, Clone, Serialize, Deserialize, ToSchema)] +pub struct BlobPreconditionFailureOutput { + #[serde(default, skip_serializing_if = "Option::is_none")] + pub current_etag: Option, +} + /// Structured details for a caller write-precondition failure: HTTP 412, a /// mutation carried `Omnigraph-If-Graph-Commit: `, and the branch /// head no longer matches that id. The write had no effect; the caller re-reads @@ -1633,6 +1823,10 @@ pub struct ErrorOutput { /// the machine-readable meaning and `code` is omitted. #[serde(skip_serializing_if = "Option::is_none")] pub precondition_failure: Option, + /// Set with HTTP 412 when a Blob write's `If-Match` did not hold for the + /// cell. The `ETag` response header repeats `current_etag` when present. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub blob_precondition_failure: Option, /// Set with HTTP 410 when retained history can no longer reconstruct a /// change continuation. Recover via the baseline handshake. #[serde(default, skip_serializing_if = "Option::is_none")] @@ -1669,6 +1863,7 @@ impl ErrorOutput { external_blob_source: None, recovery_required: None, precondition_failure: None, + blob_precondition_failure: None, change_feed_gap: None, change_diff_refusal: None, full_text_index_rebuild_required: None, @@ -2163,6 +2358,73 @@ mod tests { use omnigraph_compiler::settings::SettingScope; use serde_json::json; + #[test] + fn blob_if_match_parses_any_tag_lists_and_weak_tags_and_refuses_malformed_fields() { + use omnigraph::{BlobEtag, BlobPrecondition}; + let parse = |lines: &[&str]| parse_blob_if_match(lines.iter().map(|line| line.as_bytes())); + let tags = |list: &[&str]| { + Some(BlobPrecondition::Tags( + list.iter().map(|tag| BlobEtag::from_tag(*tag)).collect(), + )) + }; + + assert_eq!(parse(&[]), Ok(None)); + assert_eq!(parse(&["*"]), Ok(Some(BlobPrecondition::AnyExisting))); + assert_eq!(parse(&[" * "]), Ok(Some(BlobPrecondition::AnyExisting))); + assert_eq!(parse(&["\"a\""]), Ok(tags(&["\"a\""]))); + // Lists across and within lines, empty elements and OWS; a comma + // inside a tag is part of the tag. + assert_eq!( + parse(&["\"a\" ,, \"b,c\"", "\t\"d\""]), + Ok(tags(&["\"a\"", "\"b,c\"", "\"d\""])) + ); + // Weak tags never match strongly: dropped, so only-weak matches nothing. + assert_eq!(parse(&["W/\"a\", \"b\""]), Ok(tags(&["\"b\""]))); + assert_eq!(parse(&["W/\"a\""]), Ok(tags(&[]))); + assert_eq!(parse(&["\"\""]), Ok(tags(&["\"\""]))); + + for malformed in [ + &[""][..], + &[" , "], + &["a"], + &["\"a"], + &["\"a\"b"], + &["\"a\" \"b\""], + &["w/\"a\""], + &["*, \"a\""], + &["*", "\"a\""], + &["*", "*"], + &["\"a b\""], + ] { + assert!(parse(malformed).is_err(), "{malformed:?} must be refused"); + } + } + + #[test] + fn blob_write_output_reports_a_managed_put_and_a_no_op_clear() { + let selector = BlobSelectorOutput { + entity: BlobEntityKind::Node, + r#type: "Document".to_string(), + id: "readme".to_string(), + property: "content".to_string(), + }; + let cleared = blob_write_output( + selector.clone(), + "main", + &omnigraph::BlobWriteOutcome::Null { commit: None }, + Some("act-alice".to_string()), + ); + let wire = serde_json::to_value(&cleared).unwrap(); + assert_eq!(wire["kind"], "null"); + assert!( + wire["commit"].is_null(), + "a no-op clear reports `commit: null`" + ); + assert!(wire.get("size").is_none() && wire.get("etag").is_none()); + let read_back: BlobWriteOutput = serde_json::from_value(wire.clone()).unwrap(); + assert_eq!(serde_json::to_value(&read_back).unwrap(), wire); + } + #[test] fn diagnostic_suggestions_round_trip_and_old_payloads_remain_valid() { let old = json!({"code": "Q002", "expected": "missing parameters", "fix": "query q()"}); diff --git a/crates/omnigraph-server/src/handlers.rs b/crates/omnigraph-server/src/handlers.rs index 202721acd..943fa9a82 100644 --- a/crates/omnigraph-server/src/handlers.rs +++ b/crates/omnigraph-server/src/handlers.rs @@ -974,6 +974,303 @@ pub(crate) async fn server_blob_head( blob_transport::serve_blob_head(read, &headers).inspect_err(log_blob_transport_internal) } +#[utoipa::path( + put, + path = "/blob", + tag = "blobs", + operation_id = "putBlob", + params( + BlobWriteQuery, + ("If-Match" = Option, Header, description = "`*` to require a value in the cell, or a list of strong entity tags one of which must equal the cell's current ETag. Weak tags never match. The write evaluates it against the branch head it applies to."), + ), + request_body( + content = inline(BlobBinaryBody), + content_type = "application/octet-stream", + description = "The raw bytes to store: at most 33554432 bytes (32 MiB), inclusive." + ), + responses( + (status = 200, description = "The bytes are published in one graph commit", body = BlobWriteOutput, + headers( + ("ETag" = String, description = "Strong validator of the stored value; equals `etag` in the body"), + )), + (status = 400, description = "Invalid selector, query parameter, If-Match field or target, a non-Blob property, or a carried external reference the graph's policy denies", body = ErrorOutput), + (status = 401, description = "Unauthorized", body = ErrorOutput), + (status = 403, description = "Forbidden", body = ErrorOutput), + (status = 404, description = "Unknown branch, or no entity with this id", body = ErrorOutput), + (status = 408, description = "The request body did not arrive before the body deadline", body = ErrorOutput), + (status = 409, description = "Write-authority conflict, or the branch's incarnation or accepted schema changed while the write retried", body = ErrorOutput), + (status = 412, description = "If-Match did not hold for the cell; `blob_precondition_failure` names its current validator", body = ErrorOutput, + headers( + ("ETag" = String, description = "The cell's current validator, when it holds a managed value"), + )), + (status = 413, description = "The body exceeds 32 MiB, or the row's carried Blob payloads and the new value exceed the write's payload limit", body = ErrorOutput), + (status = 415, description = "Content-Type must be application/octet-stream", body = ErrorOutput), + (status = 424, description = "An allowed external Blob source carried from the row could not be read", body = ErrorOutput), + (status = 429, description = "Per-actor admission cap exceeded; honor `Retry-After` header", body = ErrorOutput), + (status = 503, description = "Write admission is closed or an overlapping durable recovery intent must be resolved before retry", body = ErrorOutput), + ), + security(("bearer_token" = [])), +)] +/// Replace one Blob value of an existing node or edge with the request body. +/// +/// The cell's old value is never read; the row's other cells are carried +/// unchanged. Authorization of `change` on the branch runs before the body is +/// read. Once admitted the write is owned by the server: a disconnect loses +/// only the response, never cancels or replays the write. +pub(crate) async fn server_blob_put( + State(state): State, + Extension(handle): Extension, + actor: Option>, + query: std::result::Result, QueryRejection>, + request: Request, +) -> std::result::Result { + let ingress = request + .extensions() + .get::() + .cloned() + .ok_or_else(|| ApiError::internal("missing ingress reservation"))?; + let query = parse_blob_write_query(query)?; + let precondition = blob_write_precondition(request.headers())?; + let actor = actor.as_ref().map(|Extension(actor)| actor); + let branch = query.branch.clone().unwrap_or_else(|| "main".to_string()); + authorize_blob_write(&handle, actor, &branch)?; + + let content_type = request + .headers() + .get(CONTENT_TYPE) + .and_then(|value| value.to_str().ok()) + .and_then(|value| value.split(';').next()) + .map(str::trim); + if !matches!(content_type, Some(value) if value.eq_ignore_ascii_case("application/octet-stream")) + { + return Err(ApiError::unsupported_media_type( + "a Blob put requires Content-Type: application/octet-stream", + )); + } + let declared = match request.headers().get(CONTENT_LENGTH) { + None => None, + Some(value) => Some( + value + .to_str() + .ok() + .and_then(|value| value.parse::().ok()) + .ok_or_else(|| ApiError::bad_request("invalid Content-Length"))?, + ), + }; + if let Some(declared) = declared.filter(|declared| *declared > omnigraph::BLOB_WRITE_MAX_BYTES) + { + return Err(blob_put_too_large(declared)); + } + + let deadline = request + .extensions() + .get::() + .copied() + .ok_or_else(|| ApiError::internal("missing request body deadline"))?; + let bytes = tokio::time::timeout_at( + deadline.0, + collect_blob_put_body(request.into_body(), declared), + ) + .await + .map_err(|_| ingress::body_timeout())??; + ingress + .shrink(bytes.len() as u64) + .map_err(ApiError::from_workload_reject)?; + let actor_arc = actor + .map(|actor| Arc::clone(&actor.actor_id)) + .unwrap_or_else(|| Arc::::from("anonymous")); + let admission = state + .workload + .try_admit(&actor_arc, bytes.len() as u64) + .map_err(ApiError::from_workload_reject)?; + + let session = state.session(&handle, None)?; + let selector = BlobSelectorOutput::from(&query); + let cell = blob_write_cell(query); + let actor_id = actor.map(|actor| actor.actor_id.to_string()); + let output = owned_write(&state, admission, ingress, handle.clone(), async move { + let outcome = session + .put_blob_at_as(&branch, cell, bytes, precondition, actor_id.as_deref()) + .await + .map_err(ApiError::from_omni)?; + Ok(api::blob_write_output(selector, branch, &outcome, actor_id)) + }) + .await?; + blob_write_response(output) +} + +#[utoipa::path( + delete, + path = "/blob", + tag = "blobs", + operation_id = "clearBlob", + params( + BlobWriteQuery, + ("If-Match" = Option, Header, description = "`*` to require a value in the cell, or a list of strong entity tags one of which must equal the cell's current ETag. A null cell satisfies neither form."), + ), + responses( + (status = 200, description = "The cell is null. `commit` is the clear's publication, or `null` when the cell already was null and nothing was published", body = BlobWriteOutput), + (status = 400, description = "Invalid selector, query parameter, If-Match field or target, a non-Blob or non-nullable property, or a carried external reference the graph's policy denies", body = ErrorOutput), + (status = 401, description = "Unauthorized", body = ErrorOutput), + (status = 403, description = "Forbidden", body = ErrorOutput), + (status = 404, description = "Unknown branch, or no entity with this id", body = ErrorOutput), + (status = 409, description = "Write-authority conflict, or the branch's incarnation or accepted schema changed while the write retried", body = ErrorOutput), + (status = 412, description = "If-Match did not hold for the cell; `blob_precondition_failure` names its current validator", body = ErrorOutput, + headers( + ("ETag" = String, description = "The cell's current validator, when it holds a managed value"), + )), + (status = 413, description = "The row's carried Blob payloads exceed the write's payload limit", body = ErrorOutput), + (status = 424, description = "An allowed external Blob source carried from the row could not be read", body = ErrorOutput), + (status = 429, description = "Per-actor admission cap exceeded; honor `Retry-After` header", body = ErrorOutput), + (status = 503, description = "Write admission is closed or an overlapping durable recovery intent must be resolved before retry", body = ErrorOutput), + ), + security(("bearer_token" = [])), +)] +/// Set one nullable Blob value of an existing node or edge to null. +/// +/// Clearing a cell that is already null publishes nothing. Like a put, the +/// clear is owned by the server once admitted. +pub(crate) async fn server_blob_delete( + State(state): State, + Extension(handle): Extension, + Extension(ingress): Extension, + actor: Option>, + headers: HeaderMap, + query: std::result::Result, QueryRejection>, +) -> std::result::Result { + let query = parse_blob_write_query(query)?; + let precondition = blob_write_precondition(&headers)?; + let actor = actor.as_ref().map(|Extension(actor)| actor); + let branch = query.branch.clone().unwrap_or_else(|| "main".to_string()); + authorize_blob_write(&handle, actor, &branch)?; + let actor_arc = actor + .map(|actor| Arc::clone(&actor.actor_id)) + .unwrap_or_else(|| Arc::::from("anonymous")); + let admission = state + .workload + .try_admit(&actor_arc, 0) + .map_err(ApiError::from_workload_reject)?; + + let session = state.session(&handle, None)?; + let selector = BlobSelectorOutput::from(&query); + let cell = blob_write_cell(query); + let actor_id = actor.map(|actor| actor.actor_id.to_string()); + let output = owned_write(&state, admission, ingress, handle.clone(), async move { + let outcome = session + .clear_blob_at_as(&branch, cell, precondition, actor_id.as_deref()) + .await + .map_err(ApiError::from_omni)?; + Ok(api::blob_write_output(selector, branch, &outcome, actor_id)) + }) + .await?; + blob_write_response(output) +} + +fn parse_blob_write_query( + query: std::result::Result, QueryRejection>, +) -> std::result::Result { + query.map(|Query(query)| query).map_err(|rejection| { + ApiError::bad_request(format!( + "invalid Blob write query parameters: {}", + rejection.body_text() + )) + }) +} + +fn blob_write_precondition( + headers: &HeaderMap, +) -> std::result::Result, ApiError> { + api::parse_blob_if_match( + headers + .get_all(axum::http::header::IF_MATCH) + .iter() + .map(|value| value.as_bytes()), + ) + .map_err(ApiError::bad_request) +} + +/// A Blob write needs `change` on its branch, checked before any body byte +/// is read. +fn authorize_blob_write( + handle: &GraphHandle, + actor: Option<&AuthenticatedActor>, + branch: &str, +) -> std::result::Result<(), ApiError> { + authorize_request( + actor, + handle.policy.as_deref(), + PolicyRequest { + action: PolicyAction::Change, + branch: Some(branch.to_string()), + target_branch: None, + }, + ) +} + +fn blob_write_cell(query: BlobWriteQuery) -> omnigraph::BlobCell { + omnigraph::BlobCell { + entity: match query.entity { + api::BlobEntityKind::Node => omnigraph::EntityKind::Node, + api::BlobEntityKind::Edge => omnigraph::EntityKind::Edge, + }, + type_name: query.r#type, + id: query.id, + property: query.property, + } +} + +/// The refusal of a put body over the limit: the same resource, limit and +/// message the engine reports for an oversized embedded put. +fn blob_put_too_large(actual: u64) -> ApiError { + ApiError::from_omni(OmniError::resource_limit( + omnigraph::BLOB_WRITE_PAYLOAD_RESOURCE, + omnigraph::BLOB_WRITE_MAX_BYTES, + actual, + )) +} + +/// Collect a put body into one buffer, sized from a declared length that is +/// already within the limit. The engine adopts the buffer without a copy. +async fn collect_blob_put_body( + body: Body, + declared: Option, +) -> std::result::Result { + let capacity = declared.map_or(Ok(0), usize::try_from).map_err(|_| { + ApiError::bad_request("Content-Length does not fit in this platform's memory") + })?; + let mut data = Vec::with_capacity(capacity); + let mut body = body.into_data_stream(); + while let Some(chunk) = body.next().await { + let chunk = chunk.map_err(|err| { + ApiError::bad_request(format!("failed to read Blob request body: {err}")) + })?; + let actual = data.len().saturating_add(chunk.len()) as u64; + if actual > omnigraph::BLOB_WRITE_MAX_BYTES { + return Err(blob_put_too_large(actual)); + } + data.extend_from_slice(&chunk); + } + if declared.is_some_and(|declared| declared != data.len() as u64) { + return Err(ApiError::bad_request( + "Blob request body length does not match its Content-Length", + )); + } + Ok(Bytes::from(data)) +} + +/// A write receipt with its `ETag` header when the cell holds a managed value. +fn blob_write_response(output: BlobWriteOutput) -> std::result::Result { + let mut headers = HeaderMap::new(); + if let Some(etag) = output.etag.as_deref() { + headers.insert( + axum::http::header::ETAG, + axum::http::HeaderValue::from_str(etag) + .map_err(|_| ApiError::internal("a managed Blob ETag is not a header value"))?, + ); + } + Ok((StatusCode::OK, headers, Json(output)).into_response()) +} + fn parse_blob_read_query( query: std::result::Result, QueryRejection>, ) -> std::result::Result { diff --git a/crates/omnigraph-server/src/ingress.rs b/crates/omnigraph-server/src/ingress.rs index e826c4a1b..d470adcad 100644 --- a/crates/omnigraph-server/src/ingress.rs +++ b/crates/omnigraph-server/src/ingress.rs @@ -133,9 +133,14 @@ pub(crate) async fn admit( AdmissionClass::Write => state.operations.try_observe_write()?, }; let receives_body = parts.method == Method::POST || parts.method == Method::PUT; - let raw = receives_body && route == "/load/ndjson"; + // A raw route authorizes before it polls its body, and collects the body + // itself under the deadline and the lease reserved here. + let blob_put = parts.method == Method::PUT && route == "/blob"; + let raw = receives_body && (route == "/load/ndjson" || blob_put); let limit = if !receives_body { 0 + } else if blob_put { + crate::BLOB_PUT_REQUEST_BODY_LIMIT_BYTES } else if matches!(route, "/cluster/deployments" | "/cluster/plan") { crate::deployment::REQUEST_BYTES } else if route == "/mcp" { @@ -186,8 +191,9 @@ pub(crate) async fn admit( } else { body }; - // Raw NDJSON authorizes its branch scope before polling any body bytes. - // Its collector consumes BodyDeadline and the same retained lease. + // Raw NDJSON and a Blob put authorize their branch scope before polling + // any body bytes. Their collectors consume BodyDeadline and the same + // retained lease. let request = Request::from_parts(parts, body); match class { AdmissionClass::Read => { diff --git a/crates/omnigraph-server/src/lib.rs b/crates/omnigraph-server/src/lib.rs index d9c52a7f8..fa1a31e09 100644 --- a/crates/omnigraph-server/src/lib.rs +++ b/crates/omnigraph-server/src/lib.rs @@ -43,12 +43,13 @@ use std::path::PathBuf; use std::sync::Arc; use api::{ - BlobReadQuery, BranchCreateOutput, BranchCreateRequest, BranchDeleteOutput, BranchListOutput, - BranchMergeOutput, BranchMergeRequest, ChangeOutput, ChangeRequest, CommitListOutput, - CommitListQuery, ErrorCode, ErrorOutput, ExportRequest, GraphBatchLoadOutput, - GraphBatchLoadQuery, GraphDiscoveryEntry, GraphDiscoveryResponse, GraphInfo, GraphListResponse, - HealthOutput, IngestOutput, IngestRequest, InvokeStoredQueryRequest, InvokeStoredQueryResponse, - QueriesCatalogOutput, QueryRequest, ReadOutput, ReadinessOutput, SchemaOutput, SnapshotQuery, + BlobReadQuery, BlobSelectorOutput, BlobWriteOutput, BlobWriteQuery, BranchCreateOutput, + BranchCreateRequest, BranchDeleteOutput, BranchListOutput, BranchMergeOutput, + BranchMergeRequest, ChangeOutput, ChangeRequest, CommitListOutput, CommitListQuery, ErrorCode, + ErrorOutput, ExportRequest, GraphBatchLoadOutput, GraphBatchLoadQuery, GraphDiscoveryEntry, + GraphDiscoveryResponse, GraphInfo, GraphListResponse, HealthOutput, IngestOutput, + IngestRequest, InvokeStoredQueryRequest, InvokeStoredQueryResponse, QueriesCatalogOutput, + QueryRequest, ReadOutput, ReadinessOutput, SchemaOutput, SnapshotQuery, graph_batch_load_receipt_output, ingest_receipt_output, snapshot_payload, }; pub use auth::{AWS_SECRET_ENV, EnvOrFileTokenSource, TokenSource, resolve_token_source}; @@ -121,6 +122,8 @@ fn hash_bearer_token(token: &str) -> BearerTokenHash { handlers::server_snapshot, handlers::server_blob_get, handlers::server_blob_head, + handlers::server_blob_put, + handlers::server_blob_delete, handlers::server_query, handlers::server_export, handlers::server_mutate, @@ -177,6 +180,8 @@ impl utoipa::Modify for SecurityAddon { const DEFAULT_REQUEST_BODY_LIMIT_BYTES: usize = 1_048_576; const INGEST_REQUEST_BODY_LIMIT_BYTES: usize = 32 * 1024 * 1024; +/// A Blob put's raw body: exactly the bytes one put may store, inclusive. +const BLOB_PUT_REQUEST_BODY_LIMIT_BYTES: usize = omnigraph::BLOB_WRITE_MAX_BYTES as usize; const SERVER_VERSION: &str = env!("CARGO_PKG_VERSION"); const SERVER_SOURCE_VERSION: Option<&str> = option_env!("OMNIGRAPH_SOURCE_VERSION"); /// The maximum internal-schema (storage-format) version this binary supports. @@ -462,6 +467,7 @@ enum ApiErrorDetails { ExternalBlobSource(api::ExternalBlobSourceOutput), RecoveryRequired(api::RecoveryRequiredOutput), PreconditionFailure(api::PreconditionFailureOutput), + BlobPreconditionFailure(api::BlobPreconditionFailureOutput), ChangeFeedGap(api::ChangeFeedGapOutput), ChangeDiffRefusal(api::ChangeDiffRefusalOutput), FullTextIndexRebuildRequired(api::FullTextIndexRebuildRequiredOutput), @@ -1080,6 +1086,27 @@ impl ApiError { } } + /// HTTP 412 for a Blob write whose `If-Match` did not hold. The + /// `blob_precondition_failure` detail carries the cell's current + /// validator, and the response repeats it in `ETag` when there is one. + /// It keeps the closed [`ErrorCode::Conflict`] signal and never the + /// graph-commit `precondition_failure` detail. + pub(crate) fn blob_write_precondition_failed(current_etag: Option) -> Self { + Self { + completion_uncertain: false, + status: StatusCode::PRECONDITION_FAILED, + code: Some(ErrorCode::Conflict), + message: format!( + "Blob write precondition failed; the cell's current validator is {}", + current_etag.as_deref().unwrap_or("") + ) + .into_boxed_str(), + details: Some(Box::new(ApiErrorDetails::BlobPreconditionFailure( + api::BlobPreconditionFailureOutput { current_etag }, + ))), + } + } + pub fn internal(message: impl Into) -> Self { Self { completion_uncertain: true, @@ -1471,8 +1498,8 @@ impl ApiError { ), api::PreconditionFailureOutput { expected, actual }, ), - err @ OmniError::BlobWritePreconditionFailed { .. } => { - Self::blob_precondition_failed(err.to_string()) + OmniError::BlobWritePreconditionFailed { current_etag } => { + Self::blob_write_precondition_failed(current_etag) } err @ OmniError::ExternalBlobPolicy { .. } => Self::bad_request(err.to_string()), err @ OmniError::StoredExternalBlobDenied { .. } => Self::bad_request(err.to_string()), @@ -1583,6 +1610,14 @@ impl IntoResponse for ApiError { axum::http::HeaderValue::from_static(RETRY_AFTER_SECONDS), ); } + if let Some(ApiErrorDetails::BlobPreconditionFailure(failure)) = self.details.as_deref() + && let Some(etag) = failure + .current_etag + .as_deref() + .and_then(|etag| axum::http::HeaderValue::from_str(etag).ok()) + { + headers.insert(axum::http::header::ETAG, etag); + } let status = self.status; (status, headers, Json(self.into_output())).into_response() } @@ -1611,6 +1646,9 @@ impl ApiError { ApiErrorDetails::PreconditionFailure(value) => { output.precondition_failure = Some(value) } + ApiErrorDetails::BlobPreconditionFailure(value) => { + output.blob_precondition_failure = Some(value) + } ApiErrorDetails::ChangeFeedGap(value) => output.change_feed_gap = Some(value), ApiErrorDetails::ChangeDiffRefusal(value) => { output.change_diff_refusal = Some(value) @@ -1966,6 +2004,38 @@ mod api_error_tests { .unwrap(); let error: ErrorOutput = serde_json::from_slice(&body).unwrap(); assert_eq!(error.code, Some(ErrorCode::Internal)); + + // A Blob write's failed If-Match: 412 under the closed conflict code, + // the current validator in the detail and in `ETag`, and never the + // graph-commit precondition detail. + for current in [Some("\"current\""), None] { + let response = ApiError::from_omni(OmniError::blob_write_precondition_failed( + current.map(str::to_string), + )) + .into_response(); + assert_eq!(response.status(), StatusCode::PRECONDITION_FAILED); + assert_eq!( + response + .headers() + .get(axum::http::header::ETAG) + .map(|etag| etag.to_str().unwrap()), + current + ); + let body = axum::body::to_bytes(response.into_body(), usize::MAX) + .await + .unwrap(); + let error: ErrorOutput = serde_json::from_slice(&body).unwrap(); + assert_eq!(error.code, Some(ErrorCode::Conflict)); + assert!(error.precondition_failure.is_none()); + assert_eq!( + error + .blob_precondition_failure + .expect("structured Blob precondition") + .current_etag + .as_deref(), + current + ); + } } #[tokio::test] @@ -2390,7 +2460,13 @@ pub fn build_app(state: AppState) -> Router { // Register HEAD explicitly. Axum's GET fallback would invoke the GET // handler and could begin payload work before stripping the body; the // dedicated handler makes the zero-payload-read contract structural. - .route("/blob", get(server_blob_get).head(server_blob_head)) + .route( + "/blob", + get(server_blob_get) + .head(server_blob_head) + .put(server_blob_put) + .delete(server_blob_delete), + ) .route("/export", post(server_export)) .route("/query", post(server_query)) .route("/mutate", post(server_mutate)) diff --git a/crates/omnigraph-server/tests/auth_policy.rs b/crates/omnigraph-server/tests/auth_policy.rs index 331820d61..a10023fef 100644 --- a/crates/omnigraph-server/tests/auth_policy.rs +++ b/crates/omnigraph-server/tests/auth_policy.rs @@ -425,6 +425,8 @@ async fn signed_data_requires_cedar_and_rejects_forgery_on_every_protected_route (Method::GET, "/snapshot"), (Method::GET, "/blob"), (Method::HEAD, "/blob"), + (Method::PUT, "/blob"), + (Method::DELETE, "/blob"), (Method::POST, "/export"), (Method::POST, "/query"), (Method::POST, "/mutate"), @@ -1369,7 +1371,13 @@ async fn policy_commit_diff_forbidden_is_indistinguishable_from_unknown() { #[tokio::test(flavor = "multi_thread")] async fn policy_blocks_change_on_protected_main_but_allows_unprotected_branch() { - let temp = init_loaded_graph().await; + let temp = init_graph_with_schema_and_data( + &fs::read_to_string(fixture("test.pg")) + .unwrap() + .replace(" age: I32?", " age: I32?\n avatar: Blob?"), + &fs::read_to_string(fixture("test.jsonl")).unwrap(), + ) + .await; let graph = graph_path(temp.path()); let db = Omnigraph::open(graph.to_str().unwrap()).await.unwrap(); db.branch_create_from(ReadTarget::branch("main"), "feature") @@ -1436,6 +1444,49 @@ async fn policy_blocks_change_on_protected_main_but_allows_unprotected_branch() assert_eq!(feature_status, StatusCode::OK); assert_eq!(feature_body["branch"], "feature"); assert_eq!(feature_body["affected_nodes"], 1); + + // A Blob put or clear is a `change` too: refused on protected main + // before its body is polled, and attributed to the actor elsewhere. + for (method, branch) in [ + (Method::PUT, "main"), + (Method::DELETE, "main"), + (Method::PUT, "feature"), + (Method::DELETE, "feature"), + ] { + let polled = Arc::new(std::sync::atomic::AtomicBool::new(false)); + let observed = Arc::clone(&polled); + let body = Body::from_stream(futures::stream::once(async move { + observed.store(true, std::sync::atomic::Ordering::SeqCst); + Ok::<_, std::convert::Infallible>(axum::body::Bytes::from_static(b"avatar")) + })); + let (status, output) = json_response( + &app, + Request::builder() + .header(HTTP_API_CONTRACT_HEADER, HTTP_API_CONTRACT) + .uri(g(&format!( + "/blob?entity=node&type=Person&id=Alice&property=avatar&branch={branch}" + ))) + .method(method.clone()) + .header("authorization", "Bearer team-token") + .header("content-type", "application/octet-stream") + .body(body) + .unwrap(), + ) + .await; + if branch == "main" { + assert_eq!(status, StatusCode::FORBIDDEN, "{method}: {output}"); + assert!( + !polled.load(std::sync::atomic::Ordering::SeqCst), + "{method}: Cedar refusal precedes the body" + ); + } else { + assert_eq!(status, StatusCode::OK, "{method}: {output}"); + assert_eq!(output["branch"], "feature"); + assert_eq!(output["actor_id"], "act-bruno"); + assert_eq!(output["commit"]["actor_id"], "act-bruno"); + assert_eq!(output["commit"]["graph_branch"], "feature"); + } + } } #[tokio::test(flavor = "multi_thread")] diff --git a/crates/omnigraph-server/tests/data_routes.rs b/crates/omnigraph-server/tests/data_routes.rs index 22974c333..7bf047abb 100644 --- a/crates/omnigraph-server/tests/data_routes.rs +++ b/crates/omnigraph-server/tests/data_routes.rs @@ -74,6 +74,59 @@ fn repeated_zero_blob_input(length: usize) -> String { format!("base64:{}{tail}", "AAAA".repeat(full_triples)) } +/// A Blob write: PUT with a raw octet-stream body, or DELETE without one. +fn blob_write_request( + method: Method, + uri: &str, + body: impl Into, + if_match: Option<&str>, +) -> Request { + let mut builder = Request::builder() + .header(HTTP_API_CONTRACT_HEADER, HTTP_API_CONTRACT) + .uri(uri) + .method(method.clone()); + if method == Method::PUT { + builder = builder.header("content-type", "application/octet-stream"); + } + if let Some(tag) = if_match { + builder = builder.header("if-match", tag); + } + builder.body(body.into()).unwrap() +} + +/// Status, `ETag` header and body of one response; the body is JSON for a +/// write and raw bytes for a read. A response chunk owns its request's +/// admission until it drops, so the body is copied out here and the request +/// releases its capacity when this returns. +async fn blob_exchange( + app: &axum::Router, + request: Request, +) -> (StatusCode, Option, Vec) { + let response = app.clone().oneshot(request).await.unwrap(); + let status = response.status(); + let etag = response + .headers() + .get("etag") + .map(|value| value.to_str().unwrap().to_string()); + let body = to_bytes(response.into_body(), usize::MAX).await.unwrap(); + (status, etag, body.to_vec()) +} + +async fn main_commit_count(app: &axum::Router) -> usize { + let (status, commits) = json_response( + app, + Request::builder() + .header(HTTP_API_CONTRACT_HEADER, HTTP_API_CONTRACT) + .uri(g("/commits")) + .method(Method::GET) + .body(Body::empty()) + .unwrap(), + ) + .await; + assert_eq!(status, StatusCode::OK, "{commits}"); + commits["commits"].as_array().unwrap().len() +} + async fn assert_receipt_commit_matches_get(app: &axum::Router, output: &Value) { let receipt = output .get("commit") @@ -622,6 +675,325 @@ async fn blob_get_preserves_empty_null_edge_and_target_semantics() { } } +/// PUT and DELETE `/blob` replace or clear one cell of a node or an edge by +/// exact id and return the exact receipt of their own commit. A failed +/// `If-Match` is a 412 naming the cell's validator in `ETag` and +/// `blob_precondition_failure`, never the graph-commit +/// `precondition_failure`, and publishes nothing. +#[tokio::test(flavor = "multi_thread")] +async fn blob_put_and_delete_return_exact_receipts_and_blob_preconditions() { + let (_temp, app) = app_for_blob_http_data(BLOB_HTTP_DATA).await; + let precondition_failure = |body: &[u8]| { + let output: Value = serde_json::from_slice(body).unwrap(); + assert_eq!(output["code"], "conflict", "{output}"); + assert!(output.get("precondition_failure").is_none(), "{output}"); + output["blob_precondition_failure"].clone() + }; + for (entity, type_name, id, property, original) in [ + ("node", "Document", "readme", "content", &b"Hello World"[..]), + ( + "edge", + "Attachment", + "attachment-1", + "payload", + &b"Edge"[..], + ), + ] { + let uri = blob_uri(entity, type_name, id, property, ""); + let get = || blob_write_request(Method::GET, &uri, Body::empty(), None); + let (status, original_etag, bytes) = blob_exchange(&app, get()).await; + assert_eq!((status, &bytes[..]), (StatusCode::OK, original)); + let original_etag = original_etag.unwrap(); + let commits = main_commit_count(&app).await; + + let (status, etag, body) = blob_exchange( + &app, + blob_write_request(Method::PUT, &uri, "never", Some("\"stale\"")), + ) + .await; + assert_eq!(status, StatusCode::PRECONDITION_FAILED, "{entity}"); + assert_eq!(etag.as_deref(), Some(original_etag.as_str())); + assert_eq!( + precondition_failure(&body), + json!({"current_etag": original_etag}) + ); + assert_eq!( + main_commit_count(&app).await, + commits, + "a 412 publishes nothing" + ); + + let (status, etag, body) = blob_exchange( + &app, + blob_write_request(Method::PUT, &uri, "Replaced", Some(&original_etag)), + ) + .await; + let receipt: Value = serde_json::from_slice(&body).unwrap(); + assert_eq!(status, StatusCode::OK, "{receipt}"); + let replaced_etag = etag.expect("a put returns the stored value's ETag"); + assert_ne!(replaced_etag, original_etag); + assert_eq!( + receipt["selector"], + json!({"entity": entity, "type": type_name, "id": id, "property": property}) + ); + assert_eq!(receipt["branch"], "main"); + assert_eq!(receipt["kind"], "managed"); + assert_eq!(receipt["size"], 8); + assert_eq!(receipt["etag"], replaced_etag); + assert!(receipt["actor_id"].is_null()); + assert_receipt_commit_matches_get(&app, &receipt).await; + let at_commit = blob_uri( + entity, + type_name, + id, + property, + &format!( + "&snapshot={}", + receipt["commit"]["graph_commit_id"].as_str().unwrap() + ), + ); + for read in [ + get(), + blob_write_request(Method::GET, &at_commit, Body::empty(), None), + ] { + let (status, etag, bytes) = blob_exchange(&app, read).await; + assert_eq!((status, &bytes[..]), (StatusCode::OK, &b"Replaced"[..])); + assert_eq!( + etag.as_deref(), + Some(replaced_etag.as_str()), + "the receipt's ETag is the one a read at its commit reports" + ); + } + + let (status, etag, body) = blob_exchange( + &app, + blob_write_request(Method::DELETE, &uri, Body::empty(), Some(&replaced_etag)), + ) + .await; + let receipt: Value = serde_json::from_slice(&body).unwrap(); + assert_eq!(status, StatusCode::OK, "{receipt}"); + assert_eq!(etag, None); + assert_eq!(receipt["kind"], "null"); + assert!(receipt.get("size").is_none() && receipt.get("etag").is_none()); + assert_receipt_commit_matches_get(&app, &receipt).await; + assert_eq!(blob_exchange(&app, get()).await.0, StatusCode::NOT_FOUND); + let commits = main_commit_count(&app).await; + + let (status, _, body) = blob_exchange( + &app, + blob_write_request(Method::DELETE, &uri, Body::empty(), None), + ) + .await; + let receipt: Value = serde_json::from_slice(&body).unwrap(); + assert_eq!(status, StatusCode::OK, "{receipt}"); + assert!( + receipt["commit"].is_null(), + "clearing a null cell publishes nothing: {receipt}" + ); + for (method, body) in [(Method::DELETE, ""), (Method::PUT, "any")] { + let (status, etag, body) = blob_exchange( + &app, + blob_write_request(method.clone(), &uri, body, Some("*")), + ) + .await; + assert_eq!(status, StatusCode::PRECONDITION_FAILED, "{method} *"); + assert_eq!(etag, None, "a null cell has no validator"); + assert_eq!(precondition_failure(&body), json!({})); + } + assert_eq!(main_commit_count(&app).await, commits); + + let (status, _, body) = blob_exchange( + &app, + blob_write_request(Method::PUT, &uri, original.to_vec(), None), + ) + .await; + assert_eq!(status, StatusCode::OK, "{}", String::from_utf8_lossy(&body)); + assert_eq!(&blob_exchange(&app, get()).await.2[..], original); + } + + let commits = main_commit_count(&app).await; + for (uri, if_match, expected) in [ + ( + blob_uri("node", "Document", "missing", "content", ""), + None, + StatusCode::NOT_FOUND, + ), + ( + blob_uri("node", "Document", "readme", "content", "&branch=absent"), + None, + StatusCode::NOT_FOUND, + ), + ( + blob_uri("node", "Document", "readme", "title", ""), + None, + StatusCode::BAD_REQUEST, + ), + ( + blob_uri("node", "Document", "readme", "content", "&snapshot=any"), + None, + StatusCode::BAD_REQUEST, + ), + ( + blob_uri("node", "Document", "readme", "content", ""), + Some("unquoted"), + StatusCode::BAD_REQUEST, + ), + ] { + for method in [Method::PUT, Method::DELETE] { + let (status, _, body) = blob_exchange( + &app, + blob_write_request(method.clone(), &uri, "x", if_match), + ) + .await; + assert_eq!( + status, + expected, + "{method} {uri}: {}", + String::from_utf8_lossy(&body) + ); + } + } + assert_eq!( + main_commit_count(&app).await, + commits, + "refusals publish nothing" + ); +} + +/// Owners release after the response; wait for them before reading the +/// capacity they held. +async fn assert_admission_open( + operations: &omnigraph_server::operations::OperationRuntime, + workload: &omnigraph_server::workload::WorkloadController, + step: &str, +) { + let settled = + tokio::time::timeout(Duration::from_secs(10), operations.wait_logical_owners()).await; + assert!( + settled.is_ok_and(|clean| clean), + "{step}: {:?} {:?}", + operations.snapshot(), + workload.snapshot() + ); + assert_eq!(operations.snapshot().active_writes, 0, "{step}"); + assert_eq!( + workload.snapshot(), + omnigraph_server::workload::WorkloadSnapshot::default(), + "{step}" + ); +} + +/// The raw PUT body: a wrong media type and a declared length over 32 MiB are +/// refused before any body byte is polled, exactly 32 MiB is stored and one +/// more byte is refused, and an expired body deadline is a 408. Every +/// refusal leaves admission open. +#[tokio::test(flavor = "multi_thread")] +async fn blob_put_raw_body_bounds_media_type_length_and_deadline() { + const LIMIT: usize = 32 * 1024 * 1024; + let temp = init_graph_with_schema_and_data(BLOB_HTTP_SCHEMA, BLOB_HTTP_DATA).await; + let graph = graph_path(temp.path()); + let workload = omnigraph_server::workload::WorkloadController::with_limits( + omnigraph_server::workload::WorkloadLimits { + body_timeout: Duration::from_secs(2), + ..Default::default() + }, + ); + let state = AppState::new_with_workload( + graph.to_string_lossy().to_string(), + Omnigraph::open(graph.to_str().unwrap()).await.unwrap(), + Vec::new(), + workload.clone(), + ); + let operations = state.operation_runtime().clone(); + let app = build_app(state); + let uri = blob_uri("node", "Document", "readme", "content", ""); + let watched_body = || { + let polled = Arc::new(AtomicBool::new(false)); + let observed = Arc::clone(&polled); + let body = Body::from_stream(futures::stream::once(async move { + observed.store(true, Ordering::SeqCst); + Ok::(Bytes::from_static(b"x")) + })); + (polled, body) + }; + + let (polled, body) = watched_body(); + let mut request = blob_write_request(Method::PUT, &uri, body, None); + request + .headers_mut() + .insert("content-type", HeaderValue::from_static("application/json")); + let (status, _, _) = blob_exchange(&app, request).await; + assert_eq!(status, StatusCode::UNSUPPORTED_MEDIA_TYPE); + assert!(!polled.load(Ordering::SeqCst), "415 precedes the body"); + assert_admission_open(&operations, &workload, "415").await; + + let (polled, body) = watched_body(); + let mut request = blob_write_request(Method::PUT, &uri, body, None); + request.headers_mut().insert( + "content-length", + HeaderValue::from_str(&(LIMIT + 1).to_string()).unwrap(), + ); + let (status, _, body) = blob_exchange(&app, request).await; + assert_eq!(status, StatusCode::PAYLOAD_TOO_LARGE); + assert!( + !polled.load(Ordering::SeqCst), + "a declared 413 precedes the body" + ); + let refused: ErrorOutput = serde_json::from_slice(&body).unwrap(); + let limit = refused.resource_limit.unwrap(); + assert_eq!( + (limit.resource.as_str(), limit.limit, limit.actual), + ("Blob write payload bytes", LIMIT as u64, LIMIT as u64 + 1) + ); + assert_admission_open(&operations, &workload, "declared 413").await; + + let mut request = blob_write_request(Method::PUT, &uri, vec![7_u8; LIMIT], None); + request.headers_mut().insert( + "content-length", + HeaderValue::from_str(&LIMIT.to_string()).unwrap(), + ); + let (status, _, body) = blob_exchange(&app, request).await; + let receipt: Value = serde_json::from_slice(&body).unwrap(); + assert_eq!(status, StatusCode::OK, "{receipt}"); + assert_eq!(receipt["size"], LIMIT as u64, "exactly 32 MiB is stored"); + + // No declared length: the collector refuses the byte past the limit. + let chunks = futures::stream::iter([ + Ok::(Bytes::from(vec![7_u8; LIMIT])), + Ok(Bytes::from_static(b"!")), + ]); + let (status, _, body) = blob_exchange( + &app, + blob_write_request(Method::PUT, &uri, Body::from_stream(chunks), None), + ) + .await; + assert_eq!(status, StatusCode::PAYLOAD_TOO_LARGE); + let refused: ErrorOutput = serde_json::from_slice(&body).unwrap(); + assert_eq!(refused.resource_limit.unwrap().actual, LIMIT as u64 + 1); + assert_admission_open(&operations, &workload, "collected 413").await; + + let stalled = futures::StreamExt::chain( + futures::stream::once(async { Ok::(Bytes::from_static(b"x")) }), + futures::stream::pending(), + ); + let (status, _, _) = tokio::time::timeout( + Duration::from_secs(10), + blob_exchange( + &app, + blob_write_request(Method::PUT, &uri, Body::from_stream(stalled), None), + ), + ) + .await + .expect("the body deadline bounds an incomplete upload"); + assert_eq!(status, StatusCode::REQUEST_TIMEOUT); + assert_admission_open(&operations, &workload, "408").await; + + let (status, _, body) = + blob_exchange(&app, blob_write_request(Method::PUT, &uri, "after", None)).await; + assert_eq!(status, StatusCode::OK, "{}", String::from_utf8_lossy(&body)); + assert_admission_open(&operations, &workload, "after").await; +} + #[tokio::test(flavor = "multi_thread")] async fn blob_external_get_and_head_redirect_without_target_io() { let temp = tempfile::tempdir().unwrap(); @@ -1639,10 +2011,43 @@ async fn disconnected_writes_keep_admission_until_the_original_operation_finishe "{\"type\":\"Company\",\"data\":{\"name\":\"OwnedCo\"}}\n", "{\"edge\":\"WorksAt\",\"from\":\"Owned\",\"to\":\"OwnedCo\"}\n" ); - for door in ["/mutate", "/load", "/load/ndjson", "/branches/merge"] { - let temp = init_loaded_graph().await; + const PHOTO: &str = "owned photo"; + for door in [ + "/mutate", + "/load", + "/load/ndjson", + "/branches/merge", + "PUT /blob", + "DELETE /blob", + ] { + // A Blob door writes Alice's photo on the standard rows; it adds no + // Person, so its Person counts stay one lower. + let blob_door = door.ends_with(" /blob"); + let temp = if blob_door { + init_graph_with_schema_and_data( + &fs::read_to_string(fixture("test.pg")) + .unwrap() + .replace(" age: I32?", " age: I32?\n photo: Blob?"), + &fs::read_to_string(fixture("test.jsonl")).unwrap(), + ) + .await + } else { + init_loaded_graph().await + }; let graph = graph_path(temp.path()); let db = Arc::new(Omnigraph::open(graph.to_str().unwrap()).await.unwrap()); + if door == "DELETE /blob" { + Session::from_defaults(Arc::clone(&db), SessionSettings::default()) + .put_blob_at_as( + "main", + alice_photo(), + Bytes::from_static(b"seeded"), + None, + None, + ) + .await + .unwrap(); + } if door == "/branches/merge" { db.branch_create("feature").await.unwrap(); Session::from_defaults(Arc::clone(&db), SessionSettings::default()) @@ -1685,6 +2090,10 @@ async fn disconnected_writes_keep_admission_until_the_original_operation_finishe .header("content-type", "application/x-ndjson") .body(Body::from(BATCH)) .unwrap(), + "PUT /blob" => blob_write_request(Method::PUT, &alice_photo_uri(), PHOTO, None), + "DELETE /blob" => { + blob_write_request(Method::DELETE, &alice_photo_uri(), Body::empty(), None) + } _ => json_post(door, &json!({"source": "feature", "delete_branch": true})), }; let holder_db = Arc::clone(&db); @@ -1753,10 +2162,15 @@ async fn disconnected_writes_keep_admission_until_the_original_operation_finishe ); assert_eq!(workload.snapshot().operation_count, 1); assert_eq!(workload.snapshot().ingress_count, 1); - assert!( - workload.snapshot().ingress_bytes > 0, - "disconnect cannot release the retained input budget" - ); + match door { + // The raw put body's reservation shrank to the bytes received. + "PUT /blob" => assert_eq!(workload.snapshot().ingress_bytes, PHOTO.len() as u64), + "DELETE /blob" => assert_eq!(workload.snapshot().ingress_bytes, 0), + _ => assert!( + workload.snapshot().ingress_bytes > 0, + "disconnect cannot release the retained input budget" + ), + } let transition = state .prepare_same_view( &view.key, @@ -1805,14 +2219,28 @@ async fn disconnected_writes_keep_admission_until_the_original_operation_finishe after[0].parent_commit_id, Some(before[0].graph_commit_id.clone()) ); - assert_eq!(harness.person_count("main").await, 5, "{door}"); - let (status, rows) = json_response(&app, json_post("/query", &json!({"query": "query q() { match { $p: Person { name: \"Owned\" } $p worksAt $c } return { $p.name, $c.name } }"}))).await; - assert_eq!(status, StatusCode::OK, "{rows}"); - assert_eq!( - rows["rows"], - json!([{ "p.name": "Owned", "c.name": "OwnedCo" }]), - "all participants must publish together" - ); + let persons = if blob_door { 4 } else { 5 }; + assert_eq!(harness.person_count("main").await, persons, "{door}"); + if blob_door { + let (status, _, bytes) = blob_exchange( + &app, + blob_write_request(Method::GET, &alice_photo_uri(), Body::empty(), None), + ) + .await; + if door == "PUT /blob" { + assert_eq!((status, &bytes[..]), (StatusCode::OK, PHOTO.as_bytes())); + } else { + assert_eq!(status, StatusCode::NOT_FOUND, "the clear published"); + } + } else { + let (status, rows) = json_response(&app, json_post("/query", &json!({"query": "query q() { match { $p: Person { name: \"Owned\" } $p worksAt $c } return { $p.name, $c.name } }"}))).await; + assert_eq!(status, StatusCode::OK, "{rows}"); + assert_eq!( + rows["rows"], + json!([{ "p.name": "Owned", "c.name": "OwnedCo" }]), + "all participants must publish together" + ); + } if door == "/branches/merge" { assert!( !db.branch_list() @@ -1828,10 +2256,23 @@ async fn disconnected_writes_keep_admission_until_the_original_operation_finishe StatusCode::OK, "same process must resume: {sentinel}" ); - assert_eq!(harness.person_count("main").await, 6); + assert_eq!(harness.person_count("main").await, persons + 1); } } +fn alice_photo() -> omnigraph::BlobCell { + omnigraph::BlobCell { + entity: omnigraph::EntityKind::Node, + type_name: "Person".to_string(), + id: "Alice".to_string(), + property: "photo".to_string(), + } +} + +fn alice_photo_uri() -> String { + blob_uri("node", "Person", "Alice", "photo", "") +} + struct ThreadEngineFault { thread: std::thread::ThreadId, reached: Arc, diff --git a/crates/omnigraph-server/tests/openapi.rs b/crates/omnigraph-server/tests/openapi.rs index 40f6ba44e..3f8f4ef70 100644 --- a/crates/omnigraph-server/tests/openapi.rs +++ b/crates/omnigraph-server/tests/openapi.rs @@ -610,6 +610,97 @@ fn openapi_blob_supports_get_and_explicit_head() { } } +/// PUT and DELETE `/blob` take the selector and a branch, never a snapshot; +/// PUT's body is raw binary; both answer the exact receipt and document the +/// Blob 412 apart from the graph-commit one. +#[test] +fn openapi_blob_put_and_delete_document_raw_body_receipt_and_blob_precondition() { + let doc = openapi_json(); + let path = &doc["paths"]["/graphs/{graph_id}/blob"]; + for method in ["put", "delete"] { + let operation = &path[method]; + assert!(operation.is_object(), "{method} /blob is documented"); + let parameters = operation["parameters"].as_array().unwrap(); + let parameter = |name: &str| { + parameters + .iter() + .find(|parameter| parameter["name"] == name) + .cloned() + }; + for required in ["entity", "type", "id", "property"] { + let parameter = parameter(required) + .unwrap_or_else(|| panic!("{method} /blob is missing `{required}`")); + assert_eq!( + (¶meter["in"], ¶meter["required"]), + (&"query".into(), &true.into()) + ); + } + assert_ne!(parameter("branch").unwrap()["required"], true); + assert!( + parameter("snapshot").is_none(), + "{method} /blob writes a branch" + ); + assert_eq!(parameter("If-Match").unwrap()["in"], "header"); + + let ok = &operation["responses"]["200"]; + assert_eq!( + ok["content"]["application/json"]["schema"]["$ref"], + "#/components/schemas/BlobWriteOutput" + ); + assert!( + operation["responses"]["412"]["headers"]["ETag"].is_object(), + "{method} /blob 412 names the current validator" + ); + for status in [ + "400", "401", "403", "404", "409", "412", "413", "424", "429", "503", + ] { + assert_eq!( + operation["responses"][status]["content"]["application/json"]["schema"]["$ref"], + "#/components/schemas/ErrorOutput", + "{method} /blob {status}" + ); + } + } + let put = &path["put"]; + let body = &put["requestBody"]["content"]["application/octet-stream"]["schema"]; + assert_eq!( + (&body["type"], &body["format"]), + (&"string".into(), &"binary".into()) + ); + assert!(put["responses"]["200"]["headers"]["ETag"].is_object()); + for status in ["408", "415"] { + assert!(put["responses"][status].is_object(), "PUT /blob {status}"); + } + + let receipt = &doc["components"]["schemas"]["BlobWriteOutput"]; + let mut fields = receipt["properties"] + .as_object() + .unwrap() + .keys() + .cloned() + .collect::>(); + fields.sort(); + assert_eq!( + fields, + [ + "actor_id", "branch", "commit", "etag", "kind", "selector", "size" + ], + "the receipt carries graph vocabulary only" + ); + assert_commit_field(&doc, "BlobWriteOutput", true); + assert_eq!( + doc["components"]["schemas"]["BlobWriteStateOutput"]["enum"], + serde_json::json!(["managed", "null"]) + ); + let error = &doc["components"]["schemas"]["ErrorOutput"]["properties"]; + assert!(error["precondition_failure"].is_object()); + let detail = error["blob_precondition_failure"].to_string(); + assert!( + detail.contains("#/components/schemas/BlobPreconditionFailureOutput"), + "{detail}" + ); +} + #[test] fn openapi_blob_documents_binary_redirect_conditional_and_range_contracts() { let doc = openapi_json(); diff --git a/crates/omnigraph/src/blob.rs b/crates/omnigraph/src/blob.rs index 2978bb9e7..476f43e19 100644 --- a/crates/omnigraph/src/blob.rs +++ b/crates/omnigraph/src/blob.rs @@ -44,6 +44,14 @@ const EXTERNAL_BLOB_URI_BYTES_RESOURCE: &str = "external Blob URI bytes"; pub const BLOB_READ_RANGE_MAX_BYTES: u64 = 4 * 1024 * 1024; const BLOB_READ_RANGE_BYTES_RESOURCE: &str = "Blob read range bytes"; +/// Maximum bytes one Blob put stores, inclusive. A transport that collects a +/// put's body refuses a larger one before reading past it; the engine refuses +/// it again before capturing anything, under the resource name +/// [`BLOB_WRITE_PAYLOAD_RESOURCE`]. +pub const BLOB_WRITE_MAX_BYTES: u64 = crate::storage_layer::KEYED_BLOB_PAYLOAD_MAX_BYTES; +/// The resource a refused oversized Blob put names. +pub const BLOB_WRITE_PAYLOAD_RESOURCE: &str = "Blob write payload bytes"; + /// One logical Blob cell addressed at graph level. #[derive(Debug, Clone, PartialEq, Eq)] pub struct BlobCell { diff --git a/crates/omnigraph/src/exec/blob_write.rs b/crates/omnigraph/src/exec/blob_write.rs index 2021b73b1..520cebf51 100644 --- a/crates/omnigraph/src/exec/blob_write.rs +++ b/crates/omnigraph/src/exec/blob_write.rs @@ -20,15 +20,16 @@ use super::mutation::{ }; use super::staging::{MutationStaging, PendingMode}; use crate::blob::{ - BlobCell, BlobDescriptor, BlobEtag, BlobPrecondition, BlobWriteOutcome, entity_label, - locate_blob_cell, managed_blob_etag, resolve_blob_cell, + BLOB_WRITE_MAX_BYTES, BLOB_WRITE_PAYLOAD_RESOURCE, BlobCell, BlobDescriptor, BlobEtag, + BlobPrecondition, BlobWriteOutcome, entity_label, locate_blob_cell, managed_blob_etag, + resolve_blob_cell, }; use crate::changes::EntityKind; use crate::db::Omnigraph; use crate::db::manifest::HistoryReleaseBytes; use crate::error::{OmniError, Result}; use crate::session::Session; -use crate::storage_layer::{KEYED_BLOB_PAYLOAD_MAX_BYTES, PendingScanBudget}; +use crate::storage_layer::PendingScanBudget; /// What one Blob cell write stores. enum BlobCellValue { @@ -78,10 +79,10 @@ impl Session { ) -> Result { let length = u64::try_from(bytes.len()) .map_err(|_| OmniError::manifest_internal("Blob write payload length exceeds u64"))?; - if length > KEYED_BLOB_PAYLOAD_MAX_BYTES { + if length > BLOB_WRITE_MAX_BYTES { return Err(OmniError::resource_limit( - "Blob write payload bytes", - KEYED_BLOB_PAYLOAD_MAX_BYTES, + BLOB_WRITE_PAYLOAD_RESOURCE, + BLOB_WRITE_MAX_BYTES, length, )); } diff --git a/crates/omnigraph/src/lib.rs b/crates/omnigraph/src/lib.rs index 9ed1bb5d9..a36150bc9 100644 --- a/crates/omnigraph/src/lib.rs +++ b/crates/omnigraph/src/lib.rs @@ -47,10 +47,10 @@ pub(crate) mod table_store; pub(crate) mod validate; pub use blob::{ - BLOB_READ_RANGE_MAX_BYTES, BlobCell, BlobContent, BlobEtag, BlobPrecondition, BlobRead, - BlobReader, BlobWriteOutcome, EXTERNAL_BLOB_URI_MAX_BYTES, ExternalBlobBase, - ExternalBlobExecutionScope, ExternalBlobPolicy, ExternalBlobRef, RangedExternalBlob, - StorageRootConflict, + BLOB_READ_RANGE_MAX_BYTES, BLOB_WRITE_MAX_BYTES, BLOB_WRITE_PAYLOAD_RESOURCE, BlobCell, + BlobContent, BlobEtag, BlobPrecondition, BlobRead, BlobReader, BlobWriteOutcome, + EXTERNAL_BLOB_URI_MAX_BYTES, ExternalBlobBase, ExternalBlobExecutionScope, ExternalBlobPolicy, + ExternalBlobRef, RangedExternalBlob, StorageRootConflict, }; pub use changes::EntityKind; pub use omnigraph_compiler::settings; diff --git a/docs/dev/blob.md b/docs/dev/blob.md index 80b62867a..6c0d05262 100644 --- a/docs/dev/blob.md +++ b/docs/dev/blob.md @@ -2,7 +2,7 @@ OmniGraph treats a Blob as one property cell on an existing node or edge. Lance owns the Blob-v2 physical placement; OmniGraph owns logical identity, snapshot selection, authorization, external-source admission, bounded delivery, and graph-level publication. -This page describes the implemented read, ingestion and embedded cell-write behavior. The HTTP and CLI put and clear surfaces proposed by [RFC 0033](../rfcs/0033-blob-management.md) are not implemented yet. +This page describes the implemented read, ingestion and cell-write behavior. The CLI put and clear commands proposed by [RFC 0033](../rfcs/0033-blob-management.md) are not implemented yet. ## Logical states @@ -84,7 +84,7 @@ An external value produces a `302` redirect with the stored URI; the server neve The CLI exposes `blob get` and `blob stat` for embedded and remote graphs. `get` streams managed bytes to stdout or `--out`; `stat` returns kind, resolved-view metadata, size/ETag for managed data, or the descriptor for external data. The CLI refuses to follow external references. -There is currently no HTTP or CLI Blob put/clear surface; the embedded writes above are the only cell writes. +`PUT` and `DELETE /graphs/{graph_id}/blob` call the engine cell writes. `PUT` is a raw ingress route like `/load/ndjson`: the ingress middleware reserves the put limit (`BLOB_WRITE_MAX_BYTES`, the engine's own constant) without collecting the body, and the handler authorizes `change` on the branch, checks `Content-Type` (415) and a declared `Content-Length` (413) before it polls a byte, then collects the body under the shared body deadline (408) into one buffer sized from that length, which the engine's value column adopts. The ingress reservation shrinks to the received size. Both verbs admit the actor's workload and submit the engine call through `owned_write`, so a disconnect after admission loses only the response. `omnigraph_api_types::parse_blob_if_match` turns `If-Match` field lines into a `BlobPrecondition` (`*` alone, or the strong tags of a list; weak tags dropped; malformed fields refused), and `blob_write_output` maps the engine outcome to the receipt; the CLI's embedded writes will share both. `BlobWritePreconditionFailed` maps to 412 with `ErrorCode::Conflict`, the `blob_precondition_failure` detail and an `ETag` header, never the graph-commit `precondition_failure`. The CLI has no put or clear yet. ## Maintenance and export @@ -101,7 +101,7 @@ Export emits managed values as base64 and whole-object external values as URI de - Base and storage-root disjointness: in-source `blob.rs` tests (root forms and S3/local overlap), `end_to_end.rs::external_blob_policy_refuses_base_overlapping_graph_root`, and the `omnigraph-cluster` tests `external_blob_config_rejects_bases_overlapping_storage_root`, `external_blob_base_overlapping_storage_root_refuses_apply_over_existing_state`, `external_blob_config_reports_uncomparable_storage_root_under_its_own_code`, `serving_quarantines_applied_policies_overlapping_storage_root` (the classifier), `serving_snapshot_quarantines_graph_whose_applied_base_overlaps_storage_root` (the snapshot reader: quarantine with a healthy sibling, refusal when none is left, ledger unchanged), and `serving_snapshot_refuses_overlapping_policy_its_digest_does_not_bind`. - Lance compatibility: `crates/omnigraph/tests/lance_surface_guards.rs`. - Cluster policy persistence and serving projection: `omnigraph-cluster` in-source tests. -- HTTP transport: `crates/omnigraph-server/tests/data_routes.rs`, `auth_policy.rs`, and `openapi.rs`. +- HTTP transport: `crates/omnigraph-server/tests/data_routes.rs`, `auth_policy.rs`, and `openapi.rs`. The writes: `data_routes.rs::blob_put_and_delete_return_exact_receipts_and_blob_preconditions`, `blob_put_raw_body_bounds_media_type_length_and_deadline`, the Blob doors of `disconnected_writes_keep_admission_until_the_original_operation_finishes`, the `change` cases of `auth_policy.rs::policy_blocks_change_on_protected_main_but_allows_unprotected_branch`, and the `If-Match` parser's tests in `omnigraph-api-types`. - CLI and embedded/remote parity: `crates/omnigraph-cli/tests/cli_data.rs` and `parity_matrix.rs`. The user contract and examples live in [Blob values](../user/blobs.md). diff --git a/docs/user/blobs.md b/docs/user/blobs.md index bf5c19b91..ad689f017 100644 --- a/docs/user/blobs.md +++ b/docs/user/blobs.md @@ -62,28 +62,41 @@ OmniGraph never deletes the object named by an external reference. ### Replacing one Blob value -An embedded `Session` replaces or clears one Blob cell of an existing node or -edge, addressed by exact id like a Blob read: - -- `Session::put_blob_at_as` stores the given bytes as a managed value and - returns its length, its ETag and the commit; -- `Session::clear_blob_at_as` sets a nullable cell to null. Clearing a cell that - is already null publishes nothing and returns no commit. - -Each call that changes the cell is one graph commit and needs the `change` -action on the branch. Neither call inserts a row: a missing entity is -`NotFound`. Clearing a property that is not nullable is refused. The write never -reads the cell's old value. Lance replaces whole rows, so the row's other cells +One Blob cell of an existing node or edge, addressed by exact id like a Blob +read, can be replaced or cleared without a load or a mutation. Over HTTP, +`PUT` stores the raw request body as a managed value and `DELETE` clears the +cell: + +```http +PUT /graphs/knowledge/blob?entity=node&type=Document&id=manual&property=content&branch=main +Omnigraph-Http-Api: 0.13 +Content-Type: application/octet-stream +If-Match: "" + + +``` + +An embedded `Session` does the same with `put_blob_at_as` and +`clear_blob_at_as`. A put returns the stored length, its ETag and the commit. +Clearing a cell that is already null publishes nothing and returns no commit. + +Each write that changes the cell is one graph commit and needs the `change` +action on the branch. Neither verb inserts a row: a missing entity is +`NotFound` (404). Clearing a property that is not nullable is refused (400). +The write never reads the cell's old value. Lance replaces whole rows, so the row's other cells are carried as an `update` carries them: a stored external reference among them needs the external Blob policy to admit its source, and their managed payloads count toward the operation's 32 MiB Blob payload limit together with the new value. -A precondition makes the write conditional on the cell's current value. -`BlobPrecondition::Tags` holds when the current ETag is one of the given tags. -`BlobPrecondition::AnyExisting` holds when the cell is not null. When it fails, -the write returns `BlobWritePreconditionFailed`, which carries the cell's -current ETag when it holds a managed value, and changes nothing. A commit that +A precondition makes the write conditional on the cell's current value: over +HTTP an `If-Match` field, embedded a `BlobPrecondition`. A list of entity tags +(`BlobPrecondition::Tags`) holds when the current ETag is one of them; weak tags +never match. `*` (`BlobPrecondition::AnyExisting`) holds when the cell is not +null. When it fails, the write changes nothing and returns +`BlobWritePreconditionFailed`, over HTTP a `412` whose +`blob_precondition_failure.current_etag` and `ETag` header carry the cell's +current ETag when it holds a managed value. A malformed `If-Match` is a `400`. A commit that lands while the write is in flight makes it start again from the new head and check again, so a stale ETag fails instead of overwriting the newer value; a write without a precondition replaces whatever is current. A schema apply or a @@ -95,8 +108,25 @@ fails after its commit became durable, for example because its acknowledgement was lost, the value is published once; read the cell for its current ETag before retrying with a precondition. -The CLI and the HTTP server do not offer these writes yet. Through them, change -a Blob value with a load or a mutation. +Over HTTP: + +- `PUT` takes a `Content-Type: application/octet-stream` body of at most 32 MiB, + inclusive. Another media type is a `415`. A body over the limit is a `413`, + refused before any of it is read when `Content-Length` declares it, and a body + that does not arrive before the server's body deadline is a `408`; +- both verbs take `branch` (default `main`) and refuse `snapshot` or any other + unknown parameter with a `400`. A missing branch is a `404`; +- `change` is authorized before the body is read; +- success is a `200` with a receipt: `selector`, `branch`, `kind` (`managed` or + `null`), `size` and `etag` for a managed value, `actor_id`, and `commit`, + which is `null` only for a clear of a cell that was already null. A `PUT` also + returns the `ETag` header; +- once the server admits a write it finishes it even if the client + disconnects, so a lost response means the write may have landed: read the + cell before writing again. + +The CLI does not offer these writes yet; through it, change a Blob value with a +load or a mutation. ## Query behavior @@ -106,9 +136,9 @@ assignment, then read an individual Blob value through the dedicated CLI or HTTP surface. There are no `blob put` or `blob clear` CLI commands yet. Use the normal graph -write path, or the embedded writes in -[Replacing one Blob value](#replacing-one-blob-value), so Blob changes remain -part of an atomic graph commit. +write path, or the HTTP and embedded writes in +[Replacing one Blob value](#replacing-one-blob-value); each is an atomic graph +commit. ## CLI reads @@ -194,7 +224,7 @@ and retry. | 32 MiB of decoded `base64:` bytes | One `base64:` value, in a load or in an insert or update mutation | `decoded blob input bytes` | | 32 MiB per touched type, and 32 MiB across all touched types, Blob payloads excluded | Incremental writes: `append` and `merge` loads, inserts, updates and branch merges. Every byte of the rows except managed Blob payloads counts, URIs and Blob framing included | `keyed write bytes for
`, `keyed entity bytes for
`, `retained keyed batch bytes per operation` | | 32 MiB of managed Blob payload per touched type, and 32 MiB across all touched types, inclusive | The same writes. Each Blob value counts its length; external bytes copied in and Blob values carried unchanged by an update count. A single value of exactly 32 MiB fits beside its row | the same names with `Blob payload bytes` in place of `bytes`, for example `keyed entity Blob payload bytes for
` | -| 32 MiB, inclusive | The bytes of one embedded Blob put, checked before the write opens a table | `Blob write payload bytes` | +| 32 MiB, inclusive | The bytes of one Blob put, embedded or the body of an HTTP `PUT`, checked before the write opens a table | `Blob write payload bytes` | | 32 MiB of external payload copied into managed storage | One incremental write operation across all its types, and each type within it: two types copying 20 MiB each exceed it although each fits its per-type limit | `materialized external blob payload bytes` | | 32 MiB of Blob payload | One branch merge that writes rows, across all types, managed and external bytes together | `materialized blob payload bytes` | | 8,192 external references | One write operation or merge | `external Blob reference cells` | @@ -210,7 +240,9 @@ encoded request, so one request carries about 24 MiB of decoded `base64:` data. The NDJSON loader that `omnigraph load` and `/load/ndjson` use caps each encoded line at 32 MiB the same way. The embedded `load` API checks each row's decoded size instead and has no line cap, so it admits a 32 MiB value; over -HTTP, `/load` keeps its 32 MiB body cap. Every HTTP request other than a load (`/load` and `/load/ndjson`) is bounded by the default 1 MiB request body limit, so a `base64:` +HTTP, `/load` keeps its 32 MiB body cap. A Blob `PUT` carries raw bytes, not +`base64:` text, so its 32 MiB body holds a full 32 MiB value. Every other HTTP +request is bounded by the default 1 MiB request body limit, so a `base64:` literal in an HTTP mutation hits that limit first. Values larger than these limits stay readable. The CLI and the HTTP server diff --git a/docs/user/operations/server.md b/docs/user/operations/server.md index f089585b7..ed207ccfd 100644 --- a/docs/user/operations/server.md +++ b/docs/user/operations/server.md @@ -215,7 +215,7 @@ keep their existing routes and do not expose MCP. | `/graphs/{id}/mutate/if-graph-commit` | Run an inline conditional mutation | | `/graphs/{id}/queries` | List and invoke stored queries, including conditional mutations | | `/graphs/{id}/load`, `/load/ndjson` | Bounded batch loading | -| `/graphs/{id}/blob` | GET/HEAD one Blob cell | +| `/graphs/{id}/blob` | GET/HEAD one Blob cell; PUT/DELETE to replace or clear it | | `/graphs/{id}/branches` | Branch management and merge | | `/graphs/{id}/snapshot`, `/commits` | Snapshot (`internal_schema_version`, `graph_manifest_version`, and the datasets of one captured graph version), history, and per-commit changes | | `/graphs/{id}/changes` | Poll a branch feed or establish a baseline | @@ -313,8 +313,11 @@ modes. `GET`/`HEAD /graphs/{id}/blob` select a cell by `entity`, `type`, `id` and `property`. They support managed ranges/ETag conditions and report external -references without fetching them. See [Blob values](../blobs.md) for details -and [Blob limits](../blobs.md#limits). +references without fetching them. `PUT` replaces the cell with a raw +`application/octet-stream` body of at most 32 MiB and `DELETE` clears it, each +as one graph commit under `If-Match`; see +[Replacing one Blob value](../blobs.md#replacing-one-blob-value). See +[Blob values](../blobs.md) for details and [Blob limits](../blobs.md#limits). ## Changes and baselines diff --git a/docs/user/operations/troubleshooting.md b/docs/user/operations/troubleshooting.md index 1c14fe26a..9b555a0a8 100644 --- a/docs/user/operations/troubleshooting.md +++ b/docs/user/operations/troubleshooting.md @@ -27,7 +27,7 @@ means the data request was not sent. HTTP discovery refusals retain their status | 404 | Graph, query, branch, entity, or route is unavailable | Check the name and applied cluster revision; stored-query denials may also appear as 404 | | 409 | Concurrent change, duplicate ID, merge conflict, existing resource, or incompatible full-text index | Inspect structured details; not every conflict is retryable | | 410 | Required change-feed history was reclaimed | Capture and durably install a new baseline, then resume from its terminal cursor | -| 412 | Blob entity-tag or graph-commit precondition failed | Refresh the Blob ETag, or re-read the branch and retry the mutation with its current graph commit | +| 412 | Blob entity-tag or graph-commit precondition failed | Refresh the Blob ETag (a Blob write's `blob_precondition_failure` names the current one), or re-read the branch and retry the mutation with its current graph commit | | 413 | Request or operation exceeded a bounded resource limit | Split or reduce the operation using the reported limit. Blob limits are listed in [Blob limits](../blobs.md#limits) | | 416 | Blob byte range is outside the value | Use the returned length to choose a valid range | | 424 | An allowed external Blob source could not be read | Restore source availability or correct its URI/credentials | diff --git a/openapi.json b/openapi.json index 254770dc4..c3010ee6b 100644 --- a/openapi.json +++ b/openapi.json @@ -1554,6 +1554,743 @@ } ] }, + "put": { + "tags": [ + "blobs" + ], + "summary": "Replace one Blob value of an existing node or edge with the request body.", + "description": "The cell's old value is never read; the row's other cells are carried\nunchanged. Authorization of `change` on the branch runs before the body is\nread. Once admitted the write is owned by the server: a disconnect loses\nonly the response, never cancels or replays the write.", + "operationId": "cluster_putBlob", + "parameters": [ + { + "name": "graph_id", + "in": "path", + "description": "Graph id to route the request to.", + "required": true, + "schema": { + "type": "string" + } + }, + { + "name": "entity", + "in": "query", + "description": "Select a logical node or edge cell.", + "required": true, + "schema": { + "$ref": "#/components/schemas/BlobEntityKind" + } + }, + { + "name": "type", + "in": "query", + "description": "Accepted-schema node or edge type name.", + "required": true, + "schema": { + "type": "string" + } + }, + { + "name": "id", + "in": "query", + "description": "Logical id of an existing entity within the selected type.", + "required": true, + "schema": { + "type": "string" + } + }, + { + "name": "property", + "in": "query", + "description": "Accepted-schema Blob property name.", + "required": true, + "schema": { + "type": "string" + } + }, + { + "name": "branch", + "in": "query", + "description": "Branch to write. Defaults to `main`.", + "required": false, + "schema": { + "type": "string" + } + }, + { + "name": "If-Match", + "in": "header", + "description": "`*` to require a value in the cell, or a list of strong entity tags one of which must equal the cell's current ETag. Weak tags never match. The write evaluates it against the branch head it applies to.", + "required": false, + "schema": { + "type": [ + "string", + "null" + ] + } + }, + { + "name": "omnigraph-http-api", + "in": "header", + "description": "exactly one omnigraph-http-api header with value 0.13 is required", + "required": true, + "schema": { + "type": "string", + "enum": [ + "0.13" + ] + } + } + ], + "requestBody": { + "description": "The raw bytes to store: at most 33554432 bytes (32 MiB), inclusive.", + "content": { + "application/octet-stream": { + "schema": { + "type": "string", + "format": "binary", + "description": "OpenAPI-only marker for an unstructured octet-stream response body." + } + } + }, + "required": true + }, + "responses": { + "200": { + "description": "The bytes are published in one graph commit", + "headers": { + "ETag": { + "schema": { + "type": "string" + }, + "description": "Strong validator of the stored value; equals `etag` in the body" + }, + "omnigraph-http-api": { + "schema": { + "type": "string", + "enum": [ + "0.13" + ] + }, + "description": "Supported OmniGraph HTTP API contract." + } + }, + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/BlobWriteOutput" + } + } + } + }, + "400": { + "description": "Invalid selector, query parameter, If-Match field or target, a non-Blob property, or a carried external reference the graph's policy denies; api_contract_mismatch when the required API contract header is missing, duplicated or unsupported", + "headers": { + "omnigraph-http-api": { + "schema": { + "type": "string", + "enum": [ + "0.13" + ] + }, + "description": "Supported OmniGraph HTTP API contract." + } + }, + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ErrorOutput" + } + } + } + }, + "401": { + "description": "Unauthorized", + "headers": { + "omnigraph-http-api": { + "schema": { + "type": "string", + "enum": [ + "0.13" + ] + }, + "description": "Supported OmniGraph HTTP API contract." + } + }, + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ErrorOutput" + } + } + } + }, + "403": { + "description": "Forbidden", + "headers": { + "omnigraph-http-api": { + "schema": { + "type": "string", + "enum": [ + "0.13" + ] + }, + "description": "Supported OmniGraph HTTP API contract." + } + }, + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ErrorOutput" + } + } + } + }, + "404": { + "description": "Unknown branch, or no entity with this id", + "headers": { + "omnigraph-http-api": { + "schema": { + "type": "string", + "enum": [ + "0.13" + ] + }, + "description": "Supported OmniGraph HTTP API contract." + } + }, + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ErrorOutput" + } + } + } + }, + "408": { + "description": "The request body did not arrive before the body deadline; Request body deadline exceeded before operation admission", + "headers": { + "omnigraph-http-api": { + "schema": { + "type": "string", + "enum": [ + "0.13" + ] + }, + "description": "Supported OmniGraph HTTP API contract." + } + }, + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ErrorOutput" + } + } + } + }, + "409": { + "description": "Write-authority conflict, or the branch's incarnation or accepted schema changed while the write retried", + "headers": { + "omnigraph-http-api": { + "schema": { + "type": "string", + "enum": [ + "0.13" + ] + }, + "description": "Supported OmniGraph HTTP API contract." + } + }, + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ErrorOutput" + } + } + } + }, + "412": { + "description": "If-Match did not hold for the cell; `blob_precondition_failure` names its current validator", + "headers": { + "ETag": { + "schema": { + "type": "string" + }, + "description": "The cell's current validator, when it holds a managed value" + }, + "omnigraph-http-api": { + "schema": { + "type": "string", + "enum": [ + "0.13" + ] + }, + "description": "Supported OmniGraph HTTP API contract." + } + }, + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ErrorOutput" + } + } + } + }, + "413": { + "description": "The body exceeds 32 MiB, or the row's carried Blob payloads and the new value exceed the write's payload limit; Request body exceeds its route limit before operation admission", + "headers": { + "omnigraph-http-api": { + "schema": { + "type": "string", + "enum": [ + "0.13" + ] + }, + "description": "Supported OmniGraph HTTP API contract." + } + }, + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ErrorOutput" + } + } + } + }, + "415": { + "description": "Content-Type must be application/octet-stream", + "headers": { + "omnigraph-http-api": { + "schema": { + "type": "string", + "enum": [ + "0.13" + ] + }, + "description": "Supported OmniGraph HTTP API contract." + } + }, + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ErrorOutput" + } + } + } + }, + "424": { + "description": "An allowed external Blob source carried from the row could not be read", + "headers": { + "omnigraph-http-api": { + "schema": { + "type": "string", + "enum": [ + "0.13" + ] + }, + "description": "Supported OmniGraph HTTP API contract." + } + }, + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ErrorOutput" + } + } + } + }, + "429": { + "description": "Per-actor admission cap exceeded; honor `Retry-After` header; Server observer, ingress or operation admission capacity exhausted; honor Retry-After", + "headers": { + "Retry-After": { + "schema": { + "type": "string" + }, + "description": "Minimum delay before a caller-bounded retry of this refused request; not a scheduled server retry." + }, + "omnigraph-http-api": { + "schema": { + "type": "string", + "enum": [ + "0.13" + ] + }, + "description": "Supported OmniGraph HTTP API contract." + } + }, + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ErrorOutput" + } + } + } + }, + "503": { + "description": "Write admission is closed or an overlapping durable recovery intent must be resolved before retry; Known graph unavailable (graph_unavailable) or server operation admission closed; reconcile any earlier write before retrying", + "headers": { + "omnigraph-http-api": { + "schema": { + "type": "string", + "enum": [ + "0.13" + ] + }, + "description": "Supported OmniGraph HTTP API contract." + } + }, + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ErrorOutput" + } + } + } + } + }, + "security": [ + { + "bearer_token": [] + } + ] + }, + "delete": { + "tags": [ + "blobs" + ], + "summary": "Set one nullable Blob value of an existing node or edge to null.", + "description": "Clearing a cell that is already null publishes nothing. Like a put, the\nclear is owned by the server once admitted.", + "operationId": "cluster_clearBlob", + "parameters": [ + { + "name": "graph_id", + "in": "path", + "description": "Graph id to route the request to.", + "required": true, + "schema": { + "type": "string" + } + }, + { + "name": "entity", + "in": "query", + "description": "Select a logical node or edge cell.", + "required": true, + "schema": { + "$ref": "#/components/schemas/BlobEntityKind" + } + }, + { + "name": "type", + "in": "query", + "description": "Accepted-schema node or edge type name.", + "required": true, + "schema": { + "type": "string" + } + }, + { + "name": "id", + "in": "query", + "description": "Logical id of an existing entity within the selected type.", + "required": true, + "schema": { + "type": "string" + } + }, + { + "name": "property", + "in": "query", + "description": "Accepted-schema Blob property name.", + "required": true, + "schema": { + "type": "string" + } + }, + { + "name": "branch", + "in": "query", + "description": "Branch to write. Defaults to `main`.", + "required": false, + "schema": { + "type": "string" + } + }, + { + "name": "If-Match", + "in": "header", + "description": "`*` to require a value in the cell, or a list of strong entity tags one of which must equal the cell's current ETag. A null cell satisfies neither form.", + "required": false, + "schema": { + "type": [ + "string", + "null" + ] + } + }, + { + "name": "omnigraph-http-api", + "in": "header", + "description": "exactly one omnigraph-http-api header with value 0.13 is required", + "required": true, + "schema": { + "type": "string", + "enum": [ + "0.13" + ] + } + } + ], + "responses": { + "200": { + "description": "The cell is null. `commit` is the clear's publication, or `null` when the cell already was null and nothing was published", + "headers": { + "omnigraph-http-api": { + "schema": { + "type": "string", + "enum": [ + "0.13" + ] + }, + "description": "Supported OmniGraph HTTP API contract." + } + }, + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/BlobWriteOutput" + } + } + } + }, + "400": { + "description": "Invalid selector, query parameter, If-Match field or target, a non-Blob or non-nullable property, or a carried external reference the graph's policy denies; api_contract_mismatch when the required API contract header is missing, duplicated or unsupported", + "headers": { + "omnigraph-http-api": { + "schema": { + "type": "string", + "enum": [ + "0.13" + ] + }, + "description": "Supported OmniGraph HTTP API contract." + } + }, + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ErrorOutput" + } + } + } + }, + "401": { + "description": "Unauthorized", + "headers": { + "omnigraph-http-api": { + "schema": { + "type": "string", + "enum": [ + "0.13" + ] + }, + "description": "Supported OmniGraph HTTP API contract." + } + }, + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ErrorOutput" + } + } + } + }, + "403": { + "description": "Forbidden", + "headers": { + "omnigraph-http-api": { + "schema": { + "type": "string", + "enum": [ + "0.13" + ] + }, + "description": "Supported OmniGraph HTTP API contract." + } + }, + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ErrorOutput" + } + } + } + }, + "404": { + "description": "Unknown branch, or no entity with this id", + "headers": { + "omnigraph-http-api": { + "schema": { + "type": "string", + "enum": [ + "0.13" + ] + }, + "description": "Supported OmniGraph HTTP API contract." + } + }, + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ErrorOutput" + } + } + } + }, + "409": { + "description": "Write-authority conflict, or the branch's incarnation or accepted schema changed while the write retried", + "headers": { + "omnigraph-http-api": { + "schema": { + "type": "string", + "enum": [ + "0.13" + ] + }, + "description": "Supported OmniGraph HTTP API contract." + } + }, + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ErrorOutput" + } + } + } + }, + "412": { + "description": "If-Match did not hold for the cell; `blob_precondition_failure` names its current validator", + "headers": { + "ETag": { + "schema": { + "type": "string" + }, + "description": "The cell's current validator, when it holds a managed value" + }, + "omnigraph-http-api": { + "schema": { + "type": "string", + "enum": [ + "0.13" + ] + }, + "description": "Supported OmniGraph HTTP API contract." + } + }, + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ErrorOutput" + } + } + } + }, + "413": { + "description": "The row's carried Blob payloads exceed the write's payload limit", + "headers": { + "omnigraph-http-api": { + "schema": { + "type": "string", + "enum": [ + "0.13" + ] + }, + "description": "Supported OmniGraph HTTP API contract." + } + }, + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ErrorOutput" + } + } + } + }, + "424": { + "description": "An allowed external Blob source carried from the row could not be read", + "headers": { + "omnigraph-http-api": { + "schema": { + "type": "string", + "enum": [ + "0.13" + ] + }, + "description": "Supported OmniGraph HTTP API contract." + } + }, + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ErrorOutput" + } + } + } + }, + "429": { + "description": "Per-actor admission cap exceeded; honor `Retry-After` header; Server observer, ingress or operation admission capacity exhausted; honor Retry-After", + "headers": { + "Retry-After": { + "schema": { + "type": "string" + }, + "description": "Minimum delay before a caller-bounded retry of this refused request; not a scheduled server retry." + }, + "omnigraph-http-api": { + "schema": { + "type": "string", + "enum": [ + "0.13" + ] + }, + "description": "Supported OmniGraph HTTP API contract." + } + }, + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ErrorOutput" + } + } + } + }, + "503": { + "description": "Write admission is closed or an overlapping durable recovery intent must be resolved before retry; Known graph unavailable (graph_unavailable) or server operation admission closed; reconcile any earlier write before retrying", + "headers": { + "omnigraph-http-api": { + "schema": { + "type": "string", + "enum": [ + "0.13" + ] + }, + "description": "Supported OmniGraph HTTP API contract." + } + }, + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ErrorOutput" + } + } + } + } + }, + "security": [ + { + "bearer_token": [] + } + ] + }, "head": { "tags": [ "blobs" @@ -7245,6 +7982,18 @@ "edge" ] }, + "BlobPreconditionFailureOutput": { + "type": "object", + "description": "A Blob write's `If-Match` did not hold for the cell (HTTP 412). The write\nhad no effect. `current_etag` is the cell's validator when it holds a\nmanaged value; it is absent for a null or external value, which no tag\nmatches. Distinct from [`PreconditionFailureOutput`], the graph-commit\nprecondition.", + "properties": { + "current_etag": { + "type": [ + "string", + "null" + ] + } + } + }, "BlobRangeOutput": { "type": "object", "description": "Normalized half-open range details for an unsatisfiable managed Blob read.\n\nHTTP also returns `Content-Range: bytes */N`; these fields let SDKs inspect\nthe failure without parsing either that header or the human-readable text.", @@ -7271,6 +8020,91 @@ } } }, + "BlobSelectorOutput": { + "type": "object", + "description": "One logical graph Blob cell.", + "required": [ + "entity", + "type", + "id", + "property" + ], + "properties": { + "entity": { + "$ref": "#/components/schemas/BlobEntityKind" + }, + "id": { + "type": "string" + }, + "property": { + "type": "string" + }, + "type": { + "type": "string" + } + } + }, + "BlobWriteOutput": { + "type": "object", + "description": "Receipt of a Blob put or clear.\n\nA put leaves managed bytes, with their `size` and `etag`; a clear leaves\nnull, with neither. The `etag` is the one a read at `commit` reports.", + "required": [ + "selector", + "branch", + "kind", + "commit" + ], + "properties": { + "actor_id": { + "type": [ + "string", + "null" + ] + }, + "branch": { + "type": "string", + "description": "The branch the write published to." + }, + "commit": { + "oneOf": [ + { + "type": "null" + }, + { + "$ref": "#/components/schemas/CommitOutput", + "description": "The write's own publication. Always present on the wire; `null` only\nfor a clear of a cell that was already null, which publishes nothing." + } + ] + }, + "etag": { + "type": [ + "string", + "null" + ] + }, + "kind": { + "$ref": "#/components/schemas/BlobWriteStateOutput" + }, + "selector": { + "$ref": "#/components/schemas/BlobSelectorOutput" + }, + "size": { + "type": [ + "integer", + "null" + ], + "format": "int64", + "minimum": 0 + } + } + }, + "BlobWriteStateOutput": { + "type": "string", + "description": "The state a Blob write left its cell in. Separate from\n[`BlobContentKindOutput`]: a write can leave the cell null, a stat never\ndescribes one.", + "enum": [ + "managed", + "null" + ] + }, "BranchCreateOutput": { "type": "object", "required": [ @@ -8228,6 +9062,17 @@ "error" ], "properties": { + "blob_precondition_failure": { + "oneOf": [ + { + "type": "null" + }, + { + "$ref": "#/components/schemas/BlobPreconditionFailureOutput", + "description": "Set with HTTP 412 when a Blob write's `If-Match` did not hold for the\ncell. The `ETag` response header repeats `current_etag` when present." + } + ] + }, "blob_range": { "oneOf": [ { From bbbd4e3c6bb0716828cf76219e0cff44e9927dbe Mon Sep 17 00:00:00 2001 From: Ragnor Comerford Date: Fri, 9 Oct 2026 11:35:34 -0700 Subject: [PATCH 05/14] fix(loader): forecast Blob payload by the declared property type The compatibility loader's pre-decode forecast charged every `base64:` string as Blob payload, so a declared String whose text starts with `base64:` used the payload ceiling: a 24 MiB Blob beside such a 12 MiB String was refused at 33 MiB although the batch check admits it. The forecast now takes the type's Blob properties, as the Arrow check does: only a Blob property's `base64:` value is payload; every other value is framing. --- crates/omnigraph/src/loader/mod.rs | 84 +++++++++++++++++++++++------- crates/omnigraph/tests/writes.rs | 54 +++++++++++++++++-- 2 files changed, 113 insertions(+), 25 deletions(-) diff --git a/crates/omnigraph/src/loader/mod.rs b/crates/omnigraph/src/loader/mod.rs index 1a6f8734f..9798c1842 100644 --- a/crates/omnigraph/src/loader/mod.rs +++ b/crates/omnigraph/src/loader/mod.rs @@ -1,4 +1,4 @@ -use std::collections::HashMap; +use std::collections::{HashMap, HashSet}; use std::fmt; use std::io::{BufRead, BufReader, Cursor}; use std::sync::Arc; @@ -665,6 +665,7 @@ async fn load_jsonl_reader_once( account_keyed_json_row( &format!("node:{type_name}"), &data, + &catalog.node_types[&type_name].blob_properties, 0, &mut keyed_input_budget, )?; @@ -716,6 +717,7 @@ async fn load_jsonl_reader_once( account_keyed_json_row( &format!("edge:{canonical}"), &data, + &edge_type.blob_properties, from.len().saturating_add(to.len()), &mut keyed_input_budget, )?; @@ -1251,7 +1253,13 @@ fn parse_strict_graph_rows( let table_key = format!("node:{type_name}"); let row = JsonValue::Object(data); if bounded_keyed_input { - account_keyed_json_row(&table_key, &row, 0, keyed_input_budget)?; + account_keyed_json_row( + &table_key, + &row, + &catalog.node_types[&type_name].blob_properties, + 0, + keyed_input_budget, + )?; } rows.nodes.entry(type_name).or_default().push(row); } @@ -1291,6 +1299,7 @@ fn parse_strict_graph_rows( account_keyed_json_row( &table_key, &JsonValue::Object(data.clone()), + &edge_type.blob_properties, from.len().saturating_add(to.len()), keyed_input_budget, )?; @@ -1447,12 +1456,15 @@ struct KeyedTableInput { /// Charge a keyed JSON record before the parse spool retains it: a lower bound /// on its Arrow bytes per table and across tables, taken before base64 is -/// decoded, split like the later batch check (`KeyedBytes`): a `base64:` -/// value's decoded length as payload, the rest as framing. Not a JSON DOM -/// bound; `MutationStaging::append_batch` is the authority. +/// decoded, split like the later batch check (`KeyedBytes`) by the type's +/// declared Blob properties: a `base64:` value of a Blob property is payload by its +/// decoded length, and every other value, a `String` that happens to start +/// with `base64:` included, is framing. Not a JSON DOM bound; +/// `MutationStaging::append_batch` is the authority. fn account_keyed_json_row( table_key: &str, data: &JsonValue, + blob_properties: &HashSet, structural_string_bytes: usize, budgets: &mut KeyedInputBudget, ) -> Result<()> { @@ -1470,8 +1482,18 @@ fn account_keyed_json_row( } let structural = u64::try_from(structural_string_bytes) .map_err(|_| OmniError::manifest_internal("keyed string bytes exceed u64"))?; - let row_bytes = - estimate_json_keyed_bytes(data)?.checked_add(KeyedBytes::framing(structural))?; + let fields = match data { + JsonValue::Object(fields) => { + fields + .iter() + .try_fold(KeyedBytes::default(), |bytes, (name, value)| { + let blob = blob_properties.contains(name); + bytes.checked_add(estimate_json_keyed_bytes(value, blob)?) + })? + } + other => estimate_json_keyed_bytes(other, false)?, + }; + let row_bytes = fields.checked_add(KeyedBytes::framing(structural))?; entry.bytes = entry .bytes .checked_add(row_bytes)? @@ -1483,17 +1505,19 @@ fn account_keyed_json_row( Ok(()) } +/// A lower bound on the Arrow bytes of a Blob property's value. fn estimate_json_arrow_bytes(value: &JsonValue) -> Result { - let bytes = estimate_json_keyed_bytes(value)?; + let bytes = estimate_json_keyed_bytes(value, true)?; bytes .framing .checked_add(bytes.payload) .ok_or_else(|| OmniError::manifest_internal("JSON value bytes overflow")) } -/// A lower bound on the Arrow bytes a JSON value becomes. A `base64:` string -/// counts its decoded length as payload; everything else is framing. -fn estimate_json_keyed_bytes(value: &JsonValue) -> Result { +/// A lower bound on the Arrow bytes a JSON value becomes. When `blob` says the +/// value is a Blob property's, a `base64:` string counts its decoded length as +/// payload; everything else, nested values included, is framing. +fn estimate_json_keyed_bytes(value: &JsonValue, blob: bool) -> Result { match value { JsonValue::Null => Ok(KeyedBytes::default()), JsonValue::Bool(_) => Ok(KeyedBytes::framing(1)), @@ -1505,7 +1529,7 @@ fn estimate_json_keyed_bytes(value: &JsonValue) -> Result { u64::try_from(bytes) .map_err(|_| OmniError::manifest_internal("JSON string bytes exceed u64")) }; - match value.strip_prefix("base64:") { + match value.strip_prefix("base64:").filter(|_| blob) { Some(encoded) => Ok(KeyedBytes::payload(string_bytes( base64::decoded_len_estimate(encoded.len()).saturating_sub( encoded @@ -1528,7 +1552,7 @@ fn estimate_json_keyed_bytes(value: &JsonValue) -> Result { values .iter() .try_fold(KeyedBytes::framing(offsets), |bytes, value| { - bytes.checked_add(estimate_json_keyed_bytes(value)?) + bytes.checked_add(estimate_json_keyed_bytes(value, false)?) }) } // Property names are schema, not per-row Arrow payload. Count values @@ -1537,7 +1561,7 @@ fn estimate_json_keyed_bytes(value: &JsonValue) -> Result { JsonValue::Object(values) => values .values() .try_fold(KeyedBytes::default(), |bytes, value| { - bytes.checked_add(estimate_json_keyed_bytes(value)?) + bytes.checked_add(estimate_json_keyed_bytes(value, false)?) }), } } @@ -3962,9 +3986,10 @@ edge WorksAt: Person -> Company ) }; + let blobs = HashSet::from(["content".to_string()]); let row = serde_json::json!({"title": "exact", "content": encode(limit)}); let mut budget = KeyedInputBudget::default(); - account_keyed_json_row("node:Document", &row, 0, &mut budget) + account_keyed_json_row("node:Document", &row, &blobs, 0, &mut budget) .expect("an exact-limit payload fits beside its row"); assert_eq!( budget.bytes, @@ -3974,10 +3999,29 @@ edge WorksAt: Person -> Company } ); + // The declared type decides: the same `base64:` text in a property + // that is not a Blob is framing by its full length. + let note = encode(1024); + let row = serde_json::json!({"title": "typed", "note": note.clone(), "content": encode(8)}); + let mut budget = KeyedInputBudget::default(); + account_keyed_json_row("node:Document", &row, &blobs, 0, &mut budget).unwrap(); + assert_eq!( + budget.bytes, + KeyedBytes { + framing: 5 + note.len() as u64, + payload: 8, + } + ); + let row = serde_json::json!({"title": "over", "content": encode(limit + 1)}); - let error = - account_keyed_json_row("node:Document", &row, 0, &mut KeyedInputBudget::default()) - .expect_err("one decoded byte over must be refused"); + let error = account_keyed_json_row( + "node:Document", + &row, + &blobs, + 0, + &mut KeyedInputBudget::default(), + ) + .expect_err("one decoded byte over must be refused"); assert!(matches!( error, OmniError::ResourceLimitExceeded { @@ -3993,8 +4037,8 @@ edge WorksAt: Person -> Company fn operation_byte_allowances_span_graph_types_and_include_their_ceiling() { let row = serde_json::json!({"payload": "x".repeat(17 * 1024 * 1024)}); let mut budget = KeyedInputBudget::default(); - account_keyed_json_row("node:Person", &row, 0, &mut budget).unwrap(); - let error = account_keyed_json_row("node:Company", &row, 0, &mut budget) + account_keyed_json_row("node:Person", &row, &HashSet::new(), 0, &mut budget).unwrap(); + let error = account_keyed_json_row("node:Company", &row, &HashSet::new(), 0, &mut budget) .expect_err("keyed parse bytes must be aggregated across types"); assert!(matches!(error, OmniError::ResourceLimitExceeded { ref resource, limit: KEYED_WRITE_MAX_BYTES, actual } diff --git a/crates/omnigraph/tests/writes.rs b/crates/omnigraph/tests/writes.rs index 4189a6121..973667185 100644 --- a/crates/omnigraph/tests/writes.rs +++ b/crates/omnigraph/tests/writes.rs @@ -1342,20 +1342,28 @@ query replace_content($c: Blob) { /// A managed Blob value of exactly the payload ceiling fits beside its row: /// payload and framing have separate ceilings, and the compatibility loader's -/// pre-decode forecast splits like the batch check. One more byte is refused -/// before any graph or table effect. The strict NDJSON loader bounds its -/// encoded line instead, so the exact-limit value's 44.7 MB line is refused. -/// Rust, not GQT: 32 MiB payloads are outside the case format. +/// pre-decode forecast splits like the batch check, by the declared property +/// types, so a `String` that starts with `base64:` stays framing. One more +/// byte is refused before any graph or table effect. The strict NDJSON loader +/// bounds its encoded line instead, so the exact-limit value's 44.7 MB line +/// is refused. Rust: this owner reads the stored bytes back, asserts nothing +/// moved on refusal and probes the compatibility loader, which GQT reaches +/// only through generated loads capped at 16 MiB per batch; a GQT case would +/// carry the 44.7 MB parameter as literal text. #[tokio::test] async fn exact_limit_blob_payload_fits_beside_its_row_and_one_more_byte_is_refused() { use base64::Engine; const LIMIT: usize = 32 * 1024 * 1024; - const SCHEMA: &str = "node Document { title: String @key content: Blob? }\n"; + const SCHEMA: &str = "node Document { title: String @key note: String? content: Blob? }\n"; const INSERT: &str = r#" query put_doc($title: String, $c: Blob) { insert Document { title: $title, content: $c } } + +query put_noted($title: String, $note: String, $c: Blob) { + insert Document { title: $title, note: $note, content: $c } +} "#; let encode = |bytes: usize| { format!( @@ -1397,6 +1405,42 @@ query put_doc($title: String, $c: Blob) { assert!(bytes.iter().all(|&byte| byte == 7), "{title}"); } + // A 24 MiB Blob beside a String whose text starts with `base64:` and holds + // 12 MiB more: the String is framing to both the mutation and the + // loader's forecast, so each admits the row. Counting the String's + // decoded 9 MiB as payload would refuse it at 33 MiB. + let blob = encode(24 * 1024 * 1024); + let note = format!("base64:{}", "A".repeat(12 * 1024 * 1024)); + db.mutate( + "main", + INSERT, + "put_noted", + ¶ms(&[("$title", "noted"), ("$note", ¬e), ("$c", &blob)]), + ) + .await + .expect("a String that looks like base64 is framing to a mutation"); + db.load_jsonl( + &serde_json::json!({"type": "Document", "data": { + "title": "noted-load", "note": note, "content": blob}}) + .to_string(), + LoadMode::Merge, + ) + .await + .expect("a String that looks like base64 is framing to the loader's forecast"); + for title in ["noted", "noted-load"] { + assert_eq!( + read_managed_blob_bytes( + &db, + ReadTarget::branch("main"), + node_blob_cell("Document", title, "content"), + ) + .await + .len(), + 24 * 1024 * 1024, + "{title}" + ); + } + let before = snapshot_main(&db).await.unwrap().graph_manifest_version(); let strict = db .load_graph_batch("main", &row("strict", &exact), LoadMode::Merge) From b5e3ea07efcb6631141ce08eaa576d11928f9054 Mon Sep 17 00:00:00 2001 From: Ragnor Comerford Date: Fri, 9 Oct 2026 11:57:35 -0700 Subject: [PATCH 06/14] docs(server): say entity, not row, in the Blob write OpenAPI descriptions The public vocabulary guard classifies every graph-storage term in the served OpenAPI. The five new descriptions now use the public entity vocabulary, so no inventory exception is needed. --- crates/omnigraph-server/src/handlers.rs | 10 +++++----- openapi.json | 10 +++++----- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/crates/omnigraph-server/src/handlers.rs b/crates/omnigraph-server/src/handlers.rs index f9394e1e2..64c6aba29 100644 --- a/crates/omnigraph-server/src/handlers.rs +++ b/crates/omnigraph-server/src/handlers.rs @@ -1004,9 +1004,9 @@ pub(crate) async fn server_blob_head( headers( ("ETag" = String, description = "The cell's current validator, when it holds a managed value"), )), - (status = 413, description = "The body exceeds 32 MiB, or the row's carried Blob payloads and the new value exceed the write's payload limit", body = ErrorOutput), + (status = 413, description = "The body exceeds 32 MiB, or the entity's carried Blob payloads and the new value exceed the write's payload limit", body = ErrorOutput), (status = 415, description = "Content-Type must be application/octet-stream", body = ErrorOutput), - (status = 424, description = "An allowed external Blob source carried from the row could not be read", body = ErrorOutput), + (status = 424, description = "An allowed external Blob source carried from the entity could not be read", body = ErrorOutput), (status = 429, description = "Per-actor admission cap exceeded; honor `Retry-After` header", body = ErrorOutput), (status = 503, description = "Write admission is closed or an overlapping durable recovery intent must be resolved before retry", body = ErrorOutput), ), @@ -1014,7 +1014,7 @@ pub(crate) async fn server_blob_head( )] /// Replace one Blob value of an existing node or edge with the request body. /// -/// The cell's old value is never read; the row's other cells are carried +/// The cell's old value is never read; the entity's other cells are carried /// unchanged. Authorization of `change` on the branch runs before the body is /// read. Once admitted the write is owned by the server: a disconnect loses /// only the response, never cancels or replays the write. @@ -1120,8 +1120,8 @@ pub(crate) async fn server_blob_put( headers( ("ETag" = String, description = "The cell's current validator, when it holds a managed value"), )), - (status = 413, description = "The row's carried Blob payloads exceed the write's payload limit", body = ErrorOutput), - (status = 424, description = "An allowed external Blob source carried from the row could not be read", body = ErrorOutput), + (status = 413, description = "The entity's carried Blob payloads exceed the write's payload limit", body = ErrorOutput), + (status = 424, description = "An allowed external Blob source carried from the entity could not be read", body = ErrorOutput), (status = 429, description = "Per-actor admission cap exceeded; honor `Retry-After` header", body = ErrorOutput), (status = 503, description = "Write admission is closed or an overlapping durable recovery intent must be resolved before retry", body = ErrorOutput), ), diff --git a/openapi.json b/openapi.json index c3010ee6b..e94e09ea9 100644 --- a/openapi.json +++ b/openapi.json @@ -1559,7 +1559,7 @@ "blobs" ], "summary": "Replace one Blob value of an existing node or edge with the request body.", - "description": "The cell's old value is never read; the row's other cells are carried\nunchanged. Authorization of `change` on the branch runs before the body is\nread. Once admitted the write is owned by the server: a disconnect loses\nonly the response, never cancels or replays the write.", + "description": "The cell's old value is never read; the entity's other cells are carried\nunchanged. Authorization of `change` on the branch runs before the body is\nread. Once admitted the write is owned by the server: a disconnect loses\nonly the response, never cancels or replays the write.", "operationId": "cluster_putBlob", "parameters": [ { @@ -1836,7 +1836,7 @@ } }, "413": { - "description": "The body exceeds 32 MiB, or the row's carried Blob payloads and the new value exceed the write's payload limit; Request body exceeds its route limit before operation admission", + "description": "The body exceeds 32 MiB, or the entity's carried Blob payloads and the new value exceed the write's payload limit; Request body exceeds its route limit before operation admission", "headers": { "omnigraph-http-api": { "schema": { @@ -1878,7 +1878,7 @@ } }, "424": { - "description": "An allowed external Blob source carried from the row could not be read", + "description": "An allowed external Blob source carried from the entity could not be read", "headers": { "omnigraph-http-api": { "schema": { @@ -2195,7 +2195,7 @@ } }, "413": { - "description": "The row's carried Blob payloads exceed the write's payload limit", + "description": "The entity's carried Blob payloads exceed the write's payload limit", "headers": { "omnigraph-http-api": { "schema": { @@ -2216,7 +2216,7 @@ } }, "424": { - "description": "An allowed external Blob source carried from the row could not be read", + "description": "An allowed external Blob source carried from the entity could not be read", "headers": { "omnigraph-http-api": { "schema": { From 2d9b84ce28e29cfa1add81e555c5a2587768f52a Mon Sep 17 00:00:00 2001 From: Ragnor Comerford Date: Fri, 9 Oct 2026 11:59:40 -0700 Subject: [PATCH 07/14] docs(server): keep the server guide within its page limit --- docs/user/operations/server.md | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/docs/user/operations/server.md b/docs/user/operations/server.md index 76e0e2d1f..a5577475d 100644 --- a/docs/user/operations/server.md +++ b/docs/user/operations/server.md @@ -324,13 +324,10 @@ modes. ## Deliver Blob values -`GET`/`HEAD /graphs/{id}/blob` select a cell by `entity`, `type`, `id` and -`property`. They support managed ranges/ETag conditions and report external -references without fetching them. `PUT` replaces the cell with a raw -`application/octet-stream` body of at most 32 MiB and `DELETE` clears it, each -as one graph commit under `If-Match`; see -[Replacing one Blob value](../blobs.md#replacing-one-blob-value). See -[Blob values](../blobs.md) for details and [Blob limits](../blobs.md#limits). +`GET`/`HEAD /graphs/{id}/blob` select a cell by `entity`, `type`, `id` and `property`, +support managed ranges/ETag conditions, and report external references without fetching +them. `PUT` and `DELETE` replace or clear the cell; see [Blob values](../blobs.md), +[Replacing one Blob value](../blobs.md#replacing-one-blob-value) and [Blob limits](../blobs.md#limits). ## Changes and baselines From 596c84550a0498f86e5e5053adcadd8ea819c6b9 Mon Sep 17 00:00:00 2001 From: Ragnor Comerford Date: Fri, 9 Oct 2026 13:39:21 -0700 Subject: [PATCH 08/14] feat(cli): blob put and blob clear `omnigraph blob put` stores the raw bytes of --file or stdin in one Blob cell and `omnigraph blob clear` nulls it, on a --store graph through the engine writes or on a served one through PUT and DELETE /blob. Both print the receipt the server returns. A malformed --if-match, input over 32 MiB and an interactive stdin are refused before the graph is addressed. --as names the actor of a --store write. A served Blob 412 is a verified request precondition and exits 4; a write is sent once and never retried. The lost-delivery fault proxy now carries a raw put body past its JSON bound and breaks delivery of a put's or clear's receipt. --- Cargo.lock | 1 + changelog.d/blob-cell-put-clear.added.md | 2 +- changelog.d/blob-cli-put-clear.added.md | 11 ++ crates/omnigraph-cli/Cargo.toml | 1 + crates/omnigraph-cli/src/blob_cli.rs | 108 ++++++++++++- crates/omnigraph-cli/src/cli.rs | 57 ++++++- crates/omnigraph-cli/src/client.rs | 128 ++++++++++++++-- crates/omnigraph-cli/src/command_outcome.rs | 7 +- crates/omnigraph-cli/src/main.rs | 66 ++++++++ crates/omnigraph-cli/src/output.rs | 33 ++++ crates/omnigraph-cli/src/planes.rs | 10 +- crates/omnigraph-cli/tests/cli_data.rs | 144 +++++++++++++++++- crates/omnigraph-cli/tests/parity_matrix.rs | 130 ++++++++++++++++ .../tests/support/managed_http.rs | 41 +++-- crates/omnigraph-cli/tests/support/mod.rs | 19 ++- crates/omnigraph-cli/tests/system_remote.rs | 123 +++++++++++++++ docs/dev/blob.md | 8 +- docs/dev/testing.md | 2 +- docs/user/blobs.md | 20 ++- docs/user/cli/index.md | 11 +- docs/user/cli/reference.md | 10 +- 21 files changed, 873 insertions(+), 59 deletions(-) create mode 100644 changelog.d/blob-cli-put-clear.added.md diff --git a/Cargo.lock b/Cargo.lock index 06829a1d4..9aab1486d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5644,6 +5644,7 @@ dependencies = [ "assert_cmd", "async-trait", "base64 0.22.1", + "bytes", "clap", "color-eyre", "futures", diff --git a/changelog.d/blob-cell-put-clear.added.md b/changelog.d/blob-cell-put-clear.added.md index 9e065980f..b47218f60 100644 --- a/changelog.d/blob-cell-put-clear.added.md +++ b/changelog.d/blob-cell-put-clear.added.md @@ -9,4 +9,4 @@ is the one a read at that commit reports. One put holds at most 32 MiB, inclusive; a larger one fails before any table is opened with resource `Blob write payload bytes`. The HTTP server exposes them as `PUT` and - `DELETE /blob`; the CLI does not expose them yet. + `DELETE /blob`, and the CLI as `blob put` and `blob clear`. diff --git a/changelog.d/blob-cli-put-clear.added.md b/changelog.d/blob-cli-put-clear.added.md new file mode 100644 index 000000000..ae198b4cd --- /dev/null +++ b/changelog.d/blob-cli-put-clear.added.md @@ -0,0 +1,11 @@ +- `omnigraph blob put ENTITY TYPE ID PROPERTY` stores the raw bytes of + `--file PATH` or stdin in one Blob cell, and `omnigraph blob clear` sets a + nullable cell to null, on a `--store` graph or a served one. Both take + `--branch`, `--if-match TAG` (`*` or entity tags) and `--json`, and print the + receipt the server returns: the selector, branch, state, size and ETag of a + managed value, the actor and the exact commit, which is `null` for a clear of + a cell that was already null. Input over 32 MiB and a malformed `--if-match` + are refused before the graph is addressed, and an interactive stdin is + refused rather than waited on. `--as` names the actor of a `--store` write. + A served Blob precondition failure exits 4, as a graph-commit one does; an + embedded one exits 1. A write is sent once and never retried. diff --git a/crates/omnigraph-cli/Cargo.toml b/crates/omnigraph-cli/Cargo.toml index 9e43c28a8..ddf7f93ba 100644 --- a/crates/omnigraph-cli/Cargo.toml +++ b/crates/omnigraph-cli/Cargo.toml @@ -13,6 +13,7 @@ name = "omnigraph" path = "src/main.rs" [dependencies] +bytes = { workspace = true } omnigraph = { package = "omnigraph-engine", path = "../omnigraph", version = "0.13.0" } omnigraph-compiler = { path = "../omnigraph-compiler", version = "0.13.0" } omnigraph-api-types = { path = "../omnigraph-api-types", version = "0.13.0" } diff --git a/crates/omnigraph-cli/src/blob_cli.rs b/crates/omnigraph-cli/src/blob_cli.rs index f290a9e57..b5c793bf2 100644 --- a/crates/omnigraph-cli/src/blob_cli.rs +++ b/crates/omnigraph-cli/src/blob_cli.rs @@ -1,20 +1,20 @@ -//! CLI-only shaping for graph-level Blob reads. +//! CLI-only shaping for graph-level Blob reads and writes. //! //! The engine and server own descriptor interpretation, snapshot fencing, and //! bounded payload reads. This module keeps the command line's selector, //! range, and error vocabulary identical across the embedded and served arms. use std::fs::File; -use std::io::{self, Write}; +use std::io::{self, IsTerminal, Read, Write}; use std::ops::Range; -use std::path::PathBuf; +use std::path::{Path, PathBuf}; use color_eyre::Report; use color_eyre::eyre::{Result, eyre}; use omnigraph::db::{ReadTarget, SnapshotId}; use omnigraph::error::{ManifestErrorKind, OmniError}; -use omnigraph::{BlobCell, EntityKind, ExternalBlobRef}; -use omnigraph_api_types::{BlobEntityKind, BlobReadQuery}; +use omnigraph::{BlobCell, BlobPrecondition, EntityKind, ExternalBlobRef}; +use omnigraph_api_types::{BlobEntityKind, BlobReadQuery, BlobWriteQuery}; use reqwest::StatusCode; use reqwest::header::{CONTENT_LENGTH, ETAG, HeaderMap, HeaderName}; @@ -164,6 +164,104 @@ pub(crate) fn blob_url(base_url: &str, query: &BlobReadQuery) -> Result crate::helpers::remote_url(base_url, &["blob"], ¶ms) } +pub(crate) fn blob_write_query( + entity: BlobEntityKind, + type_name: String, + id: String, + property: String, + branch: Option, +) -> BlobWriteQuery { + BlobWriteQuery { + entity, + r#type: type_name, + id, + property, + branch, + } +} + +pub(crate) fn blob_write_cell(query: &BlobWriteQuery) -> BlobCell { + BlobCell { + entity: match query.entity { + BlobEntityKind::Node => EntityKind::Node, + BlobEntityKind::Edge => EntityKind::Edge, + }, + type_name: query.r#type.clone(), + id: query.id.clone(), + property: query.property.clone(), + } +} + +pub(crate) fn blob_write_url(base_url: &str, query: &BlobWriteQuery) -> Result { + let entity = match query.entity { + BlobEntityKind::Node => "node", + BlobEntityKind::Edge => "edge", + }; + let mut params = vec![ + ("entity", entity), + ("type", query.r#type.as_str()), + ("id", query.id.as_str()), + ("property", query.property.as_str()), + ]; + if let Some(branch) = query.branch.as_deref() { + params.push(("branch", branch)); + } + crate::helpers::remote_url(base_url, &["blob"], ¶ms) +} + +/// Parse `--if-match` with the parser the server applies to the `If-Match` +/// header, so a malformed value is refused before any graph is addressed. +pub(crate) fn parse_if_match(raw: Option<&str>) -> Result> { + omnigraph_api_types::parse_blob_if_match(raw.map(str::as_bytes)).map_err(|error| eyre!(error)) +} + +/// Read `blob put`'s bytes from `--file` or stdin, never both, before any +/// graph is addressed. Input over the put limit is refused with the resource +/// the engine and the server report, without reading past one more byte; +/// an interactive stdin is refused rather than waited on. +pub(crate) fn read_put_input(file: Option<&Path>) -> Result { + let limit = omnigraph::BLOB_WRITE_MAX_BYTES; + let too_large = |actual: u64| -> Report { + OmniError::resource_limit(omnigraph::BLOB_WRITE_PAYLOAD_RESOURCE, limit, actual).into() + }; + let mut buffer = Vec::new(); + match file { + Some(path) => { + let file = File::open(path) + .map_err(|error| eyre!("cannot read --file {}: {error}", path.display()))?; + let length = file + .metadata() + .map_err(|error| eyre!("cannot read --file {}: {error}", path.display()))? + .len(); + if length > limit { + return Err(too_large(length)); + } + buffer.reserve(usize::try_from(length).unwrap_or(0)); + file.take(limit + 1) + .read_to_end(&mut buffer) + .map_err(|error| eyre!("cannot read --file {}: {error}", path.display()))?; + } + None => { + let stdin = io::stdin(); + if stdin.is_terminal() { + return Err(eyre!( + "blob put reads its bytes from stdin; pipe them in or pass --file PATH" + )); + } + stdin + .lock() + .take(limit + 1) + .read_to_end(&mut buffer) + .map_err(|error| eyre!("cannot read stdin: {error}"))?; + } + } + let read = buffer.len() as u64; + if read > limit { + return Err(too_large(read)); + } + Ok(bytes::Bytes::from(buffer)) +} + pub(crate) fn whole_external_uri(reference: &ExternalBlobRef) -> Result<&str> { reference .whole_object_uri() diff --git a/crates/omnigraph-cli/src/cli.rs b/crates/omnigraph-cli/src/cli.rs index e2dee2172..49ac991f0 100644 --- a/crates/omnigraph-cli/src/cli.rs +++ b/crates/omnigraph-cli/src/cli.rs @@ -247,7 +247,7 @@ pub(crate) enum Command { #[arg(long = "type")] type_names: Vec, }, - /// Read one logical node or edge Blob cell. + /// Read, replace or clear one logical node or edge Blob cell. Blob { #[command(subcommand)] command: BlobCommand, @@ -516,6 +516,61 @@ pub(crate) enum BlobCommand { #[arg(long)] json: bool, }, + /// Replace one Blob value of an existing node or edge with raw bytes from + /// a file or stdin, as one graph commit. + Put { + /// Logical entity namespace. + #[arg(value_name = "ENTITY")] + entity: BlobEntityArg, + /// Accepted-schema node or edge type. + #[arg(value_name = "TYPE")] + type_name: String, + /// Logical entity id; the entity must exist. + #[arg(value_name = "ID")] + id: String, + /// Blob property name. + #[arg(value_name = "PROPERTY")] + property: String, + /// Write a named branch (defaults to main). + #[arg(long)] + branch: Option, + /// Read the bytes from this file instead of stdin. + #[arg(long, value_name = "PATH")] + file: Option, + /// Write only while the cell's ETag is one of these entity tags, or, + /// as `*`, while the cell holds any value. + #[arg(long, value_name = "TAG")] + if_match: Option, + /// Emit the receipt as JSON. + #[arg(long)] + json: bool, + }, + /// Set one nullable Blob value of an existing node or edge to null, as one + /// graph commit; a cell that is already null publishes nothing. + Clear { + /// Logical entity namespace. + #[arg(value_name = "ENTITY")] + entity: BlobEntityArg, + /// Accepted-schema node or edge type. + #[arg(value_name = "TYPE")] + type_name: String, + /// Logical entity id; the entity must exist. + #[arg(value_name = "ID")] + id: String, + /// Blob property name. + #[arg(value_name = "PROPERTY")] + property: String, + /// Write a named branch (defaults to main). + #[arg(long)] + branch: Option, + /// Write only while the cell's ETag is one of these entity tags, or, + /// as `*`, while the cell holds any value. + #[arg(long, value_name = "TAG")] + if_match: Option, + /// Emit the receipt as JSON. + #[arg(long)] + json: bool, + }, } #[derive(Debug, Subcommand)] diff --git a/crates/omnigraph-cli/src/client.rs b/crates/omnigraph-cli/src/client.rs index 37f0c63a9..38d5a0236 100644 --- a/crates/omnigraph-cli/src/client.rs +++ b/crates/omnigraph-cli/src/client.rs @@ -30,15 +30,16 @@ use omnigraph::db::{Omnigraph, ReadTarget}; use omnigraph::settings::{SessionSettings, SettingId, SettingValue, Source}; use omnigraph::{BLOB_READ_RANGE_MAX_BYTES, BlobContent, Session}; use omnigraph_api_types::{ - BlobReadQuery, BlobStatOutput, BranchCreateOutput, BranchCreateRequest, BranchDeleteOutput, - BranchListOutput, BranchMergeOutcome, BranchMergeOutput, BranchMergeRequest, - BranchOutcomeOutput, ChangeBaselineOutput, ChangeBaselineRecord, ChangeBaselineRequest, - ChangeFeedOutput, ChangeOpOutput, ChangeOutput, ChangeRequest, CommitChangesOutput, - CommitListOutput, CommitOutput, EntityKindOutput, ExportRequest, GraphBatchLoadOutput, - GraphDiscoveryResponse, GraphListResponse, InvokeStoredQueryRequest, QueryRequest, ReadOutput, - SchemaApplyOutput, SchemaOutput, SettingsRequest, SnapshotOutput, branch_list_read_output, - change_baseline_output, change_feed_output, change_scope, commit_changes_output, commit_output, - read_output, schema_apply_output, show_read_output, snapshot_payload, + BlobReadQuery, BlobStatOutput, BlobWriteOutput, BlobWriteQuery, BranchCreateOutput, + BranchCreateRequest, BranchDeleteOutput, BranchListOutput, BranchMergeOutcome, + BranchMergeOutput, BranchMergeRequest, BranchOutcomeOutput, ChangeBaselineOutput, + ChangeBaselineRecord, ChangeBaselineRequest, ChangeFeedOutput, ChangeOpOutput, ChangeOutput, + ChangeRequest, CommitChangesOutput, CommitListOutput, CommitOutput, EntityKindOutput, + ExportRequest, GraphBatchLoadOutput, GraphDiscoveryResponse, GraphListResponse, + InvokeStoredQueryRequest, QueryRequest, ReadOutput, SchemaApplyOutput, SchemaOutput, + SettingsRequest, SnapshotOutput, branch_list_read_output, change_baseline_output, + change_feed_output, change_scope, commit_changes_output, commit_output, read_output, + schema_apply_output, show_read_output, snapshot_payload, }; use omnigraph_compiler::catalog::Catalog; use omnigraph_compiler::query::ast::BranchWrite; @@ -48,13 +49,14 @@ use reqwest::{Method, StatusCode}; use serde_json::Value; use crate::blob_cli::{ - BlobRangeRequest, blob_cell, blob_read_target, blob_url, external_response_headers, - managed_response_headers, map_embedded_blob_error, remote_blob_error, whole_external_uri, + BlobRangeRequest, blob_cell, blob_read_target, blob_url, blob_write_cell, blob_write_url, + external_response_headers, managed_response_headers, map_embedded_blob_error, + remote_blob_error, whole_external_uri, }; use crate::cli::CliLoadMode; use crate::graph_http::{ApiContractError, GraphHttpClient}; use crate::helpers::{ - apply_bearer_token, apply_server_flag, branch_statement_change_request, + PreconditionFailedCli, apply_bearer_token, apply_server_flag, branch_statement_change_request, branch_statement_query_request, is_remote_uri, precondition_failed_cli, query_params_from_json, remote_json, remote_json_bounded, remote_response_json_bounded, remote_url, resolve_cli_actor, resolve_cli_graph, resolve_remote_bearer_token, resolve_server_flag, select_named_query, @@ -1747,6 +1749,80 @@ impl GraphClient { } } + /// `blob put` (with `bytes`) or `blob clear` (without): one Blob cell + /// write, sent once. The embedded arm calls the engine with the parsed + /// precondition; the served arm sends `if_match` as the `If-Match` header + /// the server parses the same way. Both answer the same receipt. + pub(crate) async fn blob_write( + &self, + query: &BlobWriteQuery, + bytes: Option, + if_match: Option<&str>, + precondition: Option, + ) -> Result { + let branch = query.branch.as_deref().unwrap_or("main"); + match self { + GraphClient::Embedded { uri, actor } => { + let session = Self::open_write_session(uri, &[]).await?; + let cell = blob_write_cell(query); + let outcome = match bytes { + Some(bytes) => { + session + .put_blob_at_as(branch, cell, bytes, precondition, actor.as_deref()) + .await + } + None => { + session + .clear_blob_at_as(branch, cell, precondition, actor.as_deref()) + .await + } + } + .map_err(|error| match error { + // The body a server answers, so `--json` is transport-uniform. + error @ omnigraph::error::OmniError::BlobWritePreconditionFailed { .. } => { + PreconditionFailedCli { + output: omnigraph_server::engine_error_output(error), + http_status: None, + retry_after: None, + } + .into() + } + other => color_eyre::eyre::Report::from(other), + })?; + Ok(omnigraph_api_types::blob_write_output( + query.into(), + branch, + &outcome, + actor.clone(), + )) + } + GraphClient::Remote { + http, + base_url, + token, + response_limit, + } => { + let url = blob_write_url(base_url, query)?; + let request = match bytes { + Some(bytes) => http + .request(Method::PUT, url) + .header(reqwest::header::CONTENT_TYPE, "application/octet-stream") + .body(bytes), + None => http.request(Method::DELETE, url), + }; + let request = apply_bearer_token(request, token.as_deref()); + let request = match if_match { + Some(tags) => request.header(reqwest::header::IF_MATCH, tags), + None => request, + }; + let response = http.send(request).await?; + remote_response_json_bounded(response, token.as_deref(), *response_limit) + .await + .map_err(verified_blob_precondition) + } + } + } + /// `graphs list` — enumerate the graphs a multi-graph server serves /// (`GET /graphs`). Reached only through registry-addressed clients /// (`resolve_registry` / the D7 probe's `registry_client`), which always @@ -1845,6 +1921,34 @@ fn validate_content_range( Ok(()) } +/// A served Blob write's 412 whose only detail is `blob_precondition_failure` +/// is a verified request precondition, treated as a graph-commit one is: the +/// whole command exits 4 when it was the command's only effectful request. +fn verified_blob_precondition(error: color_eyre::eyre::Report) -> color_eyre::eyre::Report { + match error.downcast::() { + Ok(remote) + if remote.status == StatusCode::PRECONDITION_FAILED + && remote.output.blob_precondition_failure.is_some() + && remote.output.code == Some(omnigraph_api_types::ErrorCode::Conflict) + && crate::command_outcome::single_request() + && { + let mut rest = remote.output.clone(); + rest.blob_precondition_failure = None; + crate::command_outcome::is_plain_refusal(&rest) + } => + { + PreconditionFailedCli { + output: remote.output, + http_status: Some(StatusCode::PRECONDITION_FAILED.as_u16()), + retry_after: remote.retry_after, + } + .into() + } + Ok(remote) => remote.into(), + Err(error) => error, + } +} + /// Shared spelling of the change-surface filters across the three verbs; one /// translation (`change_scope`) is used by the embedded arm and the server. pub(crate) struct ChangeFilterArgs<'a> { diff --git a/crates/omnigraph-cli/src/command_outcome.rs b/crates/omnigraph-cli/src/command_outcome.rs index 9ff9754ee..ea7e00164 100644 --- a/crates/omnigraph-cli/src/command_outcome.rs +++ b/crates/omnigraph-cli/src/command_outcome.rs @@ -8,7 +8,7 @@ use omnigraph_api_types::{ErrorCode, ErrorOutput}; use reqwest::StatusCode; use serde::Serialize; -use crate::cli::{BranchCommand, Cli, Command, SchemaCommand}; +use crate::cli::{BlobCommand, BranchCommand, Cli, Command, SchemaCommand}; use crate::graph_http::ApiContractError; use crate::helpers::{PreconditionFailedCli, RemoteErrorCli}; @@ -38,6 +38,9 @@ pub(crate) fn applies(cli: &Cli) -> bool { | Command::Schema { command: SchemaCommand::Apply { .. } } + | Command::Blob { + command: BlobCommand::Put { .. } | BlobCommand::Clear { .. } + } ) } @@ -212,6 +215,7 @@ impl Failure { Action::Recover } else if not_started || output.precondition_failure.is_some() + || output.blob_precondition_failure.is_some() || output.read_set_conflict.is_some() || output.published_dataset_version_conflict.is_some() || output.key_conflict.is_some() @@ -263,6 +267,7 @@ pub(crate) fn is_plain_refusal(output: &ErrorOutput) -> bool { && output.external_blob_source.is_none() && output.recovery_required.is_none() && output.precondition_failure.is_none() + && output.blob_precondition_failure.is_none() && output.change_feed_gap.is_none() && output.change_diff_refusal.is_none() && output.full_text_index_rebuild_required.is_none() diff --git a/crates/omnigraph-cli/src/main.rs b/crates/omnigraph-cli/src/main.rs index 0508efe44..cb85df74d 100644 --- a/crates/omnigraph-cli/src/main.rs +++ b/crates/omnigraph-cli/src/main.rs @@ -1380,6 +1380,72 @@ async fn run(cli: Cli) -> Result<()> { print_blob_stat_human(&output); } } + BlobCommand::Put { + entity, + type_name, + id, + property, + branch, + file, + if_match, + json, + } => { + // A malformed precondition and input over the limit are + // refused before scope resolution, so neither arm starts. + let precondition = blob_cli::parse_if_match(if_match.as_deref())?; + let bytes = blob_cli::read_put_input(file.as_deref())?; + let query = + blob_cli::blob_write_query(entity.into(), type_name, id, property, branch); + let client = client::GraphClient::resolve_with_policy( + capability, + cli.server.as_deref(), + cli.graph.as_deref(), + None, + cli.as_actor.as_deref(), + cli.profile.as_deref(), + cli.store.as_deref(), + ) + .await?; + let output = client + .blob_write(&query, Some(bytes), if_match.as_deref(), precondition) + .await?; + if json { + print_json(&output)?; + } else { + print_blob_write_human(&output); + } + } + BlobCommand::Clear { + entity, + type_name, + id, + property, + branch, + if_match, + json, + } => { + let precondition = blob_cli::parse_if_match(if_match.as_deref())?; + let query = + blob_cli::blob_write_query(entity.into(), type_name, id, property, branch); + let client = client::GraphClient::resolve_with_policy( + capability, + cli.server.as_deref(), + cli.graph.as_deref(), + None, + cli.as_actor.as_deref(), + cli.profile.as_deref(), + cli.store.as_deref(), + ) + .await?; + let output = client + .blob_write(&query, None, if_match.as_deref(), precondition) + .await?; + if json { + print_json(&output)?; + } else { + print_blob_write_human(&output); + } + } }, Command::Query { name, diff --git a/crates/omnigraph-cli/src/output.rs b/crates/omnigraph-cli/src/output.rs index 7756958f6..342cc1cbc 100644 --- a/crates/omnigraph-cli/src/output.rs +++ b/crates/omnigraph-cli/src/output.rs @@ -711,6 +711,39 @@ pub(crate) fn print_snapshot_human( } } +/// A Blob write receipt: the cell, the state it was left in and the commit +/// that published it, or that nothing was published. +pub(crate) fn print_blob_write_human(output: &omnigraph_api_types::BlobWriteOutput) { + let entity = match output.selector.entity { + omnigraph_api_types::BlobEntityKind::Node => "node", + omnigraph_api_types::BlobEntityKind::Edge => "edge", + }; + println!("entity: {entity}"); + println!("type: {}", output.selector.r#type); + println!("id: {}", output.selector.id); + println!("property: {}", output.selector.property); + println!("branch: {}", output.branch); + match output.kind { + omnigraph_api_types::BlobWriteStateOutput::Managed => { + println!("kind: managed"); + if let Some(size) = output.size { + println!("size: {size}"); + } + if let Some(etag) = output.etag.as_deref() { + println!("etag: {etag}"); + } + } + omnigraph_api_types::BlobWriteStateOutput::Null => println!("kind: null"), + } + if let Some(actor) = output.actor_id.as_deref() { + println!("actor: {actor}"); + } + match &output.commit { + Some(commit) => println!("commit: {}", commit.graph_commit_id), + None => println!("commit: none (the cell was already null)"), + } +} + pub(crate) fn print_blob_stat_human(output: &BlobStatOutput) { let entity = match output.selector.entity { omnigraph_api_types::BlobEntityKind::Node => "node", diff --git a/crates/omnigraph-cli/src/planes.rs b/crates/omnigraph-cli/src/planes.rs index 44c4bf583..4fc1c6fc4 100644 --- a/crates/omnigraph-cli/src/planes.rs +++ b/crates/omnigraph-cli/src/planes.rs @@ -182,7 +182,8 @@ fn flag_applies(flag: ScopeFlag, capability: Capability, cmd: &Command) -> bool Served | Control | Local => false, }, // The actor rides direct-engine writes (`any` via --store) and cluster - // writes; Blob get/stat are reads and never consume it. Served writes + // writes; Blob get/stat are reads and never consume it, while Blob + // put/clear are writes and do. Served writes // resolve the actor from the bearer token // (rejected downstream with its own message). On `direct`, full-text // rebuild attributes its graph publication; other maintenance verbs @@ -196,8 +197,9 @@ fn flag_applies(flag: ScopeFlag, capability: Capability, cmd: &Command) -> bool // so accepting `--as` there is a silent no-op — reject it instead. Any => !matches!( cmd, - Command::Blob { .. } - | Command::Changes { .. } + Command::Blob { + command: BlobCommand::Get { .. } | BlobCommand::Stat { .. } + } | Command::Changes { .. } | Command::Commit { .. } | Command::Query { .. } ), @@ -332,6 +334,8 @@ pub(crate) fn command_label(cmd: &Command) -> &'static str { Command::Blob { command } => match command { BlobCommand::Get { .. } => "blob get", BlobCommand::Stat { .. } => "blob stat", + BlobCommand::Put { .. } => "blob put", + BlobCommand::Clear { .. } => "blob clear", }, Command::Commit { .. } => "commit", Command::Changes { command } => match command { diff --git a/crates/omnigraph-cli/tests/cli_data.rs b/crates/omnigraph-cli/tests/cli_data.rs index d7b5a636c..241d87d26 100644 --- a/crates/omnigraph-cli/tests/cli_data.rs +++ b/crates/omnigraph-cli/tests/cli_data.rs @@ -4,7 +4,7 @@ use std::fs; use assert_cmd::Command; -use serde_json::Value; +use serde_json::{Value, json}; use sha2::Digest; use tempfile::tempdir; @@ -573,6 +573,148 @@ fn blob_commands_reject_positional_and_cluster_scope_addressing() { ); } +/// `blob put` stores a file's or stdin's raw bytes and `blob clear` nulls the +/// cell, each answering the exact receipt; `--as` attributes an embedded +/// write. A stale `--if-match` fails with the Blob precondition detail, and +/// a malformed one or input over 32 MiB is refused before the graph is +/// addressed, so an unreachable store is never touched. +#[test] +fn blob_put_and_clear_write_receipts_and_refuse_bad_input_before_scope() { + let temp = tempdir().unwrap(); + let graph = graph_path(temp.path()); + init_blob_graph(&graph); + let blob = |verb: &str, selector: [&str; 4]| { + let mut command = cli(); + command + .arg("blob") + .arg(verb) + .args(selector) + .arg("--store") + .arg(&graph); + command + }; + let readme = ["node", "Document", "readme", "content"]; + let get = |selector: [&str; 4]| output_success(&mut blob("get", selector)).stdout; + let stat_etag = |selector: [&str; 4]| { + let output = output_success(blob("stat", selector).arg("--json")); + parse_stdout_json(&output)["etag"] + .as_str() + .unwrap() + .to_string() + }; + + let file = temp.path().join("payload.bin"); + fs::write(&file, b"from a file\0\xff").unwrap(); + let original = stat_etag(readme); + let put = output_success( + blob("put", readme) + .arg("--file") + .arg(&file) + .arg("--if-match") + .arg(&original) + .arg("--as") + .arg("act-writer") + .arg("--json"), + ); + let receipt = parse_stdout_json(&put); + assert_eq!( + receipt["selector"], + json!({"entity": "node", "type": "Document", "id": "readme", "property": "content"}) + ); + assert_eq!(receipt["branch"], "main"); + assert_eq!(receipt["kind"], "managed"); + assert_eq!(receipt["size"], 13); + assert_eq!(receipt["actor_id"], "act-writer"); + assert_eq!(receipt["commit"]["actor_id"], "act-writer"); + assert_eq!(receipt["etag"], stat_etag(readme).as_str()); + assert_eq!(get(readme), b"from a file\0\xff"); + + let edge = ["edge", "Attachment", "attachment-1", "payload"]; + let human = output_success(blob("put", edge).write_stdin(b"from stdin".to_vec())); + let human = stdout_string(&human); + assert!( + human.contains("kind: managed") && human.contains("size: 10"), + "{human}" + ); + assert_eq!(get(edge), b"from stdin"); + + let stale = output_failure( + blob("put", readme) + .arg("--file") + .arg(&file) + .arg("--if-match") + .arg(&original) + .arg("--json"), + ); + assert_eq!(stale.status.code(), Some(1), "an embedded mismatch exits 1"); + let refused = parse_stdout_json(&stale); + assert_eq!( + refused["blob_precondition_failure"]["current_etag"], + stat_etag(readme).as_str() + ); + assert!(refused.get("precondition_failure").is_none(), "{refused}"); + assert_eq!(refused["command_outcome"]["action"], "refresh"); + + let cleared = parse_stdout_json(&output_success( + blob("clear", readme) + .arg("--if-match") + .arg("*") + .arg("--json"), + )); + assert_eq!(cleared["kind"], "null"); + assert!( + cleared["commit"]["graph_commit_id"].is_string(), + "{cleared}" + ); + assert!(cleared.get("etag").is_none() && cleared.get("size").is_none()); + let again = parse_stdout_json(&output_success(blob("clear", readme).arg("--json"))); + assert!( + again["commit"].is_null(), + "a no-op clear publishes nothing: {again}" + ); + output_failure(&mut blob("get", readme)); + + // Refused before the graph is addressed: the store does not exist. + let absent = temp.path().join("absent.omni"); + let unreachable = |verb: &str| { + let mut command = cli(); + command + .arg("blob") + .arg(verb) + .args(readme) + .arg("--store") + .arg(&absent); + command + }; + let malformed = output_failure(unreachable("clear").arg("--if-match").arg("unquoted")); + assert!( + String::from_utf8_lossy(&malformed.stderr).contains("If-Match"), + "{malformed:?}" + ); + let oversized = temp.path().join("oversized.bin"); + fs::File::create(&oversized) + .unwrap() + .set_len(32 * 1024 * 1024 + 1) + .unwrap(); + let too_large = output_failure( + unreachable("put") + .arg("--file") + .arg(&oversized) + .arg("--json"), + ); + let refused = parse_stdout_json(&too_large); + assert_eq!( + refused["resource_limit"], + json!({ + "resource": "Blob write payload bytes", + "limit": 32 * 1024 * 1024, + "actual": 32 * 1024 * 1024 + 1, + }) + ); + assert_eq!(refused["command_outcome"]["execution"], "not_started"); + assert!(!absent.exists(), "neither arm opened the store"); +} + #[test] fn embed_seed_fills_missing_and_preserves_existing_vectors_by_default() { let temp = tempdir().unwrap(); diff --git a/crates/omnigraph-cli/tests/parity_matrix.rs b/crates/omnigraph-cli/tests/parity_matrix.rs index 3002bb3a5..0db1619ab 100644 --- a/crates/omnigraph-cli/tests/parity_matrix.rs +++ b/crates/omnigraph-cli/tests/parity_matrix.rs @@ -76,6 +76,21 @@ fn blob_parity() -> Parity { } } +/// The Blob fixture with an independent byte copy as the local arm, so each +/// arm's writes land in its own files and never in the hard-linked read twin. +fn blob_write_parity() -> Parity { + let p = blob_parity(); + let served = p + ._temp + .path() + .join("blob-parity-cluster") + .join("graphs") + .join(format!("{PARITY_GRAPH_ID}.omni")); + std::fs::remove_dir_all(&p.local).unwrap(); + copy_dir(&served, &p.local); + p +} + impl Parity { fn run(&self, args: &[&str]) -> (std::process::Output, std::process::Output) { run_both(&self.local, &self.server.base_url, args) @@ -1109,6 +1124,121 @@ fn parity_blob_shared_failures_keep_exit_codes_aligned() { } } +/// The receipts of a Blob write normalized for comparison: each arm's own +/// commit identities and the ETag, which hashes its commit's transaction +/// file, are replaced once they are shown to be present. +fn normalized_blob_receipt( + verb: &str, + arm: &str, + output: &std::process::Output, +) -> serde_json::Value { + assert!(output.status.success(), "{verb}: {arm} failed: {output:?}"); + let mut receipt = parse_stdout_json(output); + if let Some(etag) = receipt.get_mut("etag") { + assert!( + etag.as_str().is_some_and(|etag| etag.starts_with('"')), + "{verb}: {arm}" + ); + *etag = serde_json::Value::String("".into()); + } + if let Some(commit) = receipt + .get_mut("commit") + .and_then(|commit| commit.as_object_mut()) + { + assert!(commit["graph_commit_id"].is_string(), "{verb}: {arm}"); + for key in [ + "graph_commit_id", + "parent_commit_id", + "merged_parent_commit_id", + ] { + if let Some(value) = commit.get_mut(key) + && !value.is_null() + { + *value = serde_json::Value::String(format!("")); + } + } + } + scrub_volatile(&mut receipt); + receipt +} + +/// `blob ` on the fixture's readme content cell, with `--json`. +fn blob_write_args<'a>(verb: &'a str, extra: &[&'a str]) -> Vec<&'a str> { + let mut args = vec!["blob", verb, "node", "Document", "readme", "content"]; + args.extend_from_slice(extra); + args.push("--json"); + args +} + +#[test] +fn parity_blob_put_and_clear_share_receipts_and_refusals() { + let p = blob_write_parity(); + let payload = p._temp.path().join("parity-put.bin"); + std::fs::write(&payload, b"parity bytes\0\xff").unwrap(); + let payload = payload.to_str().unwrap(); + let with = blob_write_args; + + for (verb, args) in [ + ("blob put", with("put", &["--file", payload])), + ("blob clear", with("clear", &[])), + ("blob clear of a null cell", with("clear", &[])), + ] { + let (local, remote) = p.run(&args); + assert_eq!( + normalized_blob_receipt(verb, "local", &local), + normalized_blob_receipt(verb, "remote", &remote), + "{verb}: normalized receipts diverge (left=local, right=remote)" + ); + if verb == "blob put" { + assert_eq!(parse_stdout_json(&local)["actor_id"], PARITY_ACTOR); + } + if verb.ends_with("null cell") { + assert!(parse_stdout_json(&local)["commit"].is_null()); + } + } + + // A stale validator: the same Blob precondition detail on both arms. As + // for a graph-commit precondition, the served arm's verified 412 exits 4 + // and the embedded arm, which opened storage for writing, exits 1. + let (local, remote) = p.run(&with( + "put", + &["--file", payload, "--if-match", "\"stale\""], + )); + assert_eq!( + (local.status.code(), remote.status.code()), + (Some(1), Some(4)) + ); + let (local, remote) = (parse_stdout_json(&local), parse_stdout_json(&remote)); + for (arm, refused) in [("local", &local), ("remote", &remote)] { + assert_eq!(refused["code"], "conflict", "{arm}: {refused}"); + assert!( + refused.get("precondition_failure").is_none(), + "{arm}: {refused}" + ); + assert_eq!( + refused["blob_precondition_failure"], + serde_json::json!({}), + "{arm}: a null cell has no validator" + ); + assert_eq!(refused["command_outcome"]["action"], "refresh", "{arm}"); + } + assert_eq!(local["error"], remote["error"]); + + // Input over the limit is refused identically before either arm starts. + let oversized = p._temp.path().join("parity-oversized.bin"); + std::fs::File::create(&oversized) + .unwrap() + .set_len(32 * 1024 * 1024 + 1) + .unwrap(); + let (local, remote) = p.run(&with("put", &["--file", oversized.to_str().unwrap()])); + assert_eq!(local.status.code(), remote.status.code()); + assert_eq!(parse_stdout_json(&local), parse_stdout_json(&remote)); + assert_eq!( + parse_stdout_json(&local)["command_outcome"]["execution"], + "not_started" + ); +} + // ---- error parity: exit codes must match for shared failure cases ---- #[test] diff --git a/crates/omnigraph-cli/tests/support/managed_http.rs b/crates/omnigraph-cli/tests/support/managed_http.rs index 6b431303a..93660544d 100644 --- a/crates/omnigraph-cli/tests/support/managed_http.rs +++ b/crates/omnigraph-cli/tests/support/managed_http.rs @@ -34,8 +34,9 @@ pub struct IntentReply { pub body: Vec, } -/// Delivery faults applied only after a real server's successful merge reply -/// has been fully read. They cannot stand in for cancelling a server request. +/// Delivery faults applied only after a real server's successful merge or +/// Blob write reply has been fully read. They cannot stand in for cancelling a +/// server request. #[derive(Debug, Clone, Copy)] pub enum MergeDeliveryFault { Disconnect, @@ -115,6 +116,12 @@ impl IntentApiFixture { ) } + /// Forward discovery and one Blob put or clear to an actual server, then + /// break only delivery of its successful receipt. + pub fn graph_blob_write_proxy(upstream: &str, fault: MergeDeliveryFault) -> Self { + Self::graph_merge_proxy(upstream, fault) + } + pub fn graph_deployment_proxy(upstream: &str, fault: DeploymentDeliveryFault) -> Self { Self::start_with_origin( |_| Vec::new(), @@ -214,17 +221,22 @@ impl IntentApiFixture { let length = headers .get("content-length") .map_or(0, |n| n.parse::().unwrap()); - assert!(length <= 1024 * 1024, "fixture request body bound"); + // A Blob put carries up to 32 MiB of raw bytes; every other + // request stays within the JSON bound. + assert!(length <= 33 * 1024 * 1024, "fixture request body bound"); let mut body = vec![0; length]; reader.read_exact(&mut body).unwrap(); - let ndjson = headers - .get("content-type") - .is_some_and(|value| value == "application/x-ndjson"); + let raw = headers.get("content-type").is_some_and(|value| { + value == "application/x-ndjson" || value == "application/octet-stream" + }); + if !raw { + assert!(length <= 1024 * 1024, "fixture JSON request body bound"); + } let request = IntentRequest { method: request_line[0].clone(), path: request_line[1].clone(), headers, - body: if body.is_empty() || ndjson { + body: if body.is_empty() || raw { Value::Null } else { serde_json::from_slice(&body).unwrap() @@ -346,12 +358,19 @@ impl IntentApiFixture { headers, body, }; + let blob_write = matches!(request.method.as_str(), "PUT" | "DELETE") + && request + .path + .split('?') + .next() + .is_some_and(|path| path.ends_with("/blob")); if let Forwarding::Merge(_, fault) = forwarding - && request.method == "POST" - && (request.path.ends_with("/branches/merge") - || request.path.ends_with("/mutate")) + && ((request.method == "POST" + && (request.path.ends_with("/branches/merge") + || request.path.ends_with("/mutate"))) + || blob_write) { - assert_eq!(reply.status, 200, "upstream merge must succeed"); + assert_eq!(reply.status, 200, "upstream write must succeed"); captured_responses.lock().unwrap().push(reply.clone()); match fault { MergeDeliveryFault::Disconnect => continue, diff --git a/crates/omnigraph-cli/tests/support/mod.rs b/crates/omnigraph-cli/tests/support/mod.rs index 99a8baf4d..b187fb585 100644 --- a/crates/omnigraph-cli/tests/support/mod.rs +++ b/crates/omnigraph-cli/tests/support/mod.rs @@ -1410,14 +1410,17 @@ pub fn run_both( if args.first() == Some(&"blob") { local.env("NO_COLOR", "1"); } - // The read commands (blob/query/changes/commit) reject `--as`: a served read - // resolves the actor from its bearer token and a direct read attributes - // none, so passing `--as` there is an error. Only the write verbs carry the - // parity actor on the direct arm. - if !matches!( - args.first().copied(), - Some("blob") | Some("query") | Some("changes") | Some("commit") - ) { + // The read commands (blob get/stat, query, changes, commit) reject `--as`: + // a served read resolves the actor from its bearer token and a direct read + // attributes none, so passing `--as` there is an error. Only the write + // verbs, `blob put` and `blob clear` among them, carry the parity actor on + // the direct arm. + let read = match args { + ["blob", verb, ..] => matches!(*verb, "get" | "stat"), + [verb, ..] => matches!(*verb, "query" | "changes" | "commit"), + [] => false, + }; + if !read { local.arg("--as").arg(PARITY_ACTOR); } let local_out = local.output().unwrap(); diff --git a/crates/omnigraph-cli/tests/system_remote.rs b/crates/omnigraph-cli/tests/system_remote.rs index 7f94b1abe..c94c90b85 100644 --- a/crates/omnigraph-cli/tests/system_remote.rs +++ b/crates/omnigraph-cli/tests/system_remote.rs @@ -724,6 +724,129 @@ fn remote_merge_delivery_loss_never_replays_committed_effect() { } } +/// A Blob put or clear whose successful receipt is lost in delivery is +/// published once and never resent: the CLI reports the failure, the server +/// holds exactly the receipt the proxy saw, and the proxy saw one submission. +/// The put carries 2 MiB of raw bytes, past the JSON request bound. +#[test] +fn remote_blob_write_delivery_loss_never_replays_committed_effect() { + use support::managed_http::{IntentApiFixture, MergeDeliveryFault}; + + let temp = tempfile::tempdir().unwrap(); + let (cluster_dir, _) = blob_parity_config(temp.path(), &temp.path().join("local.omni")); + let server = spawn_server_with_cluster_env( + &cluster_dir, + &[( + "OMNIGRAPH_SERVER_BEARER_TOKENS_JSON", + r#"{"act-parity":"parity-tok"}"#, + )], + ); + let client = graph_http_client(); + let graph_url = format!("{}/graphs/{PARITY_GRAPH_ID}", server.base_url); + let blob_url = format!("{graph_url}/blob?entity=node&type=Document&id=readme&property=content"); + let authorized = |request: reqwest::blocking::RequestBuilder| { + request.header("authorization", format!("Bearer {PARITY_TOKEN}")) + }; + let commits = || { + authorized(client.get(format!("{graph_url}/commits?branch=main"))) + .send() + .unwrap() + .error_for_status() + .unwrap() + .json::() + .unwrap() + }; + let payload = temp.path().join("payload.bin"); + let bytes: Vec = (0..2 * 1024 * 1024) + .map(|index| (index % 251) as u8) + .collect(); + std::fs::write(&payload, &bytes).unwrap(); + + for put in [true, false] { + for fault in [ + MergeDeliveryFault::Disconnect, + MergeDeliveryFault::Truncate, + MergeDeliveryFault::GatewayTimeout, + MergeDeliveryFault::CallerWait, + ] { + if !put { + authorized(client.put(&blob_url)) + .header("content-type", "application/octet-stream") + .body(b"to clear".to_vec()) + .send() + .unwrap() + .error_for_status() + .unwrap(); + } + let before = commits(); + let proxy = IntentApiFixture::graph_blob_write_proxy(&server.base_url, fault); + let mut command = cli(); + command + .env("OMNIGRAPH_BEARER_TOKEN", PARITY_TOKEN) + .arg("blob") + .arg(if put { "put" } else { "clear" }) + .args(["node", "Document", "readme", "content"]); + if put { + command.arg("--file").arg(&payload); + } + command + .arg("--server") + .arg(&proxy.origin) + .arg("--graph") + .arg(PARITY_GRAPH_ID) + .arg("--json") + .timeout(std::time::Duration::from_secs(15)); + let output = output_failure(&mut command); + if !matches!(fault, MergeDeliveryFault::CallerWait) { + assert_eq!(output.status.code(), Some(1), "{put}/{fault:?}"); + } + if !output.stdout.is_empty() { + let error = parse_stdout_json(&output); + assert!(error.get("error").is_some(), "{put}/{fault:?}: {error}"); + assert!( + error.get("commit").is_none(), + "lost delivery is not success" + ); + } + + let captured = proxy.forwarded_responses(); + assert_eq!(captured.len(), 1, "{put}/{fault:?}: one upstream write"); + let upstream: serde_json::Value = serde_json::from_slice(&captured[0].body).unwrap(); + assert_eq!(upstream["kind"], if put { "managed" } else { "null" }); + let receipt = &upstream["commit"]; + let after = commits(); + assert_eq!(after["commits"][0], *receipt, "{put}/{fault:?}"); + assert_eq!( + after["commits"].as_array().unwrap().len(), + before["commits"].as_array().unwrap().len() + 1, + "{put}/{fault:?}: published once" + ); + let read = authorized(client.get(&blob_url)).send().unwrap(); + if put { + assert_eq!(read.status(), reqwest::StatusCode::OK); + assert_eq!( + read.headers()["etag"].to_str().unwrap(), + upstream["etag"].as_str().unwrap() + ); + assert_eq!(read.bytes().unwrap().as_ref(), bytes.as_slice()); + } else { + assert_eq!(read.status(), reqwest::StatusCode::NOT_FOUND); + } + proxy.assert_complete(); + let requests = proxy.requests(); + assert_eq!(requests.len(), 2, "discovery and exactly one submission"); + assert_eq!(requests[0].method, "HEAD"); + assert_eq!(requests[1].method, if put { "PUT" } else { "DELETE" }); + if put { + assert_eq!( + requests[1].raw_body, bytes, + "the raw body crossed unchanged" + ); + } + } + } +} + #[test] fn remote_deployment_delivery_loss_preserves_owned_completion() { use std::io::{BufRead, BufReader, Write}; diff --git a/docs/dev/blob.md b/docs/dev/blob.md index 6c0d05262..9abca5363 100644 --- a/docs/dev/blob.md +++ b/docs/dev/blob.md @@ -2,7 +2,7 @@ OmniGraph treats a Blob as one property cell on an existing node or edge. Lance owns the Blob-v2 physical placement; OmniGraph owns logical identity, snapshot selection, authorization, external-source admission, bounded delivery, and graph-level publication. -This page describes the implemented read, ingestion and cell-write behavior. The CLI put and clear commands proposed by [RFC 0033](../rfcs/0033-blob-management.md) are not implemented yet. +This page describes the implemented read, ingestion and cell-write behavior proposed by [RFC 0033](../rfcs/0033-blob-management.md). ## Logical states @@ -84,7 +84,9 @@ An external value produces a `302` redirect with the stored URI; the server neve The CLI exposes `blob get` and `blob stat` for embedded and remote graphs. `get` streams managed bytes to stdout or `--out`; `stat` returns kind, resolved-view metadata, size/ETag for managed data, or the descriptor for external data. The CLI refuses to follow external references. -`PUT` and `DELETE /graphs/{graph_id}/blob` call the engine cell writes. `PUT` is a raw ingress route like `/load/ndjson`: the ingress middleware reserves the put limit (`BLOB_WRITE_MAX_BYTES`, the engine's own constant) without collecting the body, and the handler authorizes `change` on the branch, checks `Content-Type` (415) and a declared `Content-Length` (413) before it polls a byte, then collects the body under the shared body deadline (408) into one buffer sized from that length, which the engine's value column adopts. The ingress reservation shrinks to the received size. Both verbs admit the actor's workload and submit the engine call through `owned_write`, so a disconnect after admission loses only the response. `omnigraph_api_types::parse_blob_if_match` turns `If-Match` field lines into a `BlobPrecondition` (`*` alone, or the strong tags of a list; weak tags dropped; malformed fields refused), and `blob_write_output` maps the engine outcome to the receipt; the CLI's embedded writes will share both. `BlobWritePreconditionFailed` maps to 412 with `ErrorCode::Conflict`, the `blob_precondition_failure` detail and an `ETag` header, never the graph-commit `precondition_failure`. The CLI has no put or clear yet. +`PUT` and `DELETE /graphs/{graph_id}/blob` call the engine cell writes. `PUT` is a raw ingress route like `/load/ndjson`: the ingress middleware reserves the put limit (`BLOB_WRITE_MAX_BYTES`, the engine's own constant) without collecting the body, and the handler authorizes `change` on the branch, checks `Content-Type` (415) and a declared `Content-Length` (413) before it polls a byte, then collects the body under the shared body deadline (408) into one buffer sized from that length, which the engine's value column adopts. The ingress reservation shrinks to the received size. Both verbs admit the actor's workload and submit the engine call through `owned_write`, so a disconnect after admission loses only the response. `omnigraph_api_types::parse_blob_if_match` turns `If-Match` field lines into a `BlobPrecondition` (`*` alone, or the strong tags of a list; weak tags dropped; malformed fields refused), and `blob_write_output` maps the engine outcome to the receipt; the CLI shares both. `BlobWritePreconditionFailed` maps to 412 with `ErrorCode::Conflict`, the `blob_precondition_failure` detail and an `ETag` header, never the graph-commit `precondition_failure`. + +`blob put` and `blob clear` (`crates/omnigraph-cli`) parse `--if-match` and read `put`'s file or stdin, refusing more than `BLOB_WRITE_MAX_BYTES` without reading past one more byte, before scope resolution, so neither arm starts on bad input. The embedded arm calls the engine writes and renders a precondition failure through `engine_error_output`, the body a server answers; the served arm sends the raw `If-Match` value and the bytes once, with no retry. A served 412 whose only detail is `blob_precondition_failure` is a verified request precondition, so a single-request command exits 4, as for a graph-commit precondition; the embedded arm, which opened storage for writing, exits 1. ## Maintenance and export @@ -102,6 +104,6 @@ Export emits managed values as base64 and whole-object external values as URI de - Lance compatibility: `crates/omnigraph/tests/lance_surface_guards.rs`. - Cluster policy persistence and serving projection: `omnigraph-cluster` in-source tests. - HTTP transport: `crates/omnigraph-server/tests/data_routes.rs`, `auth_policy.rs`, and `openapi.rs`. The writes: `data_routes.rs::blob_put_and_delete_return_exact_receipts_and_blob_preconditions`, `blob_put_raw_body_bounds_media_type_length_and_deadline`, the Blob doors of `disconnected_writes_keep_admission_until_the_original_operation_finishes`, the `change` cases of `auth_policy.rs::policy_blocks_change_on_protected_main_but_allows_unprotected_branch`, and the `If-Match` parser's tests in `omnigraph-api-types`. -- CLI and embedded/remote parity: `crates/omnigraph-cli/tests/cli_data.rs` and `parity_matrix.rs`. +- CLI and embedded/remote parity: `crates/omnigraph-cli/tests/cli_data.rs` and `parity_matrix.rs` (`parity_blob_put_and_clear_share_receipts_and_refusals` for the writes); `system_remote.rs::remote_blob_write_delivery_loss_never_replays_committed_effect` owns lost delivery of a put or clear through the fault proxy. The user contract and examples live in [Blob values](../user/blobs.md). diff --git a/docs/dev/testing.md b/docs/dev/testing.md index 6f99b9c97..a43d69ddc 100644 --- a/docs/dev/testing.md +++ b/docs/dev/testing.md @@ -107,7 +107,7 @@ Blob coverage is deliberately split: - engine `maintenance.rs` owns Blob compaction (the batch derived from a row's summed Blob columns, fragments with deleted rows, per-task sizing in `maintenance.rs::optimize_sizes_each_compaction_task_from_its_own_fragments`, external references counting nothing in `maintenance.rs::optimize_does_not_size_a_blob_batch_by_external_references`); - cluster tests own persisted external-source policy and serving projections; - server `data_routes.rs`, `auth_policy.rs`, and `openapi.rs` own GET/HEAD, auth, conditions, ranges, redirects, backpressure, and schema drift; -- CLI `cli_data.rs` owns `blob get/stat`; `parity_matrix.rs` compares embedded and remote results. +- CLI `cli_data.rs` owns `blob get/stat/put/clear`; `parity_matrix.rs` compares embedded and remote results; `system_remote.rs` owns a put's or clear's lost delivery through the fault proxy. Do not exercise a server promise solely through the engine facade. The complete contract is summarized in [blob.md](blob.md). diff --git a/docs/user/blobs.md b/docs/user/blobs.md index ad689f017..864168a0f 100644 --- a/docs/user/blobs.md +++ b/docs/user/blobs.md @@ -125,8 +125,21 @@ Over HTTP: disconnects, so a lost response means the write may have landed: read the cell before writing again. -The CLI does not offer these writes yet; through it, change a Blob value with a -load or a mutation. +From the CLI, `blob put` stores the bytes of `--file` or stdin and `blob clear` +nulls the cell. Both take `--branch`, `--if-match` and `--json`, and print the +receipt the server returns: + +```bash +omnigraph blob put node Document manual content --file manual.pdf \ + --if-match '""' --store graph.omni +omnigraph blob clear node Document manual content --server prod --graph knowledge +``` + +The CLI refuses input over 32 MiB and a malformed `--if-match` before it +addresses the graph. `--as` names the actor of a `--store` write; a served +write takes its actor from the bearer token. A served Blob `412` exits 4, as a +graph-commit precondition does; an embedded one exits 1. Each write is sent +once: after an unknown outcome, read the cell before writing again. ## Query behavior @@ -135,8 +148,7 @@ filtered, ordered, or aggregated. Write them through load or mutation assignment, then read an individual Blob value through the dedicated CLI or HTTP surface. -There are no `blob put` or `blob clear` CLI commands yet. Use the normal graph -write path, or the HTTP and embedded writes in +To replace or clear one value, use the CLI, HTTP or embedded writes in [Replacing one Blob value](#replacing-one-blob-value); each is an atomic graph commit. diff --git a/docs/user/cli/index.md b/docs/user/cli/index.md index 10a50fdea..f6a66e0a4 100644 --- a/docs/user/cli/index.md +++ b/docs/user/cli/index.md @@ -125,9 +125,9 @@ from a request`. A stored query takes no `--set`: `omnigraph query ` without `-e` or `--query` runs under the process defaults. -## Read Blob values +## Read and write Blob values -Read a managed Blob cell to a file or inspect its metadata: +Read a managed Blob cell to a file, inspect its metadata, or replace it: ```bash omnigraph blob get node Document doc-42 body \ @@ -135,11 +135,16 @@ omnigraph blob get node Document doc-42 body \ omnigraph blob stat node Document doc-42 body \ --store ./graph.omni --json + +omnigraph blob put node Document doc-42 body \ + --store ./graph.omni --file body.bin ``` `blob get` writes bytes to stdout when `--out` is omitted. Add `--offset` and `--length` for a range. The CLI reports external references but does not follow -them. See [Blob values](../blobs.md) for the complete contract. +them. `blob put` reads stdin when `--file` is omitted, and `blob clear` sets a +nullable cell to null; both accept `--if-match`. See [Blob values](../blobs.md) +for the complete contract. ## Inspect and maintain a graph diff --git a/docs/user/cli/reference.md b/docs/user/cli/reference.md index cafc2c48a..1c6edd710 100644 --- a/docs/user/cli/reference.md +++ b/docs/user/cli/reference.md @@ -43,7 +43,7 @@ server resolves the actor from the bearer token. Drop it, or use `--store ` | `query` | Run a read query, or the `branch list`, `show`, or `explain` statement | direct or served | | `mutate` | Run an insert/update/delete query, or a `branch create`, `branch delete`, or `branch merge` statement | direct or served | | `load` | Load graph JSONL in `overwrite`, `append`, or `merge` mode | direct or served | -| `blob get`, `blob stat` | Read or inspect one Blob cell | direct or served | +| `blob get/stat/put/clear` | Read, inspect, replace or clear one Blob cell | direct or served | | `branch create/list/delete/merge` | Manage graph branches | direct or served | | `snapshot` | Show a branch snapshot: `internal_schema_version`, `graph_manifest_version`, and the datasets of one captured graph version | direct or served | | `commit list/show/changes` | Inspect history or one commit's entity changes | direct or served | @@ -174,13 +174,13 @@ in [Changes and Change Feeds](../branching/changes.md). ## Blob commands ```text -omnigraph blob get [scope] [options] -omnigraph blob stat [scope] [options] +omnigraph blob get|stat|put|clear [scope] [options] ``` `get` accepts `--branch` or `--snapshot`, `--offset`, `--length`, and -`--out `. `stat` accepts `--branch` or `--snapshot` and `--json`. -See [Blob values](../blobs.md). +`--out `. `stat` accepts `--branch` or `--snapshot` and `--json`. `put` +reads `--file ` or stdin; `put` and `clear` accept `--branch`, +`--if-match `, `--json` and, on `--store`, `--as`. See [Blob values](../blobs.md). ## Operator configuration From b14909bded0918b482e08a990c38bd6c77de8e2b Mon Sep 17 00:00:00 2001 From: Ragnor Comerford Date: Fri, 9 Oct 2026 18:38:40 -0700 Subject: [PATCH 09/14] ci: require the Blob write delivery-loss journey in Test Workspace --- .github/workflows/ci.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 0012ec131..b7932b8e6 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -745,6 +745,7 @@ jobs: remote_read_preserves_projection_order_in_json_and_csv \ remote_branch_create_list_merge_flow \ remote_merge_delivery_loss_never_replays_committed_effect \ + remote_blob_write_delivery_loss_never_replays_committed_effect \ remote_deployment_delivery_loss_preserves_owned_completion \ remote_branch_delete_removes_branch \ remote_export_round_trips_full_branch_graph \ From 4a8a6972463ae9e55a2800098aa23d7a17972b4f Mon Sep 17 00:00:00 2001 From: Ragnor Comerford Date: Sat, 10 Oct 2026 09:51:12 -0700 Subject: [PATCH 10/14] docs(changelog): rewrap the Blob put fragment --- changelog.d/blob-cell-put-clear.added.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/changelog.d/blob-cell-put-clear.added.md b/changelog.d/blob-cell-put-clear.added.md index 31d427194..fd8ec1748 100644 --- a/changelog.d/blob-cell-put-clear.added.md +++ b/changelog.d/blob-cell-put-clear.added.md @@ -8,5 +8,5 @@ nothing. A put returns the value's length, its ETag and the commit; the ETag is the one a read at that commit reports. One put holds at most the session's `write_max_bytes` (32 MiB by default), inclusive; a larger one - fails before any table is opened with resource `Blob write payload bytes`. The CLI and HTTP server do not expose these - writes yet. + fails before any table is opened with resource `Blob write payload bytes`. + The CLI and HTTP server do not expose these writes yet. From 671641b5e8dee955119777d8c693b3b1cb7337b4 Mon Sep 17 00:00:00 2001 From: Ragnor Comerford Date: Sat, 10 Oct 2026 21:08:26 -0700 Subject: [PATCH 11/14] test(cli): observe the withheld Blob write reply instead of waiting out the caller --- crates/omnigraph-cli/tests/system_remote.rs | 74 ++++++++++++++++----- 1 file changed, 59 insertions(+), 15 deletions(-) diff --git a/crates/omnigraph-cli/tests/system_remote.rs b/crates/omnigraph-cli/tests/system_remote.rs index 898ad9b52..e28baeab2 100644 --- a/crates/omnigraph-cli/tests/system_remote.rs +++ b/crates/omnigraph-cli/tests/system_remote.rs @@ -773,6 +773,8 @@ fn assert_merge_delivery_loss_never_replays_committed_effect(statement: bool) { /// The put carries 2 MiB of raw bytes, past the JSON request bound. #[test] fn remote_blob_write_delivery_loss_never_replays_committed_effect() { + use std::process::Stdio; + use std::time::{Duration, Instant}; use support::managed_http::{IntentApiFixture, MergeDeliveryFault}; let temp = tempfile::tempdir().unwrap(); @@ -823,23 +825,65 @@ fn remote_blob_write_delivery_loss_never_replays_committed_effect() { } let before = commits(); let proxy = IntentApiFixture::graph_blob_write_proxy(&server.base_url, fault); - let mut command = cli(); - command - .env("OMNIGRAPH_BEARER_TOKEN", PARITY_TOKEN) - .arg("blob") - .arg(if put { "put" } else { "clear" }) - .args(["node", "Document", "readme", "content"]); + let mut args: Vec = ["blob", if put { "put" } else { "clear" }] + .into_iter() + .chain(["node", "Document", "readme", "content"]) + .map(Into::into) + .collect(); if put { - command.arg("--file").arg(&payload); + args.extend(["--file".into(), payload.clone().into_os_string()]); } - command - .arg("--server") - .arg(&proxy.origin) - .arg("--graph") - .arg(PARITY_GRAPH_ID) - .arg("--json") - .timeout(std::time::Duration::from_secs(15)); - let output = output_failure(&mut command); + args.extend( + [ + "--server", + proxy.origin.as_str(), + "--graph", + PARITY_GRAPH_ID, + "--json", + ] + .map(Into::into), + ); + // A withheld reply is observed, not waited out: once the proxy + // holds the consumed upstream receipt, the caller is still waiting + // and is abandoned. + let output = if matches!(fault, MergeDeliveryFault::CallerWait) { + let mut caller = cli_process() + .env("OMNIGRAPH_BEARER_TOKEN", PARITY_TOKEN) + .args(&args) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()) + .spawn() + .unwrap(); + let deadline = Instant::now() + Duration::from_secs(15); + while proxy.forwarded_responses().is_empty() { + assert!( + caller.try_wait().unwrap().is_none(), + "{put}/{fault:?}: caller must still wait while the proxy owns the consumed reply" + ); + assert!( + Instant::now() < deadline, + "{put}/{fault:?}: upstream write must complete before the caller is abandoned" + ); + std::thread::sleep(Duration::from_millis(5)); + } + assert!( + caller.try_wait().unwrap().is_none(), + "{put}/{fault:?}: caller must still wait when the consumed reply is withheld" + ); + caller.kill().unwrap(); + caller.wait_with_output().unwrap() + } else { + output_failure( + cli() + .env("OMNIGRAPH_BEARER_TOKEN", PARITY_TOKEN) + .args(&args) + .timeout(Duration::from_secs(15)), + ) + }; + assert!( + !output.status.success(), + "{put}/{fault:?}: lost delivery must not report success" + ); if !matches!(fault, MergeDeliveryFault::CallerWait) { assert_eq!(output.status.code(), Some(1), "{put}/{fault:?}"); } From c9ed133fccbd493c6ee9eeb2bbdf5e6787f49355 Mon Sep 17 00:00:00 2001 From: Ragnor Comerford Date: Sat, 10 Oct 2026 21:10:16 -0700 Subject: [PATCH 12/14] docs(rfc-0033): route PUT refuses at the server's write_max_bytes; split owners --- docs/rfcs/0033-blob-management.md | 26 +++++++++++++++----------- 1 file changed, 15 insertions(+), 11 deletions(-) diff --git a/docs/rfcs/0033-blob-management.md b/docs/rfcs/0033-blob-management.md index 05d72fbf4..dacce7891 100644 --- a/docs/rfcs/0033-blob-management.md +++ b/docs/rfcs/0033-blob-management.md @@ -1237,7 +1237,7 @@ Physical row addresses never become public stable identity. | URI credential disclosure | Reject user-info/query/fragment credentials at config and input; return URI only to an authorized Blob reader | | URI parser amplification | Reject a raw configured or input URI above 64 KiB before trimming, parsing, decoding, or filesystem resolution | | External SSRF during read | Descriptor-first classification; redirect only; no proxy or validation on GET/HEAD | -| Oversize upload | Route 32 MiB and engine `write_max_bytes` (32 MiB by default) inclusive limits in payload bytes, separate from the row-data budget (§4.3); refusal before effect | +| Oversize upload | Route and engine `write_max_bytes` (32 MiB by default, at most 32 MiB) inclusive limits in payload bytes, separate from the row-data budget (§4.3); refusal before effect, and before reading past the limit on the route | | Rewrite amplification | New logical input and row-writing branch merge pre-size all carried Blob payloads under one 32 MiB operation budget before read; predicate mutation carry applies the same cumulative byte ceiling while materializing bounded scan batches. Schema apply reads no Blob payload: its column changes are metadata-only (§8.4) | | Compaction memory | Optimize sets the compaction scanner batch from the planned fragments' largest row, summing that row's Blob columns: as many rows (1 to 8,192) as fit 32 MiB of managed payload at that row's size, so one batch materializes at most 32 MiB of managed payload. A row whose Blob values together exceed 32 MiB is compacted in a batch of its own and materialized whole. This bounds payload per batch, not heap: Lance's writer copies inline payloads into its prepared arrays while it holds the batch (see the operator guide's optimize section). External descriptors are carried unread | | External-source planning | Row-writing branch merge admits at most 8,192 external-reference cells and 32 MiB of retained URI metadata before HEAD; probes are bounded and normalized aliases deduplicate within the applicable operation or scan-batch envelope | @@ -1430,13 +1430,13 @@ The implementation extends existing owners before creating new fixtures, per reference, proving that target payload is not read while the untouched sibling remains byte-identical. Every refusal proves table HEAD, manifest, lineage, and accepted state unchanged. -- `writes.rs` also owns the payload/framing split (§4.3): an exact 32 MiB PUT is - accepted and one more byte is refused by the payload limit, not by a framing - ceiling; a 32 MiB value carried by an update, one inserted through an - embedded `.gq` parameter and one loaded by the compatibility loader are - accepted where the combined ceiling refused them, while the strict NDJSON - loader still refuses that value's encoded line at its line limit; and a - framing refusal still names its keyed-write ceiling. It owns the sibling carry rule +- The payload/row split (§4.3) is owned by the `write_max_bytes` owners: the + `write_max_bytes*.gqt` and `load_write_max_bytes_*.gqt` cases (an exact + payload beside its row, one more byte refused, a value carried by an update), + `consistency.rs` and `merge_fast_forward.rs`. `writes.rs` owns the put's + bound: an exact-limit put is accepted at the default and under a lowered + `write_max_bytes`, one more byte is refused as `Blob write payload bytes`, and + a carried sibling shares the payload allowance. It owns the sibling carry rule too: an external sibling becomes managed under an admitting policy (visible through `stat`), fails with `StoredExternalBlobDenied` naming the sibling under a denying one, and on a row with two external cells under a denying policy a @@ -1936,9 +1936,13 @@ publisher architecture. most 32 MiB, captured once per operation, for every keyed writer and the compatibility loader's forecast, which classifies a `base64:` value as payload only for a declared Blob property. The Blob put checks its value against the - session's allowance, so a lowered `write_max_bytes` lowers the put's bound too; - the PUT route and the CLI keep their 32 MiB pre-read bounds, the setting's - maximum. The two §12.3 surface guards move to 3A. Superseded: §4.3's "The raw + session's allowance, so a lowered `write_max_bytes` lowers the put's bound too. + A PUT carries no settings, so the route refuses a body over the server's + process `write_max_bytes` before reading past it and reserves only that much + ingress; the CLI keeps its 32 MiB pre-read bound, the setting's maximum, + because a served caller cannot see the server's value. The payload/row split + is owned by the `write_max_bytes` owners, so §12.2's `writes.rs` split bullet + narrows to the put's bound. The two §12.3 surface guards move to 3A. Superseded: §4.3's "The raw managed payload limit is 32 MiB inclusive" and "the same operation-wide 32 MiB pre-effect payload budget"; §10's "Route and engine 32 MiB inclusive limits in payload bytes, separate from the batch framing budget"; and §13's 3-pre From 2127c57a5219182f62519e96fe69db9cc8d1dfa0 Mon Sep 17 00:00:00 2001 From: Ragnor Comerford Date: Sat, 10 Oct 2026 21:15:26 -0700 Subject: [PATCH 13/14] feat(server): refuse a Blob PUT past the server's write_max_bytes before reading it --- crates/omnigraph-server/src/handlers.rs | 19 ++++---- crates/omnigraph-server/src/ingress.rs | 3 +- crates/omnigraph-server/src/lib.rs | 13 +++++- crates/omnigraph-server/tests/data_routes.rs | 47 ++++++++++++++++---- docs/dev/blob.md | 2 +- docs/user/blobs.md | 11 +++-- openapi.json | 2 +- 7 files changed, 68 insertions(+), 29 deletions(-) diff --git a/crates/omnigraph-server/src/handlers.rs b/crates/omnigraph-server/src/handlers.rs index 4a02b8c0f..a605273cf 100644 --- a/crates/omnigraph-server/src/handlers.rs +++ b/crates/omnigraph-server/src/handlers.rs @@ -1004,7 +1004,7 @@ pub(crate) async fn server_blob_head( headers( ("ETag" = String, description = "The cell's current validator, when it holds a managed value"), )), - (status = 413, description = "The body exceeds 32 MiB, or the value alone or with the entity's carried Blob payloads exceeds the server's `write_max_bytes`", body = ErrorOutput), + (status = 413, description = "The body exceeds the server's `write_max_bytes` (at most 32 MiB), or the value with the entity's carried Blob payloads exceeds it", body = ErrorOutput), (status = 415, description = "Content-Type must be application/octet-stream", body = ErrorOutput), (status = 424, description = "An allowed external Blob source carried from the entity could not be read", body = ErrorOutput), (status = 429, description = "Per-actor admission cap exceeded; honor `Retry-After` header", body = ErrorOutput), @@ -1058,9 +1058,9 @@ pub(crate) async fn server_blob_put( .ok_or_else(|| ApiError::bad_request("invalid Content-Length"))?, ), }; - if let Some(declared) = declared.filter(|declared| *declared > omnigraph::BLOB_WRITE_MAX_BYTES) - { - return Err(blob_put_too_large(declared)); + let limit = state.blob_put_limit(); + if let Some(declared) = declared.filter(|declared| *declared > limit) { + return Err(blob_put_too_large(limit, declared)); } let deadline = request @@ -1070,7 +1070,7 @@ pub(crate) async fn server_blob_put( .ok_or_else(|| ApiError::internal("missing request body deadline"))?; let bytes = tokio::time::timeout_at( deadline.0, - collect_blob_put_body(request.into_body(), declared), + collect_blob_put_body(request.into_body(), declared, limit), ) .await .map_err(|_| ingress::body_timeout())??; @@ -1222,10 +1222,10 @@ fn blob_write_cell(query: BlobWriteQuery) -> omnigraph::BlobCell { /// The refusal of a put body over the limit: the same resource, limit and /// message the engine reports for an oversized embedded put. -fn blob_put_too_large(actual: u64) -> ApiError { +fn blob_put_too_large(limit: u64, actual: u64) -> ApiError { ApiError::from_omni(OmniError::resource_limit( omnigraph::BLOB_WRITE_PAYLOAD_RESOURCE, - omnigraph::BLOB_WRITE_MAX_BYTES, + limit, actual, )) } @@ -1235,6 +1235,7 @@ fn blob_put_too_large(actual: u64) -> ApiError { async fn collect_blob_put_body( body: Body, declared: Option, + limit: u64, ) -> std::result::Result { let capacity = declared.map_or(Ok(0), usize::try_from).map_err(|_| { ApiError::bad_request("Content-Length does not fit in this platform's memory") @@ -1246,8 +1247,8 @@ async fn collect_blob_put_body( ApiError::bad_request(format!("failed to read Blob request body: {err}")) })?; let actual = data.len().saturating_add(chunk.len()) as u64; - if actual > omnigraph::BLOB_WRITE_MAX_BYTES { - return Err(blob_put_too_large(actual)); + if actual > limit { + return Err(blob_put_too_large(limit, actual)); } data.extend_from_slice(&chunk); } diff --git a/crates/omnigraph-server/src/ingress.rs b/crates/omnigraph-server/src/ingress.rs index d470adcad..0107ab35a 100644 --- a/crates/omnigraph-server/src/ingress.rs +++ b/crates/omnigraph-server/src/ingress.rs @@ -140,7 +140,8 @@ pub(crate) async fn admit( let limit = if !receives_body { 0 } else if blob_put { - crate::BLOB_PUT_REQUEST_BODY_LIMIT_BYTES + usize::try_from(state.blob_put_limit()) + .map_err(|_| ApiError::internal("Blob put limit exceeds usize"))? } else if matches!(route, "/cluster/deployments" | "/cluster/plan") { crate::deployment::REQUEST_BYTES } else if route == "/mcp" { diff --git a/crates/omnigraph-server/src/lib.rs b/crates/omnigraph-server/src/lib.rs index 5640b5e23..9168a72c7 100644 --- a/crates/omnigraph-server/src/lib.rs +++ b/crates/omnigraph-server/src/lib.rs @@ -181,8 +181,6 @@ impl utoipa::Modify for SecurityAddon { const DEFAULT_REQUEST_BODY_LIMIT_BYTES: usize = 1_048_576; const INGEST_REQUEST_BODY_LIMIT_BYTES: usize = 32 * 1024 * 1024; -/// A Blob put's raw body: exactly the bytes one put may store, inclusive. -const BLOB_PUT_REQUEST_BODY_LIMIT_BYTES: usize = omnigraph::BLOB_WRITE_MAX_BYTES as usize; const SERVER_VERSION: &str = env!("CARGO_PKG_VERSION"); const SERVER_SOURCE_VERSION: Option<&str> = option_env!("OMNIGRAPH_SOURCE_VERSION"); /// The maximum internal-schema (storage-format) version this binary supports. @@ -849,6 +847,17 @@ impl AppState { self } + /// The bytes one Blob put may store, inclusive: the process + /// `write_max_bytes`, at most `BLOB_WRITE_MAX_BYTES`. A put takes no + /// request settings, so this is known before its body is read; ingress + /// reserves it and the route refuses a body past it. + pub(crate) fn blob_put_limit(&self) -> u64 { + self.process_defaults + .settings + .write_max_bytes() + .min(omnigraph::BLOB_WRITE_MAX_BYTES) + } + /// One session for one request: the process defaults, then the typed /// `settings` field with source `request`. The source's own `set` lines /// apply inside the session method that runs the text. diff --git a/crates/omnigraph-server/tests/data_routes.rs b/crates/omnigraph-server/tests/data_routes.rs index 79768587c..df12e92d4 100644 --- a/crates/omnigraph-server/tests/data_routes.rs +++ b/crates/omnigraph-server/tests/data_routes.rs @@ -993,8 +993,9 @@ async fn blob_put_raw_body_bounds_media_type_length_and_deadline() { assert_eq!(status, StatusCode::OK, "{}", String::from_utf8_lossy(&body)); assert_admission_open(&operations, &workload, "after").await; - // The server's process `write_max_bytes` bounds the value once the body is - // read; a `PUT` carries no settings that could change it. + // A put takes no settings, so the server's process `write_max_bytes` is its + // limit before the body arrives: a declared body past it is refused + // unread, and a streamed one at the byte past it. let (settings, sources) = omnigraph::settings::from_env_with(|variable| { (variable == "OMNIGRAPH_WRITE_MAX_BYTES").then(|| "4093".to_string()) }) @@ -1005,20 +1006,48 @@ async fn blob_put_raw_body_bounds_media_type_length_and_deadline() { .unwrap() .with_process_defaults(ProcessDefaults { settings, sources }), ); + let refused_at = |body: &[u8]| { + let refused = serde_json::from_slice::(body) + .unwrap() + .resource_limit + .unwrap(); + (refused.resource, refused.limit, refused.actual) + }; + let (polled, body) = watched_body(); + let mut request = blob_write_request(Method::PUT, &uri, body, None); + request + .headers_mut() + .insert("content-length", HeaderValue::from_static("4094")); + let (status, _, body) = blob_exchange(&lowered, request).await; + assert_eq!(status, StatusCode::PAYLOAD_TOO_LARGE); + assert!( + !polled.load(Ordering::SeqCst), + "a declared body past the process limit is refused unread" + ); + assert_eq!( + refused_at(&body), + ("Blob write payload bytes".to_string(), 4093, 4094) + ); + let chunks = futures::stream::iter([ + Ok::(Bytes::from(vec![7_u8; 4093])), + Ok(Bytes::from_static(b"!")), + ]); let (status, _, body) = blob_exchange( &lowered, - blob_write_request(Method::PUT, &uri, vec![7_u8; 4094], None), + blob_write_request(Method::PUT, &uri, Body::from_stream(chunks), None), ) .await; assert_eq!(status, StatusCode::PAYLOAD_TOO_LARGE); - let refused = serde_json::from_slice::(&body) - .unwrap() - .resource_limit - .unwrap(); assert_eq!( - (refused.resource.as_str(), refused.limit, refused.actual), - ("Blob write payload bytes", 4093, 4094) + refused_at(&body), + ("Blob write payload bytes".to_string(), 4093, 4094) ); + let (status, _, body) = blob_exchange( + &lowered, + blob_write_request(Method::PUT, &uri, vec![7_u8; 4093], None), + ) + .await; + assert_eq!(status, StatusCode::OK, "{}", String::from_utf8_lossy(&body)); } #[tokio::test(flavor = "multi_thread")] diff --git a/docs/dev/blob.md b/docs/dev/blob.md index 06412f5b5..9387c3b6d 100644 --- a/docs/dev/blob.md +++ b/docs/dev/blob.md @@ -86,7 +86,7 @@ An external value produces a `302` redirect with the stored URI; the server neve The CLI exposes `blob get` and `blob stat` for embedded and remote graphs. `get` streams managed bytes to stdout or `--out`; `stat` returns kind, resolved-view metadata, size/ETag for managed data, or the descriptor for external data. The CLI refuses to follow external references. -`PUT` and `DELETE /graphs/{graph_id}/blob` call the engine cell writes. `PUT` is a raw ingress route like `/load/ndjson`: the ingress middleware reserves the put limit (`BLOB_WRITE_MAX_BYTES`, the engine's own constant) without collecting the body, and the handler authorizes `change` on the branch, checks `Content-Type` (415) and a declared `Content-Length` (413) before it polls a byte, then collects the body under the shared body deadline (408) into one buffer sized from that length, which the engine's value column adopts. The ingress reservation shrinks to the received size. Both verbs admit the actor's workload and submit the engine call through `owned_write`, so a disconnect after admission loses only the response. `omnigraph_api_types::parse_blob_if_match` turns `If-Match` field lines into a `BlobPrecondition` (`*` alone, or the strong tags of a list; weak tags dropped; malformed fields refused), and `blob_write_output` maps the engine outcome to the receipt; the CLI's embedded writes will share both. `BlobWritePreconditionFailed` maps to 412 with `ErrorCode::Conflict`, the `blob_precondition_failure` detail and an `ETag` header, never the graph-commit `precondition_failure`. The CLI has no put or clear yet. +`PUT` and `DELETE /graphs/{graph_id}/blob` call the engine cell writes. `PUT` is a raw ingress route like `/load/ndjson`: the ingress middleware reserves the put limit (`AppState::blob_put_limit`: the process `write_max_bytes`, at most the engine's `BLOB_WRITE_MAX_BYTES`, known before the body because a put takes no request settings) without collecting the body, and the handler authorizes `change` on the branch, checks `Content-Type` (415) and a declared `Content-Length` against that limit (413) before it polls a byte, then collects the body under the shared body deadline (408) into one buffer sized from that length, which the engine's value column adopts. The ingress reservation shrinks to the received size. Both verbs admit the actor's workload and submit the engine call through `owned_write`, so a disconnect after admission loses only the response. `omnigraph_api_types::parse_blob_if_match` turns `If-Match` field lines into a `BlobPrecondition` (`*` alone, or the strong tags of a list; weak tags dropped; malformed fields refused), and `blob_write_output` maps the engine outcome to the receipt; the CLI's embedded writes will share both. `BlobWritePreconditionFailed` maps to 412 with `ErrorCode::Conflict`, the `blob_precondition_failure` detail and an `ETag` header, never the graph-commit `precondition_failure`. The CLI has no put or clear yet. ## Maintenance and export diff --git a/docs/user/blobs.md b/docs/user/blobs.md index a83fab263..8c2caa0db 100644 --- a/docs/user/blobs.md +++ b/docs/user/blobs.md @@ -110,12 +110,11 @@ before retrying with a precondition. Over HTTP: -- `PUT` takes a `Content-Type: application/octet-stream` body of at most 32 MiB, - inclusive. Another media type is a `415`. A body over 32 MiB is a `413`, - refused before any of it is read when `Content-Length` declares it, and a body - that does not arrive before the server's body deadline is a `408`. The value - must also fit the server's `write_max_bytes`, which a request cannot change; - a larger one is a `413` once the body is read; +- `PUT` takes a `Content-Type: application/octet-stream` body of at most the + server's `write_max_bytes` (32 MiB by default), inclusive; a request cannot + change it. Another media type is a `415`. A larger body is a `413`, refused + before any of it is read when `Content-Length` declares it, and a body that + does not arrive before the server's body deadline is a `408`; - both verbs take `branch` (default `main`) and refuse `snapshot` or any other unknown parameter with a `400`. A missing branch is a `404`; - `change` is authorized before the body is read; diff --git a/openapi.json b/openapi.json index 973c71f28..97eed0ffd 100644 --- a/openapi.json +++ b/openapi.json @@ -1836,7 +1836,7 @@ } }, "413": { - "description": "The body exceeds 32 MiB, or the value alone or with the entity's carried Blob payloads exceeds the server's `write_max_bytes`; Request body exceeds its route limit before operation admission", + "description": "The body exceeds the server's `write_max_bytes` (at most 32 MiB), or the value with the entity's carried Blob payloads exceeds it; Request body exceeds its route limit before operation admission", "headers": { "omnigraph-http-api": { "schema": { From e610d1dfc5c45e26782c61c678c233b859c500ad Mon Sep 17 00:00:00 2001 From: Ragnor Comerford Date: Sun, 11 Oct 2026 16:18:25 -0700 Subject: [PATCH 14/14] test(cli): order the Blob delivery-loss imports for the integration binary --- crates/omnigraph-cli/tests/integration/system_remote.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/omnigraph-cli/tests/integration/system_remote.rs b/crates/omnigraph-cli/tests/integration/system_remote.rs index 6b28b991d..780d68d58 100644 --- a/crates/omnigraph-cli/tests/integration/system_remote.rs +++ b/crates/omnigraph-cli/tests/integration/system_remote.rs @@ -771,9 +771,9 @@ fn assert_merge_delivery_loss_never_replays_committed_effect(statement: bool) { /// The put carries 2 MiB of raw bytes, past the JSON request bound. #[test] fn remote_blob_write_delivery_loss_never_replays_committed_effect() { + use crate::support::managed_http::{IntentApiFixture, MergeDeliveryFault}; use std::process::Stdio; use std::time::{Duration, Instant}; - use crate::support::managed_http::{IntentApiFixture, MergeDeliveryFault}; let temp = tempfile::tempdir().unwrap(); let (cluster_dir, _) = blob_parity_config(temp.path(), &temp.path().join("local.omni"));