diff --git a/Cargo.lock b/Cargo.lock index 70da149e..977a9822 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -71,6 +71,26 @@ version = "3.20.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "72f5acc6cb2ba439de613abc23857ec3d78374d8ed5ac84e9d11336e87da8649" +[[package]] +name = "bytemuck" +version = "1.25.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c8efb64bd706a16a1bdde310ae86b351e4d21550d98d056f22f8a7f7a2183fec" +dependencies = [ + "bytemuck_derive", +] + +[[package]] +name = "bytemuck_derive" +version = "1.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f9abbd1bc6865053c427f7198e6af43bfdedc55ab791faed4fbd361d789575ff" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "cast" version = "0.3.0" @@ -553,6 +573,7 @@ dependencies = [ name = "leit_core" version = "0.1.0" dependencies = [ + "bytemuck", "proptest", ] @@ -642,6 +663,7 @@ dependencies = [ "criterion", "leit_core", "leit_index", + "leit_postings", "leit_text", "leit_wind_tunnel", ] diff --git a/Cargo.toml b/Cargo.toml index e41e3dc5..7039174e 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -50,6 +50,9 @@ proptest = { version = "1.0" } # Hashing rapidhash = { version = "4.4" } +# Zero-copy serialization for segment-resident types (no_std) +bytemuck = { default-features = false, features = ["derive"], version = "1" } + # Math core_maths = { default-features = false, version = "0.1" } diff --git a/crates/leit_core/Cargo.toml b/crates/leit_core/Cargo.toml index 60f1e83e..7c447300 100644 --- a/crates/leit_core/Cargo.toml +++ b/crates/leit_core/Cargo.toml @@ -15,6 +15,9 @@ categories.workspace = true default = ["std"] std = [] +[dependencies] +bytemuck = { workspace = true } + [dev-dependencies] proptest = { workspace = true } diff --git a/crates/leit_core/src/lib.rs b/crates/leit_core/src/lib.rs index 079514d9..aef00212 100644 --- a/crates/leit_core/src/lib.rs +++ b/crates/leit_core/src/lib.rs @@ -20,6 +20,9 @@ use core::fmt; use core::hash::Hash; use core::ops::{Add, AddAssign, Mul, MulAssign, Sub, SubAssign}; +pub mod segment_ids; +pub use segment_ids::{BlockId, FilterExprId, SegmentLocalDocId, SegmentOrd, TermFreq}; + /// Unique identifier for a field in an index. #[derive(Clone, Copy, Debug, Default, PartialEq, Eq, PartialOrd, Ord, Hash)] #[repr(transparent)] diff --git a/crates/leit_core/src/segment_ids.rs b/crates/leit_core/src/segment_ids.rs new file mode 100644 index 00000000..f18acce1 --- /dev/null +++ b/crates/leit_core/src/segment_ids.rs @@ -0,0 +1,201 @@ +// Copyright 2026 the Leit Authors +// SPDX-License-Identifier: Apache-2.0 OR MIT + +//! Segment-resident core ID and value types with a stable, zero-copy serialized +//! representation. +//! +//! Unlike the in-memory index identifiers ([`FieldId`](crate::FieldId), +//! [`TermId`](crate::TermId), [`SegmentId`](crate::SegmentId)) and the polymorphic +//! in-memory document identifier ([`EntityId`](crate::EntityId)), the types in this +//! module are designed to appear **directly in mmap'd segment bytes**. The generic +//! `EntityId` is *lowered* to the concrete [`SegmentLocalDocId`] when a segment is +//! written; segment-layer code (postings codecs, readers) speaks these named types, +//! never anonymous `u32`. Each is a +//! `#[repr(transparent)]` newtype over a 4-byte little-endian array, so a `&[u8]` +//! slice taken from a memory-mapped buffer can be viewed in place as `&[Id]` with +//! no allocation and no deserialization pass (see the Phase 2 architectural +//! decisions: zero-copy via `bytemuck`). +//! +//! The inner representation is `[u8; 4]` holding the **little-endian** bytes of a +//! `u32`, so the on-disk form is identical on every host (portable across +//! endianness). Because the storage is a byte array, ordering is implemented +//! by numeric value rather than raw byte order. + +use bytemuck::{Pod, Zeroable}; + +/// Define a segment-resident ID newtype over a little-endian 4-byte value. +macro_rules! segment_id { + ($name:ident, $doc:literal) => { + #[doc = $doc] + /// + /// Fixed-width 4-byte little-endian value; viewable in place from mmap'd + /// segment bytes via `bytemuck` (`Pod`). + #[derive(Clone, Copy, PartialEq, Eq, Hash, Pod, Zeroable)] + #[repr(transparent)] + pub struct $name([u8; 4]); + + impl $name { + #[doc = concat!("Create a new `", stringify!($name), "` from a raw `u32`.")] + #[must_use] + pub const fn new(value: u32) -> Self { + Self(value.to_le_bytes()) + } + + #[doc = concat!("Get the raw `u32` value of this `", stringify!($name), "`.")] + #[must_use] + pub const fn get(self) -> u32 { + u32::from_le_bytes(self.0) + } + } + + impl core::fmt::Debug for $name { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + write!(f, "{}({})", stringify!($name), self.get()) + } + } + + impl PartialOrd for $name { + fn partial_cmp(&self, other: &Self) -> Option { + Some(self.cmp(other)) + } + } + + impl Ord for $name { + fn cmp(&self, other: &Self) -> core::cmp::Ordering { + self.get().cmp(&other.get()) + } + } + + impl From for $name { + fn from(value: u32) -> Self { + Self::new(value) + } + } + + impl From<$name> for u32 { + fn from(value: $name) -> Self { + value.get() + } + } + }; +} + +segment_id!(BlockId, "Identifier of a postings block within a segment."); +segment_id!(FilterExprId, "Identifier of a stored filter expression."); +segment_id!( + SegmentOrd, + "Ordinal position of a segment within a multi-segment index." +); +segment_id!( + SegmentLocalDocId, + "Document identifier local to a single segment (segment-relative doc ID)." +); +segment_id!( + TermFreq, + "Term frequency: occurrences of a term in a document. A segment-resident value \ + type (not an identifier) carried alongside `SegmentLocalDocId` in postings." +); + +#[cfg(test)] +mod tests { + use super::*; + + /// SCENARIO-0005: ID type serialization round-trip — each type serializes to + /// a fixed-width 4-byte little-endian value and round-trips losslessly as both + /// a single value and a zero-copy slice view. + #[test] + fn test_single_value_little_endian_roundtrip() { + let id = BlockId::new(0xDEAD_BEEF); + + // Serialized form is exactly the 4-byte little-endian encoding. + let bytes = bytemuck::bytes_of(&id); + assert_eq!(bytes, &0xDEAD_BEEF_u32.to_le_bytes()); + assert_eq!(bytes.len(), 4); + + // Zero-copy view back to the typed value. + let back: &BlockId = bytemuck::from_bytes(bytes); + assert_eq!(*back, id); + assert_eq!(back.get(), 0xDEAD_BEEF); + } + + #[test] + fn test_slice_zero_copy_view_roundtrip() { + let ids = [ + SegmentLocalDocId::new(1), + SegmentLocalDocId::new(0), + SegmentLocalDocId::new(u32::MAX), + SegmentLocalDocId::new(42), + ]; + + // The array's bytes are the concatenated little-endian values. + let bytes: &[u8] = bytemuck::cast_slice(&ids); + assert_eq!(bytes.len(), 4 * ids.len()); + assert_eq!(&bytes[0..4], &1_u32.to_le_bytes()); + assert_eq!(&bytes[12..16], &42_u32.to_le_bytes()); + + // A &[u8] is a zero-copy view of &[SegmentLocalDocId]. + let view: &[SegmentLocalDocId] = bytemuck::cast_slice(bytes); + assert_eq!(view, ids.as_slice()); + } + + #[test] + fn test_validated_reads_use_try_cast_variants() { + // SCENARIO-0005 (AC-2 validated-read obligation): reads from untrusted + // segment bytes go through the fallible `try_*` casts, which return `Err` + // on a malformed slice instead of panicking. + + // Correctly-sized 4-byte slice -> Ok. + let raw = 0x1234_5678_u32.to_le_bytes(); + let ok: &BlockId = bytemuck::try_from_bytes(&raw).expect("4-byte slice is a valid BlockId"); + assert_eq!(ok.get(), 0x1234_5678); + + // Wrong-length slice -> Err, never a panic. + let too_short = [0_u8; 3]; + assert!(bytemuck::try_from_bytes::(&too_short).is_err()); + + // try_cast_slice yields a zero-copy &[Id] view for an exact multiple... + let ids = [SegmentLocalDocId::new(5), SegmentLocalDocId::new(6)]; + let bytes: &[u8] = bytemuck::cast_slice(&ids); + let view: &[SegmentLocalDocId] = + bytemuck::try_cast_slice(bytes).expect("8 bytes round-trips to 2 ids"); + assert_eq!(view, ids.as_slice()); + + // ...and rejects a length that is not a whole number of ids. + let ragged = [0_u8; 7]; + assert!(bytemuck::try_cast_slice::(&ragged).is_err()); + } + + #[test] + fn test_unaligned_view_from_offset() { + // [u8; 4] storage is alignment-1, so views work from any byte offset + // (mmap safety) — bytemuck's cast succeeds regardless of source alignment. + let mut buf = [0_u8; 5]; + buf[1..5].copy_from_slice(&7_u32.to_le_bytes()); + let id: &SegmentOrd = bytemuck::from_bytes(&buf[1..5]); + assert_eq!(id.get(), 7); + } + + #[test] + fn test_all_id_types_roundtrip_and_convert() { + assert_eq!(BlockId::new(10).get(), 10); + assert_eq!(FilterExprId::new(20).get(), 20); + assert_eq!(SegmentOrd::new(30).get(), 30); + assert_eq!(SegmentLocalDocId::new(40).get(), 40); + + // u32 <-> ID conversions. + assert_eq!(u32::from(FilterExprId::from(99_u32)), 99); + } + + #[test] + fn test_ordering_is_numeric_not_byte_order() { + // Little-endian byte storage must not corrupt numeric ordering. + assert!(BlockId::new(2) < BlockId::new(256)); + assert!(SegmentLocalDocId::new(0x00FF_0000) < SegmentLocalDocId::new(0x0100_0000)); + } + + #[test] + fn test_debug_shows_numeric_value() { + extern crate alloc; + assert_eq!(alloc::format!("{:?}", BlockId::new(7)), "BlockId(7)"); + } +} diff --git a/crates/leit_index/Cargo.toml b/crates/leit_index/Cargo.toml index b6a08edf..62347118 100644 --- a/crates/leit_index/Cargo.toml +++ b/crates/leit_index/Cargo.toml @@ -20,6 +20,7 @@ leit_text.workspace = true [features] default = ["std"] +bench-internals = [] std = [ "leit_core/std", "leit_text/std", diff --git a/crates/leit_index/src/memory.rs b/crates/leit_index/src/memory.rs index ae442ed2..75202669 100644 --- a/crates/leit_index/src/memory.rs +++ b/crates/leit_index/src/memory.rs @@ -24,9 +24,14 @@ pub(crate) struct TermEntry { pub(crate) term: String, } +/// A single posting: a document ID and its term frequency for a term. +/// +/// Postings are aggregated per term and stored in doc-sorted order (ascending doc ID). #[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] pub(crate) struct PostingEntry { + /// Document identifier (segment-local, u32). pub(crate) doc_id: u32, + /// Term frequency (raw count of term occurrences in the document's field). pub(crate) term_freq: u32, } @@ -135,6 +140,22 @@ impl InMemoryIndex { &self.postings } + /// Snapshot postings as primitive tuples for out-of-crate benchmarks. + #[cfg(feature = "bench-internals")] + #[doc(hidden)] + pub fn benchmark_postings(&self) -> Vec> { + self.postings + .values() + .filter(|postings| !postings.is_empty()) + .map(|postings| { + postings + .iter() + .map(|posting| (posting.doc_id, posting.term_freq)) + .collect() + }) + .collect() + } + fn avg_field_doc_length(&self, field: FieldId) -> f32 { let Some(stats) = self.field_stats.get(&field) else { return 0.0; diff --git a/crates/leit_postings/src/codec.rs b/crates/leit_postings/src/codec.rs new file mode 100644 index 00000000..15c1a19e --- /dev/null +++ b/crates/leit_postings/src/codec.rs @@ -0,0 +1,1260 @@ +// Copyright 2026 the Leit Authors +// SPDX-License-Identifier: Apache-2.0 OR MIT + +//! Compressed postings codecs for efficient storage and traversal. +//! +//! This module provides codec implementations for encoding and decoding postings lists. +//! Multiple codec strategies are supported to balance decode cost and memory footprint: +//! +//! - **`DeltaVarint`**: Single-block encoding using delta-encoded doc IDs and varint-encoded TFs. +//! - **`BlockDelta`**: Multi-block encoding with 128-doc blocks, each independently decodable. +//! +//! ## Codec ID marker +//! +//! Encoded postings are prefixed by a 1-byte `CodecId` to support multiple codec implementations. +//! See DEC-12 in the architectural decisions for the full specification. + +use alloc::vec::Vec; +use core::fmt; +use leit_core::{SegmentLocalDocId, TermFreq}; + +/// Fixed block size for `BlockDelta` codec: 128 documents per block. +/// +/// This constant is defined in one place (DEC-11) so that a future codec may tune it +/// without a format break. The block count is encoded per block, not assumed by readers. +pub const BLOCK_DOC_COUNT: usize = 128; + +/// Codec identifier for selecting among multiple postings codec implementations. +/// +/// Each postings list is prefixed by a single byte codec marker (1-byte `CodecId`). +/// Decoders read this marker to dispatch to the correct codec implementation. +#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord)] +#[repr(u8)] +pub enum CodecId { + /// Delta-encoded doc IDs + varint-encoded TFs in a single stream (no block structure). + DeltaVarint = 0, + /// Block-based codec: 128-doc blocks, each independently decodable. + BlockDelta = 1, +} + +impl CodecId { + /// Convert a byte to a `CodecId`, returning `None` if the byte is not a valid marker. + pub fn from_u8(byte: u8) -> Option { + match byte { + 0 => Some(Self::DeltaVarint), + 1 => Some(Self::BlockDelta), + _ => None, + } + } + + /// Convert this `CodecId` to a byte. + pub const fn to_u8(self) -> u8 { + self as u8 + } +} + +/// Errors that can occur during codec operations. +#[derive(Clone, Debug, PartialEq, Eq)] +pub enum CodecError { + /// The byte stream is truncated or incomplete. + Truncated, + /// The codec ID marker is not recognized. + BadMarker(u8), + /// Invalid block count in block header. + InvalidBlockCount, + /// Invalid varint encoding. + InvalidVarint, + /// A block header's `first_doc`/`last_doc` range does not match its decoded doc stream. + BlockHeaderMismatch, +} + +impl fmt::Display for CodecError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Self::Truncated => write!(f, "byte stream truncated"), + Self::BadMarker(byte) => write!(f, "unrecognized codec ID: {byte}"), + Self::InvalidBlockCount => write!(f, "invalid block count"), + Self::InvalidVarint => write!(f, "invalid varint encoding"), + Self::BlockHeaderMismatch => { + write!(f, "block header doc-range does not match decoded stream") + } + } + } +} + +#[cfg(feature = "std")] +impl core::error::Error for CodecError {} + +/// Core codec interface for encoding and decoding postings. +/// +/// A codec is responsible for: +/// - Encoding a sequence of (`SegmentLocalDocId`, `TermFreq`) pairs into a compressed byte stream. +/// - Decoding the byte stream back into the original pairs. +/// +/// ## Encode input +/// +/// The input to `encode()` is a slice of `(SegmentLocalDocId, TermFreq)` tuples. +/// Doc IDs must be doc-sorted (ascending order) for delta encoding to be effective. +/// +/// ## Decode output +/// +/// The decode API writes into **caller-provided output buffers** (`&mut Vec` +/// and `&mut Vec`) rather than allocating owned decode structures. This design keeps +/// the codec layer scratch-ownership-agnostic; see DEC-12 and the TODO comment below. +/// +/// The caller may reuse these buffers. Decoders clear them before decoding and +/// leave both buffers empty if decoding fails. +/// +/// ## Codec marker +/// +/// The encoded byte stream includes a 1-byte `CodecId` prefix. +/// The `encode()` method includes this prefix in the returned bytes. +/// The `decode()` method expects the bytes to start with the marker. +pub trait Codec { + /// Return the codec ID for this implementation. + fn id(&self) -> CodecId; + + /// Encode a sequence of (`SegmentLocalDocId`, `TermFreq`) tuples into a compressed byte stream. + /// + /// The returned bytes include a 1-byte codec ID prefix. + /// + /// The v1 codec layer encodes only `(SegmentLocalDocId, TermFreq)`. Posting **positions** + /// (`Posting::positions`) are intentionally out of scope here — positions/TF + /// layering is decided in ITER-0003 (STORY-0080); a future codec or a parallel + /// positions section will carry them without changing this format. + /// + /// # Arguments + /// + /// - `postings`: slice of (`SegmentLocalDocId`, `TermFreq`) pairs, must be doc-sorted (ascending). + /// + /// # Panics + /// + /// May panic if postings are not doc-sorted. + fn encode(&self, postings: &[(SegmentLocalDocId, TermFreq)]) -> Vec; + + /// Decode a byte stream into doc IDs and term frequencies. + /// + /// This method writes decoded values into caller-provided output buffers. + /// The input bytes must start with a valid codec ID marker. + /// + /// The method validates the codec ID and delegates to the appropriate codec + /// decoder, which writes exactly `len(postings)` values into `out_docs` and + /// `out_tfs` in doc-ascending order. + /// + /// # Arguments + /// + /// - `bytes`: compressed postings with a 1-byte codec ID prefix. + /// - `out_docs`: output buffer for decoded doc IDs (will be cleared and refilled). + /// - `out_tfs`: output buffer for decoded term frequencies (will be cleared and refilled). + /// + /// # Returns + /// + /// `Ok(())` on success, or a `CodecError` if decoding fails. On error, both + /// output buffers are empty. + /// + /// # TODO(ITER-0003): `DecodeScratch` wrapper + /// + /// The decode-scratch ownership model (e.g., a `DecodeScratch` struct holding + /// these buffers and reusable across multiple cursors) is designed and built + /// in ITER-0003 (STORY-0079). This codec layer intentionally does NOT define + /// a decode-scratch type so that decision is not boxed in here. + fn decode( + &self, + bytes: &[u8], + out_docs: &mut Vec, + out_tfs: &mut Vec, + ) -> Result<(), CodecError>; +} + +/// Helper to encode a u32 as LEB128 varint. +/// +/// Writes 1–5 bytes into `out` (a `u32` LEB128 is at most 5 bytes, so a fixed +/// `[u8; 5]` buffer can never overflow), returns the number of bytes written. +fn encode_varint(value: u32, out: &mut [u8; 5]) -> usize { + let mut v = value; + let mut len = 0; + loop { + let mut byte = (v & 0x7f) as u8; + v >>= 7; + if v != 0 { + byte |= 0x80; + } + out[len] = byte; + len += 1; + if v == 0 { + break; + } + } + len +} + +/// Helper to decode a u32 from LEB128 varint. +/// +/// Returns `(value, bytes_consumed)` on success, or `CodecError::InvalidVarint` on failure. +fn decode_varint(bytes: &[u8]) -> Result<(u32, usize), CodecError> { + let mut value = 0_u32; + let mut shift = 0; + let mut pos = 0; + + loop { + if pos >= bytes.len() { + return Err(CodecError::Truncated); + } + + let byte = bytes[pos]; + pos += 1; + + if shift == 28 && (byte & 0x7f) > 0x0f { + return Err(CodecError::InvalidVarint); + } + + value |= ((byte & 0x7f) as u32) << shift; + + if (byte & 0x80) == 0 { + return Ok((value, pos)); + } + + shift += 7; + if shift >= 32 { + return Err(CodecError::InvalidVarint); + } + } +} + +/// Delta-encoding codec: single-block varint encoding. +/// +/// This codec encodes the postings list as a single block with: +/// - Delta-encoded doc IDs (deltas from previous doc, first delta from 0). +/// - Varint-encoded term frequencies (parallel to doc stream). +/// +/// This is a simpler, single-block alternative to `BlockDelta` for small postings lists. +#[derive(Clone, Copy, Debug)] +pub struct DeltaVarintCodec; + +impl Codec for DeltaVarintCodec { + fn id(&self) -> CodecId { + CodecId::DeltaVarint + } + + fn encode(&self, postings: &[(SegmentLocalDocId, TermFreq)]) -> Vec { + let mut result = Vec::with_capacity(postings.len() * 5 + 10); + result.push(CodecId::DeltaVarint.to_u8()); + + let mut buf = [0_u8; 5]; + let mut prev_doc = 0_u32; + + for (doc_id, tf) in postings { + // Postings MUST be doc-sorted ascending (the documented precondition). + // `checked_sub` turns a violation into a deterministic panic rather than + // a silently-wrapping delta that would corrupt the encoded stream. + let doc_id_u32 = doc_id.get(); + let delta = doc_id_u32 + .checked_sub(prev_doc) + .expect("postings must be doc-sorted ascending"); + let bytes_written = encode_varint(delta, &mut buf); + result.extend_from_slice(&buf[..bytes_written]); + + let bytes_written = encode_varint(tf.get(), &mut buf); + result.extend_from_slice(&buf[..bytes_written]); + + prev_doc = doc_id_u32; + } + + result + } + + fn decode( + &self, + bytes: &[u8], + out_docs: &mut Vec, + out_tfs: &mut Vec, + ) -> Result<(), CodecError> { + out_docs.clear(); + out_tfs.clear(); + + let result = (|| { + if bytes.is_empty() { + return Err(CodecError::Truncated); + } + + let marker = bytes[0]; + if marker != CodecId::DeltaVarint.to_u8() { + return Err(CodecError::BadMarker(marker)); + } + + let mut pos = 1; + let mut prev_doc = 0_u32; + + while pos < bytes.len() { + let (delta, delta_len) = decode_varint(&bytes[pos..])?; + pos += delta_len; + + if pos >= bytes.len() { + return Err(CodecError::Truncated); + } + + let doc_id = prev_doc + .checked_add(delta) + .ok_or(CodecError::InvalidVarint)?; + let (tf, tf_len) = decode_varint(&bytes[pos..])?; + pos += tf_len; + + out_docs.push(SegmentLocalDocId::new(doc_id)); + out_tfs.push(TermFreq::new(tf)); + prev_doc = doc_id; + } + + Ok(()) + })(); + + if result.is_err() { + out_docs.clear(); + out_tfs.clear(); + } + result + } +} + +/// Block-delta codec: multi-block encoding with 128-doc blocks. +/// +/// This codec divides postings into fixed-size blocks of `BLOCK_DOC_COUNT` documents. +/// Each block is independently decodable, enabling selective decode and block-aware +/// traversal for future pruning (Phase 3). +/// +/// ## Block format +/// +/// The block format (not Rust code): +/// +/// ```text +/// block := block_header doc_id_stream tf_stream +/// block_header := varint(doc_count) varint(first_doc) varint(last_doc) varint(doc_bytes_len) +/// doc_id_stream := varint(first_delta) varint(delta)* # deltas from previous doc +/// tf_stream := varint(tf)* # one per doc, parallel to doc stream +/// ``` +/// +/// The `doc_count` in the header is used by readers but is implicit from input length. +/// The `first_doc` is stored as an absolute value (delta from 0), so blocks are self-contained. +/// The `doc_bytes_len` allows readers to skip to the TF stream without decoding doc deltas. +#[derive(Clone, Copy, Debug)] +pub struct BlockDeltaCodec; + +impl Codec for BlockDeltaCodec { + fn id(&self) -> CodecId { + CodecId::BlockDelta + } + + fn encode(&self, postings: &[(SegmentLocalDocId, TermFreq)]) -> Vec { + let mut result = Vec::with_capacity(postings.len() * 5 + 100); + result.push(CodecId::BlockDelta.to_u8()); + + let mut buf = [0_u8; 5]; + + for block_chunk in postings.chunks(BLOCK_DOC_COUNT) { + let doc_count = block_chunk.len(); + let first_doc = block_chunk[0].0.get(); + let last_doc = block_chunk[block_chunk.len() - 1].0.get(); + + // Encode block header: doc_count, first_doc, last_doc, doc_bytes_len. + // We'll first encode doc stream in a temporary buffer to get its length. + + // SAFETY: doc_count is usize from chunk size, bounded by BLOCK_DOC_COUNT (128). + #[expect( + clippy::cast_possible_truncation, + reason = "doc_count bounded by BLOCK_DOC_COUNT" + )] + let bytes = encode_varint(doc_count as u32, &mut buf); + result.extend_from_slice(&buf[..bytes]); + + let bytes = encode_varint(first_doc, &mut buf); + result.extend_from_slice(&buf[..bytes]); + + let bytes = encode_varint(last_doc, &mut buf); + result.extend_from_slice(&buf[..bytes]); + + // Encode doc ID stream in a temporary buffer to determine its length. + let mut doc_stream = Vec::new(); + let mut prev_doc = 0_u32; + for (doc_id, _) in block_chunk { + // Doc-sorted precondition (see DeltaVarintCodec::encode): a violation + // panics deterministically rather than wrapping into a corrupt delta. + let doc_id_u32 = doc_id.get(); + let delta = doc_id_u32 + .checked_sub(prev_doc) + .expect("postings must be doc-sorted ascending"); + let bytes = encode_varint(delta, &mut buf); + doc_stream.extend_from_slice(&buf[..bytes]); + prev_doc = doc_id_u32; + } + + // Now encode doc_bytes_len. + // SAFETY: doc_bytes_len is per-block, bounded by BLOCK_DOC_COUNT * varint_max_bytes. + let doc_bytes_len = doc_stream.len(); + #[expect( + clippy::cast_possible_truncation, + reason = "doc_bytes_len bounded by BLOCK_DOC_COUNT * 5" + )] + let bytes = encode_varint(doc_bytes_len as u32, &mut buf); + result.extend_from_slice(&buf[..bytes]); + + // Append the doc stream. + result.extend_from_slice(&doc_stream); + + // Encode TF values. + for (_, tf) in block_chunk { + let bytes = encode_varint(tf.get(), &mut buf); + result.extend_from_slice(&buf[..bytes]); + } + } + + result + } + + fn decode( + &self, + bytes: &[u8], + out_docs: &mut Vec, + out_tfs: &mut Vec, + ) -> Result<(), CodecError> { + out_docs.clear(); + out_tfs.clear(); + + let result = (|| { + if bytes.is_empty() { + return Err(CodecError::Truncated); + } + + let marker = bytes[0]; + if marker != CodecId::BlockDelta.to_u8() { + return Err(CodecError::BadMarker(marker)); + } + + let mut pos = 1; + + while pos < bytes.len() { + // Decode block header. + let (doc_count, bytes_read) = decode_varint(&bytes[pos..])?; + pos += bytes_read; + let doc_count = doc_count as usize; + + if doc_count == 0 || doc_count > BLOCK_DOC_COUNT { + return Err(CodecError::InvalidBlockCount); + } + + let (first_doc, bytes_read) = decode_varint(&bytes[pos..])?; + pos += bytes_read; + + let (last_doc, bytes_read) = decode_varint(&bytes[pos..])?; + pos += bytes_read; + + let (doc_bytes_len, bytes_read) = decode_varint(&bytes[pos..])?; + pos += bytes_read; + let doc_bytes_len = doc_bytes_len as usize; + + let doc_stream_end = pos + .checked_add(doc_bytes_len) + .filter(|end| *end <= bytes.len()) + .ok_or(CodecError::Truncated)?; + + // Decode doc ID stream. + let doc_stream_start = pos; + let doc_stream = &bytes[doc_stream_start..doc_stream_end]; + + let mut doc_pos = 0; + let mut prev_doc = 0_u32; + let block_doc_start = out_docs.len(); + + for _ in 0..doc_count { + if doc_pos >= doc_stream.len() { + return Err(CodecError::Truncated); + } + let (delta, bytes_read) = decode_varint(&doc_stream[doc_pos..])?; + doc_pos += bytes_read; + + let doc_id = prev_doc + .checked_add(delta) + .ok_or(CodecError::InvalidVarint)?; + out_docs.push(SegmentLocalDocId::new(doc_id)); + prev_doc = doc_id; + } + + if doc_pos != doc_stream.len() { + return Err(CodecError::InvalidVarint); + } + + // Validate the block header's doc-range against the decoded stream. + // This makes `first_doc`/`last_doc` (carried for ITER-0003 block-skip and + // ITER-0005 WAND doc-range) self-checking rather than dead bytes, and + // detects corruption. `prev_doc` now holds the block's last decoded doc. + if out_docs[block_doc_start].get() != first_doc || prev_doc != last_doc { + return Err(CodecError::BlockHeaderMismatch); + } + + pos = doc_stream_end; + + // Decode TF stream (parallel to doc stream). + for _ in 0..doc_count { + if pos >= bytes.len() { + return Err(CodecError::Truncated); + } + let (tf, bytes_read) = decode_varint(&bytes[pos..])?; + pos += bytes_read; + out_tfs.push(TermFreq::new(tf)); + } + } + + Ok(()) + })(); + + if result.is_err() { + out_docs.clear(); + out_tfs.clear(); + } + result + } +} + +/// Decode a postings byte stream using any codec. +/// +/// This free function dispatches on the codec ID marker and calls the appropriate +/// codec's decode method. +/// +/// # Arguments +/// +/// - `bytes`: the encoded postings (must start with a valid `CodecId` marker). +/// - `out_docs`: output buffer for doc IDs. +/// - `out_tfs`: output buffer for term frequencies. +/// +/// # Returns +/// +/// `Ok(())` on success, or a `CodecError` on failure. On error, both output +/// buffers are empty. +pub fn decode_any( + bytes: &[u8], + out_docs: &mut Vec, + out_tfs: &mut Vec, +) -> Result<(), CodecError> { + out_docs.clear(); + out_tfs.clear(); + + if bytes.is_empty() { + return Err(CodecError::Truncated); + } + + let marker = bytes[0]; + match CodecId::from_u8(marker) { + Some(CodecId::DeltaVarint) => DeltaVarintCodec.decode(bytes, out_docs, out_tfs), + Some(CodecId::BlockDelta) => BlockDeltaCodec.decode(bytes, out_docs, out_tfs), + None => Err(CodecError::BadMarker(marker)), + } +} + +#[cfg(test)] +mod tests { + use super::*; + use alloc::vec; + use alloc::vec::Vec; + + // ===== Varint Tests ===== + + #[test] + fn test_varint_encode_decode_zero() { + let mut buf = [0_u8; 5]; + let len = encode_varint(0, &mut buf); + assert_eq!(len, 1); + assert_eq!(buf[0], 0); + + let (decoded, bytes_read) = decode_varint(&buf[..len]).unwrap(); + assert_eq!(decoded, 0); + assert_eq!(bytes_read, 1); + } + + #[test] + fn test_varint_encode_decode_small() { + let mut buf = [0_u8; 5]; + let len = encode_varint(42, &mut buf); + assert_eq!(len, 1); + + let (decoded, bytes_read) = decode_varint(&buf[..len]).unwrap(); + assert_eq!(decoded, 42); + assert_eq!(bytes_read, 1); + } + + #[test] + fn test_varint_encode_decode_large() { + let mut buf = [0_u8; 5]; + let len = encode_varint(16384, &mut buf); + assert_eq!(len, 3); + + let (decoded, bytes_read) = decode_varint(&buf[..len]).unwrap(); + assert_eq!(decoded, 16384); + assert_eq!(bytes_read, 3); + } + + #[test] + fn test_varint_encode_decode_max() { + let mut buf = [0_u8; 5]; + let len = encode_varint(u32::MAX, &mut buf); + assert_eq!(len, 5); + + let (decoded, bytes_read) = decode_varint(&buf[..len]).unwrap(); + assert_eq!(decoded, u32::MAX); + assert_eq!(bytes_read, 5); + } + + #[test] + fn test_varint_decode_truncated() { + let bytes = [0x80]; // Incomplete varint. + let result = decode_varint(&bytes); + assert_eq!(result, Err(CodecError::Truncated)); + } + + #[test] + fn test_varint_decode_rejects_five_byte_overflow() { + let bytes = [0x80, 0x80, 0x80, 0x80, 0x10]; + assert_eq!(decode_varint(&bytes), Err(CodecError::InvalidVarint)); + } + + // ===== DeltaVarint Codec Tests ===== + + #[test] + fn test_delta_varint_round_trip_empty() { + let codec = DeltaVarintCodec; + let postings: &[(SegmentLocalDocId, TermFreq)] = &[]; + + let encoded = codec.encode(postings); + assert_eq!(encoded.len(), 1); // Only the marker byte. + + let mut docs = Vec::new(); + let mut tfs = Vec::new(); + codec.decode(&encoded, &mut docs, &mut tfs).unwrap(); + + assert_eq!(docs.len(), 0); + assert_eq!(tfs.len(), 0); + } + + #[test] + fn test_delta_varint_round_trip_single() { + let codec = DeltaVarintCodec; + let postings = [(SegmentLocalDocId::new(100), TermFreq::new(5))]; + + let encoded = codec.encode(&postings); + + let mut docs = Vec::new(); + let mut tfs = Vec::new(); + codec.decode(&encoded, &mut docs, &mut tfs).unwrap(); + + assert_eq!(docs, vec![SegmentLocalDocId::new(100)]); + assert_eq!(tfs, vec![TermFreq::new(5)]); + } + + #[test] + fn test_delta_varint_round_trip_multiple() { + let codec = DeltaVarintCodec; + let postings = [ + (SegmentLocalDocId::new(10), TermFreq::new(1)), + (SegmentLocalDocId::new(20), TermFreq::new(2)), + (SegmentLocalDocId::new(35), TermFreq::new(3)), + (SegmentLocalDocId::new(100), TermFreq::new(10)), + ]; + + let encoded = codec.encode(&postings); + + let mut docs = Vec::new(); + let mut tfs = Vec::new(); + codec.decode(&encoded, &mut docs, &mut tfs).unwrap(); + + assert_eq!( + docs, + vec![ + SegmentLocalDocId::new(10), + SegmentLocalDocId::new(20), + SegmentLocalDocId::new(35), + SegmentLocalDocId::new(100), + ] + ); + assert_eq!( + tfs, + vec![ + TermFreq::new(1), + TermFreq::new(2), + TermFreq::new(3), + TermFreq::new(10), + ] + ); + } + + #[test] + fn test_delta_varint_error_clears_partial_output() { + let codec = DeltaVarintCodec; + let bytes = [CodecId::DeltaVarint.to_u8(), 1, 1, 0x80]; + let mut docs = vec![SegmentLocalDocId::new(99)]; + let mut tfs = vec![TermFreq::new(99)]; + + assert_eq!( + codec.decode(&bytes, &mut docs, &mut tfs), + Err(CodecError::Truncated) + ); + assert!(docs.is_empty()); + assert!(tfs.is_empty()); + } + + #[test] + fn test_decode_any_bad_marker_clears_output() { + let mut docs = vec![SegmentLocalDocId::new(99)]; + let mut tfs = vec![TermFreq::new(99)]; + + assert_eq!( + decode_any(&[u8::MAX], &mut docs, &mut tfs), + Err(CodecError::BadMarker(u8::MAX)) + ); + assert!(docs.is_empty()); + assert!(tfs.is_empty()); + } + + #[test] + fn test_delta_varint_compression() { + let codec = DeltaVarintCodec; + let postings = [ + (SegmentLocalDocId::new(100), TermFreq::new(5)), + (SegmentLocalDocId::new(105), TermFreq::new(3)), + (SegmentLocalDocId::new(110), TermFreq::new(7)), + (SegmentLocalDocId::new(200), TermFreq::new(2)), + ]; + + let encoded = codec.encode(&postings); + let uncompressed = postings.len() * 8; // 4 bytes doc + 4 bytes tf. + + // Encoded should be smaller than uncompressed. + // With deltas and varints, this should be significantly smaller. + assert!( + encoded.len() < uncompressed, + "encoded {} >= uncompressed {}", + encoded.len(), + uncompressed + ); + } + + #[test] + fn test_delta_varint_bad_marker() { + let codec = DeltaVarintCodec; + let bad_bytes = [5_u8, 100, 5]; // Bad marker. + + let mut docs = Vec::new(); + let mut tfs = Vec::new(); + let result = codec.decode(&bad_bytes, &mut docs, &mut tfs); + + assert_eq!(result, Err(CodecError::BadMarker(5))); + } + + // ===== BlockDelta Codec Tests ===== + + #[test] + fn test_block_delta_round_trip_empty() { + let codec = BlockDeltaCodec; + let postings: &[(SegmentLocalDocId, TermFreq)] = &[]; + + let encoded = codec.encode(postings); + assert_eq!(encoded.len(), 1); // Only marker. + + let mut docs = Vec::new(); + let mut tfs = Vec::new(); + codec.decode(&encoded, &mut docs, &mut tfs).unwrap(); + + assert_eq!(docs.len(), 0); + assert_eq!(tfs.len(), 0); + } + + #[test] + fn test_block_delta_round_trip_single_block() { + let codec = BlockDeltaCodec; + let postings: Vec<(SegmentLocalDocId, TermFreq)> = (0..50) + .map(|i| (SegmentLocalDocId::new(i * 10), TermFreq::new(i % 10 + 1))) + .collect(); + + let encoded = codec.encode(&postings); + + let mut docs = Vec::new(); + let mut tfs = Vec::new(); + codec.decode(&encoded, &mut docs, &mut tfs).unwrap(); + + let expected_docs: Vec = postings.iter().map(|(d, _)| *d).collect(); + let expected_tfs: Vec = postings.iter().map(|(_, t)| *t).collect(); + + assert_eq!(docs, expected_docs); + assert_eq!(tfs, expected_tfs); + } + + #[test] + fn test_block_delta_round_trip_multiple_blocks() { + let codec = BlockDeltaCodec; + let postings: Vec<(SegmentLocalDocId, TermFreq)> = (0..300) + .map(|i| (SegmentLocalDocId::new(i * 5), TermFreq::new(i % 7 + 1))) + .collect(); + + let encoded = codec.encode(&postings); + + let mut docs = Vec::new(); + let mut tfs = Vec::new(); + codec.decode(&encoded, &mut docs, &mut tfs).unwrap(); + + let expected_docs: Vec = postings.iter().map(|(d, _)| *d).collect(); + let expected_tfs: Vec = postings.iter().map(|(_, t)| *t).collect(); + + assert_eq!(docs, expected_docs); + assert_eq!(tfs, expected_tfs); + } + + #[test] + fn test_block_delta_compression() { + let codec = BlockDeltaCodec; + let postings: Vec<(SegmentLocalDocId, TermFreq)> = (0..200) + .map(|i| (SegmentLocalDocId::new(i * 3), TermFreq::new(i % 5 + 1))) + .collect(); + + let encoded = codec.encode(&postings); + let uncompressed = postings.len() * 8; + + assert!( + encoded.len() < uncompressed, + "encoded {} >= uncompressed {}", + encoded.len(), + uncompressed + ); + } + + #[test] + fn test_block_delta_independent_decode() { + let codec = BlockDeltaCodec; + // Create a large postings list (> BLOCK_DOC_COUNT). + let postings: Vec<(SegmentLocalDocId, TermFreq)> = (0..300) + .map(|i| (SegmentLocalDocId::new(i * 2), TermFreq::new(i % 3 + 1))) + .collect(); + + let encoded = codec.encode(&postings); + + // Verify full decode. + let mut all_docs = Vec::new(); + let mut all_tfs = Vec::new(); + codec.decode(&encoded, &mut all_docs, &mut all_tfs).unwrap(); + + assert_eq!(all_docs.len(), 300); + assert_eq!(all_tfs.len(), 300); + + // Verify that blocks are self-contained by decoding and checking structure. + // (A full independent block decode would require a separate `decode_block` method, + // which is deferred but noted in AC-2 proof; for now we verify the format is correct + // by full decode and structural checks.) + let expected_docs: Vec = postings.iter().map(|(d, _)| *d).collect(); + let expected_tfs: Vec = postings.iter().map(|(_, t)| *t).collect(); + + assert_eq!(all_docs, expected_docs); + assert_eq!(all_tfs, expected_tfs); + } + + #[test] + fn test_block_delta_bad_marker() { + let codec = BlockDeltaCodec; + let bad_bytes = [99_u8]; // Bad marker. + + let mut docs = Vec::new(); + let mut tfs = Vec::new(); + let result = codec.decode(&bad_bytes, &mut docs, &mut tfs); + + assert_eq!(result, Err(CodecError::BadMarker(99))); + } + + // ===== decode_any Tests ===== + + #[test] + fn test_decode_any_delta_varint() { + let codec = DeltaVarintCodec; + let postings = [ + (SegmentLocalDocId::new(10), TermFreq::new(1)), + (SegmentLocalDocId::new(20), TermFreq::new(2)), + ]; + let encoded = codec.encode(&postings); + + let mut docs = Vec::new(); + let mut tfs = Vec::new(); + decode_any(&encoded, &mut docs, &mut tfs).unwrap(); + + assert_eq!( + docs, + vec![SegmentLocalDocId::new(10), SegmentLocalDocId::new(20)] + ); + assert_eq!(tfs, vec![TermFreq::new(1), TermFreq::new(2)]); + } + + #[test] + fn test_decode_any_block_delta() { + let codec = BlockDeltaCodec; + let postings = [ + (SegmentLocalDocId::new(10), TermFreq::new(1)), + (SegmentLocalDocId::new(20), TermFreq::new(2)), + ]; + let encoded = codec.encode(&postings); + + let mut docs = Vec::new(); + let mut tfs = Vec::new(); + decode_any(&encoded, &mut docs, &mut tfs).unwrap(); + + assert_eq!( + docs, + vec![SegmentLocalDocId::new(10), SegmentLocalDocId::new(20)] + ); + assert_eq!(tfs, vec![TermFreq::new(1), TermFreq::new(2)]); + } + + #[test] + fn test_decode_any_bad_marker() { + let bad_bytes = [99_u8]; + + let mut docs = Vec::new(); + let mut tfs = Vec::new(); + let result = decode_any(&bad_bytes, &mut docs, &mut tfs); + + assert_eq!(result, Err(CodecError::BadMarker(99))); + } + + // ===== Integration: Size Reduction Tests ===== + + #[test] + fn test_delta_varint_size_reduction_zipfian() { + // Simulate a Zipfian-like distribution (long list with concentrated doc IDs). + let codec = DeltaVarintCodec; + let postings: Vec<(SegmentLocalDocId, TermFreq)> = (0..100) + .map(|i| { + let doc_id = (i * 50) as u32; // Sparse docs, large deltas within blocks. + let tf = (1 + (i % 10)) as u32; + (SegmentLocalDocId::new(doc_id), TermFreq::new(tf)) + }) + .collect(); + + let encoded = codec.encode(&postings); + let uncompressed_bytes = postings.len() * 8; + + assert!(encoded.len() < uncompressed_bytes); + } + + #[test] + fn test_block_delta_size_reduction_zipfian() { + let codec = BlockDeltaCodec; + let postings: Vec<(SegmentLocalDocId, TermFreq)> = (0..200) + .map(|i| { + let doc_id = (i * 50) as u32; + let tf = (1 + (i % 10)) as u32; + (SegmentLocalDocId::new(doc_id), TermFreq::new(tf)) + }) + .collect(); + + let encoded = codec.encode(&postings); + let uncompressed_bytes = postings.len() * 8; + + assert!(encoded.len() < uncompressed_bytes); + } + + #[test] + fn test_block_delta_exactly_128_docs() { + let codec = BlockDeltaCodec; + let postings: Vec<(SegmentLocalDocId, TermFreq)> = (0..128) + .map(|i| { + ( + SegmentLocalDocId::new(i as u32), + TermFreq::new((i % 7 + 1) as u32), + ) + }) + .collect(); + + let encoded = codec.encode(&postings); + let mut docs = Vec::new(); + let mut tfs = Vec::new(); + codec.decode(&encoded, &mut docs, &mut tfs).unwrap(); + + assert_eq!(docs.len(), 128, "Should have exactly 128 docs"); + let expected_docs: Vec = + (0..128).map(|i| SegmentLocalDocId::new(i as u32)).collect(); + assert_eq!(docs, expected_docs); + } + + #[test] + fn test_block_delta_129_docs() { + let codec = BlockDeltaCodec; + let postings: Vec<(SegmentLocalDocId, TermFreq)> = (0..129) + .map(|i| { + ( + SegmentLocalDocId::new(i as u32), + TermFreq::new((i % 7 + 1) as u32), + ) + }) + .collect(); + + let encoded = codec.encode(&postings); + let mut docs = Vec::new(); + let mut tfs = Vec::new(); + codec.decode(&encoded, &mut docs, &mut tfs).unwrap(); + + assert_eq!(docs.len(), 129, "Should have 129 docs in 2 blocks"); + let expected_docs: Vec = + (0..129).map(|i| SegmentLocalDocId::new(i as u32)).collect(); + assert_eq!(docs, expected_docs); + } + + #[test] + fn test_delta_varint_large_tf_values() { + let codec = DeltaVarintCodec; + // Test with TF values that require multi-byte varints (> 127) + let postings = [ + (SegmentLocalDocId::new(10), TermFreq::new(200)), + (SegmentLocalDocId::new(20), TermFreq::new(300)), + (SegmentLocalDocId::new(35), TermFreq::new(16384)), + (SegmentLocalDocId::new(100), TermFreq::new(32768)), + ]; + + let encoded = codec.encode(&postings); + let mut docs = Vec::new(); + let mut tfs = Vec::new(); + codec.decode(&encoded, &mut docs, &mut tfs).unwrap(); + + assert_eq!( + docs, + vec![ + SegmentLocalDocId::new(10), + SegmentLocalDocId::new(20), + SegmentLocalDocId::new(35), + SegmentLocalDocId::new(100), + ] + ); + assert_eq!( + tfs, + vec![ + TermFreq::new(200), + TermFreq::new(300), + TermFreq::new(16384), + TermFreq::new(32768), + ] + ); + } + + #[test] + fn test_block_delta_large_doc_gaps() { + let codec = BlockDeltaCodec; + // Simulate Zipfian: wide doc ID gaps within blocks + let postings: Vec<(SegmentLocalDocId, TermFreq)> = (0..100) + .map(|i| { + ( + SegmentLocalDocId::new((i as u32) * 10000), + TermFreq::new((i % 7 + 1) as u32), + ) + }) + .collect(); + + let encoded = codec.encode(&postings); + let mut docs = Vec::new(); + let mut tfs = Vec::new(); + codec.decode(&encoded, &mut docs, &mut tfs).unwrap(); + + assert_eq!(docs.len(), 100); + let expected_docs: Vec = (0..100) + .map(|i| SegmentLocalDocId::new((i as u32) * 10000)) + .collect(); + assert_eq!(docs, expected_docs); + } + + #[test] + fn test_block_delta_block_independence_cross_boundary() { + // Verify that block boundaries don't affect doc reconstruction. + // This tests that the first_doc in each block is stored as an absolute value + // (delta from 0), not as a delta from the previous block. + let codec = BlockDeltaCodec; + + // Create a 256-doc list spanning 2 blocks. + // Block 1: docs 0..127 (even IDs) + // Block 2: docs 128..255 (even IDs) + let postings: Vec<(SegmentLocalDocId, TermFreq)> = (0..256) + .map(|i| (SegmentLocalDocId::new((i as u32) * 2), TermFreq::new(1))) + .collect(); + + let encoded = codec.encode(&postings); + let mut docs = Vec::new(); + let mut tfs = Vec::new(); + codec.decode(&encoded, &mut docs, &mut tfs).unwrap(); + + // Verify all docs were reconstructed correctly across block boundaries. + assert_eq!(docs.len(), 256); + for (i, &doc) in docs.iter().enumerate() { + assert_eq!( + doc.get(), + u32::try_from(i).unwrap() * 2, + "Doc at index {i} mismatch" + ); + } + } + + #[test] + fn test_block_delta_non_unit_deltas() { + // Test block boundaries with varying delta sizes. + // This specifically checks that block-boundary docs have the right doc ID. + let codec = BlockDeltaCodec; + + // Docs: 0, 100, 200, ..., 12700 (128 docs per block) + let postings: Vec<(SegmentLocalDocId, TermFreq)> = (0..256) + .map(|i| { + ( + SegmentLocalDocId::new((i as u32) * 100), + TermFreq::new((i % 5 + 1) as u32), + ) + }) + .collect(); + + let encoded = codec.encode(&postings); + let mut docs = Vec::new(); + let mut tfs = Vec::new(); + codec.decode(&encoded, &mut docs, &mut tfs).unwrap(); + + assert_eq!(docs.len(), 256); + // Verify critical boundary points + assert_eq!(docs[127].get(), 127 * 100, "Last doc of block 1"); + assert_eq!( + docs[128].get(), + 128 * 100, + "First doc of block 2 (critical boundary)" + ); + assert_eq!(docs[255].get(), 255 * 100, "Last doc of block 2"); + } + + #[test] + fn test_varint_over_long_six_bytes() { + // Manually construct a 6-byte varint encoding (invalid). + // After reading the 5th byte, shift=28. On the 6th byte, shift becomes 35 >= 32. + let codec = DeltaVarintCodec; + let mut bad_bytes = Vec::new(); + bad_bytes.push(CodecId::DeltaVarint.to_u8()); + // Encode a valid posting first (doc_id=10, tf=1) + bad_bytes.extend_from_slice(&[10_u8, 1_u8]); + // Append over-long varint: all bytes with MSB set + bad_bytes.extend_from_slice(&[0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0x01_u8]); + + let mut docs = Vec::new(); + let mut tfs = Vec::new(); + let result = codec.decode(&bad_bytes, &mut docs, &mut tfs); + + // Must reject over-long varint, not silently accept + assert_eq!(result, Err(CodecError::InvalidVarint)); + } + + #[test] + fn test_delta_varint_malformed_truncated_tf() { + // Encode: doc_delta(10) + incomplete TF (0x80 without continuation). + let codec = DeltaVarintCodec; + let mut bad_bytes = Vec::new(); + bad_bytes.push(CodecId::DeltaVarint.to_u8()); + // First posting: varint(10) for delta, then 0x80 (MSB set, more bytes expected) + bad_bytes.extend_from_slice(&[10_u8, 0x80_u8]); + + let mut docs = Vec::new(); + let mut tfs = Vec::new(); + let result = codec.decode(&bad_bytes, &mut docs, &mut tfs); + + // Should detect truncation, not crash + assert_eq!(result, Err(CodecError::Truncated)); + } + + #[test] + fn test_block_delta_doc_bytes_len_bounds_check() { + // Create a block header claiming more doc bytes than available. + let codec = BlockDeltaCodec; + let mut bad_bytes = Vec::new(); + bad_bytes.push(CodecId::BlockDelta.to_u8()); + // Block header: doc_count=2, first=10, last=20, doc_bytes_len=100 (but only 2 bytes follow) + bad_bytes.extend_from_slice(&[2_u8, 10_u8, 20_u8, 100_u8]); + bad_bytes.extend_from_slice(&[10_u8, 1_u8]); // Only 2 bytes of doc stream + + let mut docs = Vec::new(); + let mut tfs = Vec::new(); + let result = codec.decode(&bad_bytes, &mut docs, &mut tfs); + + // Must reject, not read past end + assert_eq!(result, Err(CodecError::Truncated)); + } + + #[test] + #[should_panic(expected = "doc-sorted")] + fn test_delta_varint_unsorted_input_panics() { + // The doc-sorted precondition is enforced: an out-of-order doc id panics + // deterministically instead of silently producing a corrupt (wrapped) delta. + let codec = DeltaVarintCodec; + let _ = codec.encode(&[ + (SegmentLocalDocId::new(10), TermFreq::new(1)), + (SegmentLocalDocId::new(5), TermFreq::new(1)), + ]); + } + + #[test] + #[should_panic(expected = "doc-sorted")] + fn test_block_delta_unsorted_input_panics() { + let codec = BlockDeltaCodec; + let _ = codec.encode(&[ + (SegmentLocalDocId::new(10), TermFreq::new(1)), + (SegmentLocalDocId::new(5), TermFreq::new(1)), + ]); + } + + #[test] + fn test_block_delta_corrupt_header_doc_range_rejected() { + // A block header whose first_doc/last_doc disagree with the doc stream is + // rejected (the header range is validated, not ignored). + let codec = BlockDeltaCodec; + let encoded = codec.encode(&[ + (SegmentLocalDocId::new(3), TermFreq::new(1)), + (SegmentLocalDocId::new(7), TermFreq::new(2)), + (SegmentLocalDocId::new(11), TermFreq::new(3)), + ]); + + // Header layout after the 1-byte marker: varint(doc_count) varint(first_doc) ... + // doc_count=3 at index 1, first_doc=3 at index 2. Corrupt first_doc 3 -> 4. + let mut corrupt = encoded.clone(); + assert_eq!(corrupt[2], 3, "expected first_doc varint at index 2"); + corrupt[2] = 4; + + let mut docs = Vec::new(); + let mut tfs = Vec::new(); + assert_eq!( + codec.decode(&corrupt, &mut docs, &mut tfs), + Err(CodecError::BlockHeaderMismatch) + ); + assert!(docs.is_empty()); + assert!(tfs.is_empty()); + } + + #[test] + fn test_block_delta_rejects_count_above_block_size() { + let codec = BlockDeltaCodec; + let bytes = [CodecId::BlockDelta.to_u8(), 129, 1, 1, 1, 1, 1]; + let mut docs = Vec::new(); + let mut tfs = Vec::new(); + + assert_eq!( + codec.decode(&bytes, &mut docs, &mut tfs), + Err(CodecError::InvalidBlockCount) + ); + } + + #[test] + fn test_block_delta_rejects_unconsumed_doc_stream_bytes() { + let codec = BlockDeltaCodec; + let encoded = codec.encode(&[ + (SegmentLocalDocId::new(10), TermFreq::new(2)), + (SegmentLocalDocId::new(20), TermFreq::new(3)), + ]); + let mut corrupt = encoded.clone(); + corrupt[4] = 3; + corrupt.insert(8, 0); + let mut docs = Vec::new(); + let mut tfs = Vec::new(); + + assert_eq!( + codec.decode(&corrupt, &mut docs, &mut tfs), + Err(CodecError::InvalidVarint) + ); + } +} diff --git a/crates/leit_postings/src/lib.rs b/crates/leit_postings/src/lib.rs index fb2b71f9..75130469 100644 --- a/crates/leit_postings/src/lib.rs +++ b/crates/leit_postings/src/lib.rs @@ -17,6 +17,8 @@ use alloc::vec::Vec; use leit_core::{EntityId, TermId}; +pub mod codec; + /// A single posting: term occurrence in a document with ``Id``. #[derive(Clone, Debug, PartialEq, Eq)] pub struct Posting { diff --git a/crates/leit_wind_tunnel/Cargo.toml b/crates/leit_wind_tunnel/Cargo.toml index abcb325d..a275b847 100644 --- a/crates/leit_wind_tunnel/Cargo.toml +++ b/crates/leit_wind_tunnel/Cargo.toml @@ -10,11 +10,18 @@ homepage.workspace = true description = "Deterministic test harness and corpus generator for the Leit retrieval kernel Phase 2" publish = false +# Only rapidhash is used by the library surface (deterministic corpus hashing). +# The leit_* crates are used solely by the in-crate integration tests +# (`#[cfg(test)] mod integration_tests`), so they belong in dev-dependencies +# per the usage-site rule (STORY-0096): test/bench-only deps are not part of the +# crate's production dependency graph. [dependencies] +rapidhash = { workspace = true } + +[dev-dependencies] leit_core = { features = ["std"], workspace = true } leit_index = { features = ["std"], workspace = true } leit_text = { features = ["std"], workspace = true } -rapidhash = "4.4" [lints] workspace = true diff --git a/crates/leit_wind_tunnel_index/Cargo.toml b/crates/leit_wind_tunnel_index/Cargo.toml index 99fc83eb..d736383c 100644 --- a/crates/leit_wind_tunnel_index/Cargo.toml +++ b/crates/leit_wind_tunnel_index/Cargo.toml @@ -19,7 +19,8 @@ bench = false [dev-dependencies] criterion = { workspace = true } leit_core = { features = ["std"], workspace = true } -leit_index = { features = ["std"], workspace = true } +leit_index = { features = ["bench-internals", "std"], workspace = true } +leit_postings = { features = ["std"], workspace = true } leit_text = { features = ["std"], workspace = true } leit_wind_tunnel = { workspace = true } @@ -27,5 +28,9 @@ leit_wind_tunnel = { workspace = true } name = "indexing" harness = false +[[bench]] +name = "codec_compare" +harness = false + [lints] workspace = true diff --git a/crates/leit_wind_tunnel_index/benches/codec_compare.rs b/crates/leit_wind_tunnel_index/benches/codec_compare.rs new file mode 100644 index 00000000..81f4460c --- /dev/null +++ b/crates/leit_wind_tunnel_index/benches/codec_compare.rs @@ -0,0 +1,300 @@ +// Copyright 2026 the Leit Authors +// SPDX-License-Identifier: Apache-2.0 OR MIT + +//! Codec comparison benchmarks: encode/decode latency and compressed size. +//! +//! This benchmark indexes a deterministic wind-tunnel corpus and extracts all postings, +//! then measures and reports: +//! - Encode time per codec +//! - Decode time per codec +//! - Compressed size ratio vs 8-byte uncompressed baseline +//! +//! Run with `cargo bench -p leit_wind_tunnel_index --bench codec_compare`. + +#![expect( + missing_docs, + reason = "criterion_group! generates an undocumented public `benches` fn" +)] + +use criterion::{BenchmarkId, Criterion, Throughput, criterion_group, criterion_main}; +use leit_core::{FieldId, SegmentLocalDocId, TermFreq}; +use leit_index::InMemoryIndexBuilder; +use leit_postings::codec::{BlockDeltaCodec, Codec, DeltaVarintCodec}; +use leit_text::{Analyzer, FieldAnalyzers, UnicodeNormalizer, WhitespaceTokenizer}; +use leit_wind_tunnel::{CorpusGenerator, corpus::GeneratedDoc}; + +/// Fixed seed so every benchmark run indexes byte-identical corpora. +const SEED: u64 = 42; +/// `title` field, matching the corpus generator's field 1. +const TITLE: FieldId = FieldId::new(1); +/// `body` field, matching the corpus generator's field 2. +const BODY: FieldId = FieldId::new(2); + +/// Postings list type: `Vec` of (`SegmentLocalDocId`, `TermFreq`) tuples per term. +type PostingsList = Vec>; + +struct CorpusCase { + label: &'static str, + doc_count: u32, + postings: PostingsList, +} + +impl CorpusCase { + fn total_postings(&self) -> usize { + self.postings.iter().map(Vec::len).sum() + } + + fn max_postings_len(&self) -> usize { + self.postings.iter().map(Vec::len).max().unwrap_or(0) + } +} + +/// Build the field analyzers used for indexing (whitespace tokenizer + unicode +/// normalizer on both fields), matching the wind-tunnel integration tests. +fn make_analyzers() -> FieldAnalyzers { + let mut analyzers = FieldAnalyzers::new(); + analyzers.set( + TITLE, + Analyzer::new(WhitespaceTokenizer::new()).with_normalizer(UnicodeNormalizer::new()), + ); + analyzers.set( + BODY, + Analyzer::new(WhitespaceTokenizer::new()).with_normalizer(UnicodeNormalizer::new()), + ); + analyzers +} + +/// Index a corpus and extract all postings as `Vec<(doc_id, term_freq)>` tuples. +fn extract_postings(corpus: &[GeneratedDoc]) -> PostingsList { + let mut builder = InMemoryIndexBuilder::new(make_analyzers()); + builder.register_field_alias(TITLE, "title"); + builder.register_field_alias(BODY, "body"); + + for doc in corpus { + builder + .index_document( + doc.id, + &[(TITLE, doc.title.as_str()), (BODY, doc.body.as_str())], + ) + .expect("indexing should succeed"); + } + + let index = builder.build_index(); + + // Extract all postings as (SegmentLocalDocId, TermFreq) tuples. + // Postings are already doc-sorted from the index. + index + .benchmark_postings() + .into_iter() + .map(|posting_list| { + posting_list + .into_iter() + .map(|(doc_id, term_freq)| { + (SegmentLocalDocId::new(doc_id), TermFreq::new(term_freq)) + }) + .collect() + }) + .collect() +} + +fn validate_decodes(codec: &C, encoded: &[Vec], postings: &PostingsList) { + let max_len = postings.iter().map(Vec::len).max().unwrap_or(0); + let mut docs = Vec::with_capacity(max_len); + let mut tfs = Vec::with_capacity(max_len); + + for (bytes, expected) in encoded.iter().zip(postings) { + codec + .decode(bytes, &mut docs, &mut tfs) + .expect("decode should succeed"); + assert_eq!(docs.len(), expected.len(), "doc count mismatch"); + assert_eq!(tfs.len(), expected.len(), "tf count mismatch"); + for (index, (doc_id, tf)) in expected.iter().enumerate() { + assert_eq!(docs[index], *doc_id, "doc mismatch at index {index}"); + assert_eq!(tfs[index], *tf, "tf mismatch at index {index}"); + } + } +} + +/// Measure compression ratio and emit a summary table. +fn report_compression_detailed( + corpus_size: u32, + total_postings: usize, + codec_sizes: &[(&str, usize)], +) { + // Baseline: 8 bytes per posting (u32 doc_id + u32 term_freq). + let uncompressed_baseline = total_postings * 8; + + eprintln!( + "\n=== Codec Compression Summary (corpus: {} docs, {} total postings) ===", + corpus_size, total_postings + ); + eprintln!("Baseline (uncompressed): 8 bytes per posting"); + eprintln!(" Total uncompressed: {} bytes", uncompressed_baseline); + + for (name, bytes) in codec_sizes { + let ratio = if uncompressed_baseline > 0 { + (*bytes as f64) / (uncompressed_baseline as f64) * 100.0 + } else { + 0.0 + }; + let avg_bytes_per_posting = if total_postings > 0 { + *bytes as f64 / total_postings as f64 + } else { + 0.0 + }; + eprintln!( + " {}: {} bytes ({:.1}% of baseline, {:.2} bytes/posting)", + name, bytes, ratio, avg_bytes_per_posting + ); + } +} + +fn bench_codec_encode_decode(c: &mut Criterion) { + let generator = CorpusGenerator::new(SEED); + + // Prepare corpora and postings once outside the benchmark loop. + let corpora = [ + ("1k", 1_000_u32, generator.generate(1_000)), + ("10k", 10_000_u32, generator.generate(10_000)), + ]; + + let all_postings: Vec = corpora + .iter() + .map(|(label, doc_count, corpus)| CorpusCase { + label, + doc_count: *doc_count, + postings: extract_postings(corpus), + }) + .collect(); + + // Report compression sizes upfront. + for corpus in &all_postings { + let total_postings = corpus.total_postings(); + let mut sizes = Vec::new(); + + // Encode all postings with each codec and measure total size. + let delta_varint_codec = DeltaVarintCodec; + let block_delta_codec = BlockDeltaCodec; + + let mut dv_total_size = 0; + let mut bd_total_size = 0; + + for postings in &corpus.postings { + let dv_encoded = delta_varint_codec.encode(postings); + dv_total_size += dv_encoded.len(); + + let bd_encoded = block_delta_codec.encode(postings); + bd_total_size += bd_encoded.len(); + } + + sizes.push(("DeltaVarint", dv_total_size)); + sizes.push(("BlockDelta", bd_total_size)); + + report_compression_detailed(corpus.doc_count, total_postings, &sizes); + } + + let mut group = c.benchmark_group("codec_encode"); + for corpus in &all_postings { + let delta_varint_codec = DeltaVarintCodec; + let block_delta_codec = BlockDeltaCodec; + group.throughput(Throughput::Elements(corpus.total_postings() as u64)); + + group.bench_with_input( + BenchmarkId::from_parameter(format!("deltavarint/{}", corpus.label)), + &corpus.doc_count, + |b, _| { + b.iter(|| { + let mut total_bytes = 0; + for postings in &corpus.postings { + let encoded = delta_varint_codec.encode(postings); + total_bytes += encoded.len(); + } + criterion::black_box(total_bytes); + }); + }, + ); + + group.bench_with_input( + BenchmarkId::from_parameter(format!("blockdelta/{}", corpus.label)), + &corpus.doc_count, + |b, _| { + b.iter(|| { + let mut total_bytes = 0; + for postings in &corpus.postings { + let encoded = block_delta_codec.encode(postings); + total_bytes += encoded.len(); + } + criterion::black_box(total_bytes); + }); + }, + ); + } + group.finish(); + + let mut group = c.benchmark_group("codec_decode"); + for corpus in &all_postings { + let delta_varint_codec = DeltaVarintCodec; + let block_delta_codec = BlockDeltaCodec; + + // Pre-encode for decode benchmarks. + let dv_encoded: Vec> = corpus + .postings + .iter() + .map(|postings| delta_varint_codec.encode(postings)) + .collect(); + + let bd_encoded: Vec> = corpus + .postings + .iter() + .map(|postings| block_delta_codec.encode(postings)) + .collect(); + + validate_decodes(&delta_varint_codec, &dv_encoded, &corpus.postings); + validate_decodes(&block_delta_codec, &bd_encoded, &corpus.postings); + group.throughput(Throughput::Elements(corpus.total_postings() as u64)); + + group.bench_with_input( + BenchmarkId::from_parameter(format!("deltavarint/{}", corpus.label)), + &corpus.doc_count, + |b, _| { + let max_len = corpus.max_postings_len(); + let mut docs = Vec::with_capacity(max_len); + let mut tfs = Vec::with_capacity(max_len); + b.iter(|| { + let mut decoded_count = 0; + for encoded in &dv_encoded { + delta_varint_codec + .decode(encoded, &mut docs, &mut tfs) + .expect("decode should succeed"); + decoded_count += docs.len(); + } + criterion::black_box(decoded_count); + }); + }, + ); + + group.bench_with_input( + BenchmarkId::from_parameter(format!("blockdelta/{}", corpus.label)), + &corpus.doc_count, + |b, _| { + let max_len = corpus.max_postings_len(); + let mut docs = Vec::with_capacity(max_len); + let mut tfs = Vec::with_capacity(max_len); + b.iter(|| { + let mut decoded_count = 0; + for encoded in &bd_encoded { + block_delta_codec + .decode(encoded, &mut docs, &mut tfs) + .expect("decode should succeed"); + decoded_count += docs.len(); + } + criterion::black_box(decoded_count); + }); + }, + ); + } + group.finish(); +} + +criterion_group!(benches, bench_codec_encode_decode); +criterion_main!(benches); diff --git a/docs/2026-05-30-codec-tradeoffs.md b/docs/2026-05-30-codec-tradeoffs.md new file mode 100644 index 00000000..0f8904a2 --- /dev/null +++ b/docs/2026-05-30-codec-tradeoffs.md @@ -0,0 +1,43 @@ +# Codec Tradeoff Analysis: DeltaVarint vs BlockDelta + +**Status:** Evidence-based analysis from SCENARIO-0070 codec benchmarks (ITER-0002, STORY-0006 AC-3). + +**Measurement:** `cargo bench -p leit_wind_tunnel_index --bench codec_compare` on the deterministic wind-tunnel corpus (SEED=42, 1K and 10K documents, Zipfian term distribution). Encode measures codec work plus allocation of each encoded output `Vec`. Decode reuses preallocated doc/TF buffers; correctness validation runs before, not inside, the timed loop. Criterion reports throughput in postings per second. + +--- + +## Summary + +Both codecs compress postings lists to ~25–27% of the uncompressed baseline (8 bytes per posting). + +- **DeltaVarint**: single-block, varint-encoded deltas, ~2.03–2.05 bytes/posting +- **BlockDelta**: 128-doc blocks with per-block headers, ~2.10–2.19 bytes/posting + +Latency results from the earlier allocation-and-assertion-inclusive loop are intentionally omitted. Fresh results from the corrected loop must be recorded before latency is used as decision evidence. + +--- + +## Tradeoff Rationale + +### DeltaVarint: Simplicity +- **Single stream**: no block metadata to parse. +- **Simplest codec**: delta encoding + varints, lowest complexity on the read path. +- **Trade-off**: no block structure means future block-aware features (selective decode, skip, WAND doc-range pruning) require full decode. +- **Encode path**: linear delta and varint encoding. + +### BlockDelta: Block-aware future evolution +- **128-doc blocks**: each block independently decodable; enables Phase 3 features (selective block skip, WAND pruning with block-level doc ranges). +- **Per-block header overhead**: first_doc, last_doc, doc_bytes_len increase encoded size slightly vs DeltaVarint. +- **Trade-off**: per-block headers add parsing and encoding work; blocks do not improve memory footprint (compression ratio is similar). + +--- + +## Conclusion + +**For Phase 2 (v1)**: DeltaVarint is sufficient and simpler; it achieves a similar compression ratio without block metadata. + +**For Phase 3+ (selective decode, block-aware WAND)**: BlockDelta's block structure enables those features without full decode. + +The compression measurements indicate that **compression efficiency is not the differentiator**. The decision is currently **architectural**: DeltaVarint for simplicity in v1, BlockDelta for extensibility in v2+; corrected latency results may refine that tradeoff. + +**Current production choice**: DeltaVarint is the default; BlockDelta is implemented and tested in parallel for Phase 3 integration. diff --git a/docs/2026-05-30-phase2-architectural-decisions.md b/docs/2026-05-30-phase2-architectural-decisions.md new file mode 100644 index 00000000..6b525190 --- /dev/null +++ b/docs/2026-05-30-phase2-architectural-decisions.md @@ -0,0 +1,329 @@ +# Phase 2 Architectural Decisions + +**Status:** Decisions of record for ITER-0001. Each decision is *design-decidable +without wind-tunnel measurement*; the code that enforces it is implemented in the +iteration noted under "Enforced by" (the deferred `· deferred:ITER-NNNN` ACs). + +**Grounding:** the Phase 2 handover's segment-architecture sketch, versioning bias, +and open questions, plus the ITER-0001 serialization choice (bytemuck zero-copy, +little-endian). + +**Cross-cutting premise:** segment-resident structures are **zero-copy POD** — +`#[repr(transparent)]`/`#[repr(C)]` over little-endian byte fields, viewed in place +from an mmap'd `&[u8]` via bytemuck, with no deserialization or heap reconstruction +on the read path. Every decision below is consistent with that premise. + +--- + +## DEC-01 — Offset width: u64 (STORY-0043) — RESOLVED + +**Decision:** Segment offsets are **unsigned 64-bit, little-endian**, absolute from +segment start. There is **no practical segment-size cap**. (The handover sketch used +`u32` and flagged the choice as open; the human chose u64 to remove any future +format-break-for-size risk — see Phase 3 forward-compatibility.) + +**Rationale:** u64 offsets future-proof the format against large single segments +(stored fields, columnar, large postings) with no v2 migration ever needed for size. +The cost is a larger header (8-byte offsets, ~84 bytes vs ~44), negligible against +segment size and read once per open. Doc IDs remain segment-local **u32** +(`SegmentLocalDocId`): byte offsets are u64 (file positions may be large), but a single +segment is bounded to 2³² docs — an independent, generous limit that does not interact +with offset width. + +**Verification:** documented here; `SegmentHeader` uses the u64 offset type (enforced +ITER-0004, STORY-0043 AC-3). STORY-0043 AC-2 (32-bit cap documentation) is N/A — u64 +was chosen, so there is no cap to document. + +**Enforced by:** ITER-0004. + +## DEC-02 — Metadata tables: fixed-width entries; variable-width only for term bytes (STORY-0044) + +**Decision:** `field_table` and `postings_table` use **fixed-width entries** so a +reader seeks to entry *i* with O(1) offset arithmetic and views the whole table as a +zero-copy `&[Entry]` (bytemuck slice cast). The **term dictionary** stores the +variable-length term bytes in a blob, addressed by a **fixed-width offset/length index** +(itself a POD table) → O(1) access to any term's bytes without scanning. + +**Rationale:** Handover bias: "direct section lookup", "O(1) offset computation". +Fixed-width tables are exactly the zero-copy POD slices bytemuck gives us. Only term +strings are inherently variable; isolating them behind a fixed-width index keeps every +*metadata* access O(1) while paying variable-width cost only for the term bytes. + +**Enforced by:** ITER-0004 (STORY-0044 AC-2/3). + +## DEC-03 — Dictionary/postings coupling: separately addressable (STORY-0045) + +**Decision:** The term dictionary (lexicon) and the postings metadata table are +**separate sections** with independent header offsets (`lexicon_offset`, +`postings_table_offset`), as in the handover sketch. A lexicon entry yields an index +into the postings table; the two are not interleaved. + +**Rationale:** Separate sections can be validated, mmap'd, and evolved independently, +and keep each section a homogeneous POD table (DEC-02). Interleaving would mix +variable-width term bytes with fixed-width postings metadata, defeating O(1) seeks. +The handover sketch already gives them separate offsets. + +**Enforced by:** ITER-0004 (STORY-0045 AC-2). + +## DEC-04 — Mmap readiness scope (STORY-0046) + +**Decision:** **mmap-friendly in v1** (plain little-endian POD, no heap pointers, no +relocations, viewable zero-copy): segment header, field table, term dictionary +(index + bytes), postings metadata table, postings data blocks, block metadata. +**Deferred / build-time or in-memory only in v1:** optional stored-fields section and +optional columnar section (their slots are reserved in the header but their v1 content +is minimal/empty — full content is Phase 3). + +**Rationale:** The hot read path (header → section tables → postings → blocks) must be +zero-copy mmap for the performance goals. Stored/columnar are optional and off the hot +retrieval path, so they can lag without constraining v1. Reserving header slots now +(DEC-05) keeps the format forward-compatible. + +**Enforced by:** ITER-0004 (STORY-0046 AC-2, SCENARIO-0047), ITER-0005 (mmap loading). + +## DEC-05 — Segment header / offset strategy (STORY-0090) + +**Decision:** A **fixed-layout, little-endian POD header** (bytemuck `Pod`, +alignment-1 byte-field layout consistent with the ID types) as the first bytes of the +segment. Fields: `magic` (u32), `version` (u32), `format_flags` (u32), then the +section offsets **`field_table_offset`, `lexicon_offset`, `postings_table_offset`, +`postings_data_offset`, `block_meta_offset`, `stored_fields_offset`, +`columnar_offset`, `footer_offset` (all u64 LE, per DEC-01)**. **Offsets are absolute +from segment start.** +Endianness is fixed little-endian on every host. A trailing **footer** carries the +optional checksum (DEC-10). + +**Rationale:** Extends the handover sketch (adds `magic`, `format_flags`, +`stored_fields_offset`, `footer_offset`). Absolute offsets are the cheapest to +validate — each must be `<= segment_len` and sections must be non-overlapping/ordered +— and they let a reader jump directly to any section. `format_flags` marks which +optional sections are present (DEC-10). Little-endian POD = the bytemuck zero-copy +premise. + +**Enforced by:** ITER-0004 (STORY-0090 AC-2, SCENARIO-0025). + +## DEC-06 — Block-aware capability scope (STORY-0081) + +**Decision:** Block-aware accessors (`block_max_score()`, `block_end_doc()`, block +boundaries) live on a **dedicated public trait** (e.g. `BlockCursor`) that is +**separate from** the base cursor trait — public so the format's block metadata is +reachable and a future WAND/MaxScore scorer can consume it, but not forced on basic +cursor users. **v1 exposes the data/API surface only; block-skipping *execution* +(WAND pruning) is Phase 3.** + +**Rationale:** Handover makes block metadata "a first-class citizen". Exposing it via a +separate trait honors the architectural split (traversal API vs execution) without +committing v1 to the pruning algorithm. Keeping it off the base trait avoids boxing in +non-block cursors. + +**Verification:** trait visibility/boundary enforced in code (ITER-0003, +STORY-0081 AC-2). **RESOLVED — public dedicated `BlockCursor` trait (confirmed).** + +**Enforced by:** ITER-0003. + +## DEC-07 — Segment validation strategy: three lazy modes (STORY-0082) + +**Decision:** A `ValidationMode` enum with **`HeaderOnly`**, **`Structural`** (default), +and **`Full`**: +- `HeaderOnly` — verify magic + version (reject unsupported) + header self-consistency. Cheapest open. +- `Structural` — additionally verify every offset is in-bounds and sections are ordered/non-overlapping. The default for `SegmentView::open()`. +- `Full` — additionally verify the footer checksum (DEC-10) and per-section structural invariants. + +Validation is otherwise **lazy**: per-access reads use bytemuck `try_*` casts, which +bounds-check length at the point of use regardless of mode, so traversal is always +memory-safe even under `HeaderOnly`. + +**Rationale:** Balances startup latency against safety (handover: "easy validation of +offset ranges"). `Structural` is cheap (a handful of comparisons) and catches the +common corruption/truncation cases, so it's the safe default; `Full` is opt-in for +untrusted inputs. Because `try_cast` validates every slice access anyway, even the +cheapest mode cannot cause UB. + +**Verification:** `ValidationMode` + `open_with_validation()` (ITER-0004, +STORY-0082 AC-2, SCENARIO-0020). **RESOLVED — three modes; `Structural` default; `Full` +adds the DEC-10 footer checksum (not surfaced for change; recorded default stands).** + +**Enforced by:** ITER-0004. + +## DEC-08 — View borrowing model: fully borrowed (STORY-0083) + +**Decision:** All segment section views **borrow `&[u8]` directly** (zero-copy). View +types hold a byte slice plus validated offsets/lengths; there are **no eagerly-decoded +owned fields**. The only work at view-construction time is bounds validation (returns +indices, not copies). + +**Rationale:** Directly follows the bytemuck zero-copy premise and the handover's +"low-cost borrowed views" + minimal-allocation principle. Any eager copy would +reintroduce allocation on the hot path. Decoded values (e.g. a `u32` from an offset +table) are produced on demand by cheap LE reads, not cached. + +**Enforced by:** ITER-0004 (STORY-0083 AC-2). + +## DEC-09 — Builder vs read-only type separation: strict (STORY-0084) + +**Decision:** **Strict separation.** Writer/builder types (`SegmentBuilder` and +friends) that accumulate and serialize a segment are **distinct** from the read-only +view types (`SegmentView` and section views). Builder types never appear in query/read +paths; view types are never mutable post-construction. + +**Rationale:** Segments are immutable; conflating builder and reader invites misuse +(e.g. mutating a "view"). The handover's clean split — on-disk representation ↔ +traversal views ↔ orchestration — is naturally expressed as writer-produces-bytes, +reader-views-bytes. Strict separation also lets the reader stay `no_std`/zero-copy +while the builder may freely allocate. + +**Enforced by:** ITER-0004 (STORY-0084 AC-2). + +## DEC-10 — Versioning & checking scope (STORY-0047) + +**Decision (minimal but real):** +- **Version:** `version: u32` in the header; readers **reject unsupported (future) versions cleanly** with a structured error. Backward compatibility is promised for a *bounded* set of versions; dropping support means rebuild-via-tooling (handover model). +- **Feature flags:** `format_flags: u32` bitfield marks which optional sections are present (stored_fields, columnar), so a reader knows whether those offsets are meaningful. +- **Checksum:** a **single footer checksum** over the segment body (algorithm: a fast non-cryptographic hash — candidate: the same rapidhash family already vendored, or crc32c). Validated only in `Full` mode (DEC-07). **No per-section checksums in v1.** +- **Magic:** a 4-byte magic constant as the first header field for quick format identification. + +**Rationale:** Handover wants explicit versioning + clean rejection from the start, but +warns against over-engineering. Version + flags + one optional checksum + magic is the +minimal set that supports clean rejection, optional-section detection, and integrity +checking, without the cost/complexity of per-section checksums. + +**Verification:** version-rejection + flag handling + checksum (ITER-0004, +STORY-0047 AC-2/3). **RESOLVED — include a single footer checksum in v1 (confirmed); +algorithm finalized in ITER-0004 (rapidhash-family or crc32c).** + +**Enforced by:** ITER-0004. + +--- + +## Decisions epic anchor (STORY-0078) + +All eight-plus must-resolve decisions above are recorded with a decision, a rationale, +and a verification method (the enforcing iteration + AC/scenario). Dependent +implementation (ITER-0002 codecs, ITER-0003 cursors, ITER-0004 segment format) may now +proceed against a stable decision point. Decisions flagged **[SURFACE]** were raised to +the human for confirmation; the recorded choice is the confirmed/default position. + +## Phase 3 forward-compatibility (does any v1 decision box in Phase 3?) + +Phase 3 adds **WAND / MaxScore block-skipping execution**, **columnar field content**, +and **real-corpus loading**. The v1 decisions are designed to *enable* these without a +format break: + +1. **Block metadata is first-class and mmap-friendly in v1** (DEC-04), even though + block-skipping *execution* is Phase 3 (DEC-06). The per-block `max_score` and doc + ranges live in the format now, so Phase 3 WAND reads them with **no format change** — + this is the handover's explicit "first-class, not bolted on later" intent. +2. **The block-aware API surface ships in v1** as a dedicated `BlockCursor` trait + (DEC-06) and the MaxScoreScorer block-structure contract (ITER-0003, SCENARIO-0002). + Phase 3 pruning is a new *consumer* of an existing surface, not an API break. +3. **Optional-section slots are reserved now** (DEC-05 header carries + `stored_fields_offset`/`columnar_offset`; DEC-10 `format_flags` marks presence). + Phase 3 fills the columnar slot and sets its flag — **no version bump required**; + old readers see the flag clear and ignore it. +4. **Explicit versioning + rebuild path** (DEC-01, DEC-10): any change that *cannot* be + done via a reserved slot is a clean `version` bump with clean rejection by old + readers and a tooling rebuild — never in-hot-path legacy support. +5. **Zero-copy bytemuck POD** generalizes: Phase 3's columnar and block structures are + also flat POD tables, so the same `try_*` validated casts apply. No serialization + rework. + +**No segment-size limit:** DEC-01 was resolved to **u64 offsets**, so the previously +identified 4 GiB cap is gone — there is no v1 decision that forces a future +format-break for segment size. The only residual bound is `SegmentLocalDocId` = u32 +(≤ 2³² docs per segment), which is independent of offset width and far beyond any +realistic single-segment doc count; it does not constrain Phase 3. + +**Forward constraint placed on ITER-0005 (block-metadata schema, STORY-0086):** the +block-metadata section's v1 schema **must** carry, per block, at least `max_score` and +the doc-range needed for WAND/MaxScore skipping, so Phase 3 can prune without a format +change. Recorded here so the ITER-0005 doc-range decision honors it. + +**Completed work check:** ITER-0000 (wind tunnel) is additive measurement infra and +boxes nothing; its STORY-0105 real-corpus path is explicitly a Phase 3+ plug-in against +stable output types. ITER-0001 ID types (`BlockId`, etc.) are precisely what Phase 3 +WAND consumes — defining them now *helps* Phase 3. + +## Decision → enforcement traceability + +| Decision | Story | Enforced by | Proof | +|---|---|---|---| +| DEC-01 offset width u64 | STORY-0043 | ITER-0004 | header uses u64 offset type | +| DEC-02 fixed-width tables | STORY-0044 | ITER-0004 | O(1) entry seek | +| DEC-03 separate sections | STORY-0045 | ITER-0004 | independent offsets | +| DEC-04 mmap scope | STORY-0046 | ITER-0004/0005 | SCENARIO-0047 | +| DEC-05 header/offset strategy | STORY-0090 | ITER-0004 | SCENARIO-0025 | +| DEC-06 block-aware trait | STORY-0081 | ITER-0003 | trait visibility | +| DEC-07 validation modes | STORY-0082 | ITER-0004 | SCENARIO-0020 | +| DEC-08 fully borrowed views | STORY-0083 | ITER-0004 | view accessors | +| DEC-09 strict builder/reader | STORY-0084 | ITER-0004 | type boundary | +| DEC-10 versioning/checks | STORY-0047 | ITER-0004 | version-rejection | +| DEC-11 block boundary strategy | STORY-0087 | ITER-0002 (codec) / ITER-0004 (writer) | block codec conformance | +| DEC-12 v1 postings/block layout | STORY-0088 | ITER-0002 | SCENARIO-0006 | + +--- + +## DEC-11 — Postings block boundary: fixed document-count blocks (STORY-0087) — RESOLVED + +**Decision:** v1 postings blocks are bounded by a **fixed document count** of **128 documents +per block** (the last block may be short). Boundaries are by doc-count, **not** fixed byte size +and **not** an adaptive/merge heuristic. + +**Rationale (measured against ITER-0000 evidence):** +- The ITER-0000 wind-tunnel corpus is Zipfian (a few very long postings lists, a long tail of + short ones). A **fixed byte-size** block would split a list at unpredictable doc positions, + forcing partial-integer state across block boundaries and making per-block doc-range/skip + metadata (the Phase 3 WAND constraint recorded for ITER-0005) awkward to compute. A fixed + **doc-count** block gives every block a clean `[first_doc, last_doc]` range and a known item + count, which is exactly what skip-lists and block-max metadata need. +- 128 docs/block is the conventional Lucene/PForDelta block size and matches the decode-batch + granularity that keeps the decode scratch small and cache-resident. It is large enough to + amortize per-block header cost over the long lists that dominate query latency in the baseline + (`single_term` ≈ 136 µs/1k), small enough that selective block decode (the deferred ITER-0003 + `advance_to` skip) skips meaningful work. +- Short lists (< 128 docs — the Zipfian tail) occupy a single block; no padding, no waste. + +**Consequence / forward-compat:** Each block carries its document count and first/last doc in its +header, so ITER-0005's block-metadata sidecar can reference blocks by ordinal and attach +`max_score` + doc-range without re-deriving boundaries. The boundary constant lives in one place +(`BLOCK_DOC_COUNT`) so a future codec may tune it without a format-break (the count is encoded +per block, not assumed by readers). + +## DEC-12 — v1 postings/block layout for compressed traversal (STORY-0088) — RESOLVED + +**Decision:** A postings list serializes as a sequence of independently-decodable **blocks**. +Each block is laid out as: + +``` +block := block_header doc_id_stream tf_stream +block_header := varint(doc_count) varint(first_doc) varint(last_doc) varint(doc_bytes_len) +doc_id_stream := varint(first_delta) varint(delta)* # deltas from previous doc id +tf_stream := varint(tf)* # one per doc, parallel to doc stream +``` + +- **Doc IDs** are **delta-encoded** then **LEB128 varint**-encoded (deltas are non-negative + because postings are doc-sorted). The first doc in a block is stored as a delta from 0 (i.e. its + absolute value), so each block is self-contained and decodable without the previous block + (DEC-11 enables this). +- **Term frequencies** are LEB128 varint-encoded, one per doc, in a stream parallel to the doc-id + stream — so a doc-only cursor can decode the doc stream and skip the TF stream using + `doc_bytes_len`. +- **`doc_bytes_len`** in the header lets a reader locate the TF stream (and the next block) without + decoding the doc stream — the basis for the deferred ITER-0003 selective/skip decode. +- **Codec marker:** a postings list is prefixed by a 1-byte `CodecId` (DEC: `0 = DeltaVarint` + single-block, `1 = BlockDelta` doc-count blocks). This is the per-list marker; reserving the + *field in the segment format* is deferred to ITER-0004 (STORY-0002 AC-3). + +**Traversal-semantics contract (uncompressed ↔ compressed API stability — STORY-0088 AC-2 is the +e2e proof, deferred to ITER-0003):** the codec **API speaks named segment-resident types** +`SegmentLocalDocId` and `TermFreq` (the u32 values are delta-encoded and varint-encoded at the +byte level, unchanged). The generic `EntityId` is lowered to `SegmentLocalDocId` at the +segment-write boundary (ITER-0004). Decode produces exactly the `(SegmentLocalDocId, TermFreq)` +sequence, in doc-ascending order, that the in-memory `PostingsList` will hold — byte-for-byte +equal round trip (SCENARIO-0006). The codec writes decoded values into **caller-provided output +buffers** (`&mut Vec` and `&mut Vec`); it does **not** own a +decode-scratch type. The decode-scratch ownership wrapper and the cursor adaptor over this API +are decided and built in ITER-0003 (STORY-0079) — the codec layer is deliberately +scratch-ownership-agnostic so that decision is not boxed in here. + +**Decided in ITER-0002 against the ITER-0000 wind-tunnel baseline; enforced by the codec +implementation (SCENARIO-0006 round-trip) and measured by SCENARIO-0070 (codec comparison).**