From c3c5ac8af7ec0870ce49f029f2f4d85729fa81a6 Mon Sep 17 00:00:00 2001 From: "Norman Nunley, Jr" Date: Fri, 29 May 2026 21:41:32 -0400 Subject: [PATCH 1/9] refactor: move wind-tunnel harness test-only deps to dev-dependencies (STORY-0096) ITER-0001 dependency hygiene per the usage-site rule: the leit_wind_tunnel harness uses only rapidhash in its library surface; leit_core/leit_index/ leit_text are used solely by its #[cfg(test)] integration tests, so they move to [dev-dependencies] and no longer appear in the harness's production dependency graph. The bench crates were already correct (empty lib; all deps dev). Library build, 17 unit tests, and both bench crates verified green. --- crates/leit_wind_tunnel/Cargo.toml | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/crates/leit_wind_tunnel/Cargo.toml b/crates/leit_wind_tunnel/Cargo.toml index abcb325..a275b84 100644 --- a/crates/leit_wind_tunnel/Cargo.toml +++ b/crates/leit_wind_tunnel/Cargo.toml @@ -10,11 +10,18 @@ homepage.workspace = true description = "Deterministic test harness and corpus generator for the Leit retrieval kernel Phase 2" publish = false +# Only rapidhash is used by the library surface (deterministic corpus hashing). +# The leit_* crates are used solely by the in-crate integration tests +# (`#[cfg(test)] mod integration_tests`), so they belong in dev-dependencies +# per the usage-site rule (STORY-0096): test/bench-only deps are not part of the +# crate's production dependency graph. [dependencies] +rapidhash = { workspace = true } + +[dev-dependencies] leit_core = { features = ["std"], workspace = true } leit_index = { features = ["std"], workspace = true } leit_text = { features = ["std"], workspace = true } -rapidhash = "4.4" [lints] workspace = true From a60c0168afa4388542060d2c3c0c89d28c623c92 Mon Sep 17 00:00:00 2001 From: "Norman Nunley, Jr" Date: Fri, 29 May 2026 21:50:27 -0400 Subject: [PATCH 2/9] feat: add segment-resident core ID types with zero-copy serialization (STORY-0112) ITER-0001: BlockId, FilterExprId, SegmentOrd, SegmentLocalDocId in leit_core, each a #[repr(transparent)] newtype over a [u8; 4] little-endian inner deriving bytemuck Pod/Zeroable. The on-disk form is the in-memory form: a &[u8] slice from an mmap'd buffer casts in place to &[Id] with no allocation or deserialization (zero-copy), stable across host endianness; ordering is numeric. bytemuck chosen over zerocopy because zerocopy's derives emit internal #[allow(non_ascii_idents)]/#[allow(non_local_definitions)] that conflict with the workspace's forbid-level Linebender lints (E0453); bytemuck is no_std and lint-clean under the same forbid set. Proven by SCENARIO-0005 (6 unit tests: value + slice + unaligned round-trip, numeric ordering, LE byte layout). --- Cargo.lock | 21 ++++ Cargo.toml | 3 + crates/leit_core/Cargo.toml | 3 + crates/leit_core/src/lib.rs | 3 + crates/leit_core/src/segment_ids.rs | 164 ++++++++++++++++++++++++++++ 5 files changed, 194 insertions(+) create mode 100644 crates/leit_core/src/segment_ids.rs diff --git a/Cargo.lock b/Cargo.lock index 70da149..dde280a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -71,6 +71,26 @@ version = "3.20.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "72f5acc6cb2ba439de613abc23857ec3d78374d8ed5ac84e9d11336e87da8649" +[[package]] +name = "bytemuck" +version = "1.25.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c8efb64bd706a16a1bdde310ae86b351e4d21550d98d056f22f8a7f7a2183fec" +dependencies = [ + "bytemuck_derive", +] + +[[package]] +name = "bytemuck_derive" +version = "1.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f9abbd1bc6865053c427f7198e6af43bfdedc55ab791faed4fbd361d789575ff" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "cast" version = "0.3.0" @@ -553,6 +573,7 @@ dependencies = [ name = "leit_core" version = "0.1.0" dependencies = [ + "bytemuck", "proptest", ] diff --git a/Cargo.toml b/Cargo.toml index e41e3dc..7039174 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -50,6 +50,9 @@ proptest = { version = "1.0" } # Hashing rapidhash = { version = "4.4" } +# Zero-copy serialization for segment-resident types (no_std) +bytemuck = { default-features = false, features = ["derive"], version = "1" } + # Math core_maths = { default-features = false, version = "0.1" } diff --git a/crates/leit_core/Cargo.toml b/crates/leit_core/Cargo.toml index 60f1e83..7c44730 100644 --- a/crates/leit_core/Cargo.toml +++ b/crates/leit_core/Cargo.toml @@ -15,6 +15,9 @@ categories.workspace = true default = ["std"] std = [] +[dependencies] +bytemuck = { workspace = true } + [dev-dependencies] proptest = { workspace = true } diff --git a/crates/leit_core/src/lib.rs b/crates/leit_core/src/lib.rs index 079514d..c2f5973 100644 --- a/crates/leit_core/src/lib.rs +++ b/crates/leit_core/src/lib.rs @@ -20,6 +20,9 @@ use core::fmt; use core::hash::Hash; use core::ops::{Add, AddAssign, Mul, MulAssign, Sub, SubAssign}; +pub mod segment_ids; +pub use segment_ids::{BlockId, FilterExprId, SegmentLocalDocId, SegmentOrd}; + /// Unique identifier for a field in an index. #[derive(Clone, Copy, Debug, Default, PartialEq, Eq, PartialOrd, Ord, Hash)] #[repr(transparent)] diff --git a/crates/leit_core/src/segment_ids.rs b/crates/leit_core/src/segment_ids.rs new file mode 100644 index 0000000..6256026 --- /dev/null +++ b/crates/leit_core/src/segment_ids.rs @@ -0,0 +1,164 @@ +// Copyright 2026 the Leit Authors +// SPDX-License-Identifier: Apache-2.0 OR MIT + +//! Segment-resident core ID types with a stable, zero-copy serialized representation. +//! +//! Unlike the in-memory index identifiers ([`FieldId`](crate::FieldId), +//! [`TermId`](crate::TermId), [`SegmentId`](crate::SegmentId)), the types in this +//! module are designed to appear **directly in mmap'd segment bytes**. Each is a +//! `#[repr(transparent)]` newtype over a 4-byte little-endian array, so a `&[u8]` +//! slice taken from a memory-mapped buffer can be viewed in place as `&[Id]` with +//! no allocation and no deserialization pass (see the Phase 2 architectural +//! decisions: zero-copy via `bytemuck`). +//! +//! The inner representation is `[u8; 4]` holding the **little-endian** bytes of a +//! `u32`, so the on-disk form is identical on every host (portable across +//! endianness). Because the storage is a byte array, ordering is implemented +//! by numeric value rather than raw byte order. + +use bytemuck::{Pod, Zeroable}; + +/// Define a segment-resident ID newtype over a little-endian 4-byte value. +macro_rules! segment_id { + ($name:ident, $doc:literal) => { + #[doc = $doc] + /// + /// Fixed-width 4-byte little-endian value; viewable in place from mmap'd + /// segment bytes via `bytemuck` (`Pod`). + #[derive(Clone, Copy, Default, PartialEq, Eq, Hash, Pod, Zeroable)] + #[repr(transparent)] + pub struct $name([u8; 4]); + + impl $name { + #[doc = concat!("Create a new `", stringify!($name), "` from a raw `u32`.")] + #[must_use] + pub const fn new(value: u32) -> Self { + Self(value.to_le_bytes()) + } + + #[doc = concat!("Get the raw `u32` value of this `", stringify!($name), "`.")] + #[must_use] + pub const fn get(self) -> u32 { + u32::from_le_bytes(self.0) + } + } + + impl core::fmt::Debug for $name { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + write!(f, "{}({})", stringify!($name), self.get()) + } + } + + impl PartialOrd for $name { + fn partial_cmp(&self, other: &Self) -> Option { + Some(self.cmp(other)) + } + } + + impl Ord for $name { + fn cmp(&self, other: &Self) -> core::cmp::Ordering { + self.get().cmp(&other.get()) + } + } + + impl From for $name { + fn from(value: u32) -> Self { + Self::new(value) + } + } + + impl From<$name> for u32 { + fn from(value: $name) -> Self { + value.get() + } + } + }; +} + +segment_id!(BlockId, "Identifier of a postings block within a segment."); +segment_id!(FilterExprId, "Identifier of a stored filter expression."); +segment_id!( + SegmentOrd, + "Ordinal position of a segment within a multi-segment index." +); +segment_id!( + SegmentLocalDocId, + "Document identifier local to a single segment (segment-relative doc ID)." +); + +#[cfg(test)] +mod tests { + use super::*; + + /// SCENARIO-0005: ID type serialization round-trip — each type serializes to + /// a fixed-width 4-byte little-endian value and round-trips losslessly as both + /// a single value and a zero-copy slice view. + #[test] + fn test_single_value_little_endian_roundtrip() { + let id = BlockId::new(0xDEAD_BEEF); + + // Serialized form is exactly the 4-byte little-endian encoding. + let bytes = bytemuck::bytes_of(&id); + assert_eq!(bytes, &0xDEAD_BEEF_u32.to_le_bytes()); + assert_eq!(bytes.len(), 4); + + // Zero-copy view back to the typed value. + let back: &BlockId = bytemuck::from_bytes(bytes); + assert_eq!(*back, id); + assert_eq!(back.get(), 0xDEAD_BEEF); + } + + #[test] + fn test_slice_zero_copy_view_roundtrip() { + let ids = [ + SegmentLocalDocId::new(1), + SegmentLocalDocId::new(0), + SegmentLocalDocId::new(u32::MAX), + SegmentLocalDocId::new(42), + ]; + + // The array's bytes are the concatenated little-endian values. + let bytes: &[u8] = bytemuck::cast_slice(&ids); + assert_eq!(bytes.len(), 4 * ids.len()); + assert_eq!(&bytes[0..4], &1_u32.to_le_bytes()); + assert_eq!(&bytes[12..16], &42_u32.to_le_bytes()); + + // A &[u8] is a zero-copy view of &[SegmentLocalDocId]. + let view: &[SegmentLocalDocId] = bytemuck::cast_slice(bytes); + assert_eq!(view, ids.as_slice()); + } + + #[test] + fn test_unaligned_view_from_offset() { + // [u8; 4] storage is alignment-1, so views work from any byte offset + // (mmap safety) — bytemuck's cast succeeds regardless of source alignment. + let mut buf = [0_u8; 5]; + buf[1..5].copy_from_slice(&7_u32.to_le_bytes()); + let id: &SegmentOrd = bytemuck::from_bytes(&buf[1..5]); + assert_eq!(id.get(), 7); + } + + #[test] + fn test_all_id_types_roundtrip_and_convert() { + assert_eq!(BlockId::new(10).get(), 10); + assert_eq!(FilterExprId::new(20).get(), 20); + assert_eq!(SegmentOrd::new(30).get(), 30); + assert_eq!(SegmentLocalDocId::new(40).get(), 40); + + // u32 <-> ID conversions. + assert_eq!(u32::from(FilterExprId::from(99_u32)), 99); + } + + #[test] + fn test_ordering_is_numeric_not_byte_order() { + // Little-endian byte storage must not corrupt numeric ordering. + assert!(BlockId::new(2) < BlockId::new(256)); + assert!(SegmentLocalDocId::new(0x00FF_0000) < SegmentLocalDocId::new(0x0100_0000)); + } + + #[test] + fn test_debug_shows_numeric_value() { + extern crate alloc; + assert_eq!(alloc::format!("{:?}", BlockId::new(7)), "BlockId(7)"); + } +} From 37611421aec96f5187a4f33a8bcc636fb058b547 Mon Sep 17 00:00:00 2001 From: "Norman Nunley, Jr" Date: Fri, 29 May 2026 22:06:53 -0400 Subject: [PATCH 3/9] docs: add Phase 2 architectural decisions (ITER-0001) Records the design-decidable decisions for the Phase 2 segment format (DEC-01..10) with rationale, a Phase 3 forward-compatibility audit, and decision->enforcement traceability. Human-confirmed key calls: - DEC-01 segment offsets: u64 (no size cap; removes the only Phase 3 format-migration risk) - DEC-10 integrity: single footer checksum, verified in Full validation mode - DEC-06 block-aware API: public dedicated BlockCursor trait (Phase 3 WAND consumes it without a format/API break) - DEC-05 header: fixed-layout little-endian POD, absolute u64 section offsets, magic + version + format_flags, reserved stored-fields/columnar slots Decision-documentation ACs of STORY-0078/0081-0084/0090/0043-0047 are satisfied here (decided:ITER-0001); their code-enforcement ACs are deferred to ITER-0003/0004. Forward constraint recorded for ITER-0005: block-metadata schema must carry per-block max_score + doc-range for Phase 3 WAND/MaxScore. --- ...26-05-30-phase2-architectural-decisions.md | 259 ++++++++++++++++++ 1 file changed, 259 insertions(+) create mode 100644 docs/2026-05-30-phase2-architectural-decisions.md diff --git a/docs/2026-05-30-phase2-architectural-decisions.md b/docs/2026-05-30-phase2-architectural-decisions.md new file mode 100644 index 0000000..ee83e6a --- /dev/null +++ b/docs/2026-05-30-phase2-architectural-decisions.md @@ -0,0 +1,259 @@ +# Phase 2 Architectural Decisions + +**Status:** Decisions of record for ITER-0001. Each decision is *design-decidable +without wind-tunnel measurement*; the code that enforces it is implemented in the +iteration noted under "Enforced by" (the deferred `· deferred:ITER-NNNN` ACs). + +**Grounding:** `docs/leit_kernel_handover.md` §"Segment Architecture" (the format +sketch, versioning bias, and Open Questions) and the ITER-0001 serialization choice +(bytemuck zero-copy, little-endian — see `docs/superpowers/iterations/requirements/EPIC-009.md`). + +**Cross-cutting premise:** segment-resident structures are **zero-copy POD** — +`#[repr(transparent)]`/`#[repr(C)]` over little-endian byte fields, viewed in place +from an mmap'd `&[u8]` via bytemuck, with no deserialization or heap reconstruction +on the read path. Every decision below is consistent with that premise. + +--- + +## DEC-01 — Offset width: u64 (STORY-0043) — RESOLVED + +**Decision:** Segment offsets are **unsigned 64-bit, little-endian**, absolute from +segment start. There is **no practical segment-size cap**. (The handover sketch used +`u32` and flagged the choice as open; the human chose u64 to remove any future +format-break-for-size risk — see Phase 3 forward-compatibility.) + +**Rationale:** u64 offsets future-proof the format against large single segments +(stored fields, columnar, large postings) with no v2 migration ever needed for size. +The cost is a larger header (8-byte offsets, ~84 bytes vs ~44), negligible against +segment size and read once per open. Doc IDs remain segment-local **u32** +(`SegmentLocalDocId`): byte offsets are u64 (file positions may be large), but a single +segment is bounded to 2³² docs — an independent, generous limit that does not interact +with offset width. + +**Verification:** documented here; `SegmentHeader` uses the u64 offset type (enforced +ITER-0004, STORY-0043 AC-3). STORY-0043 AC-2 (32-bit cap documentation) is N/A — u64 +was chosen, so there is no cap to document. + +**Enforced by:** ITER-0004. + +## DEC-02 — Metadata tables: fixed-width entries; variable-width only for term bytes (STORY-0044) + +**Decision:** `field_table` and `postings_table` use **fixed-width entries** so a +reader seeks to entry *i* with O(1) offset arithmetic and views the whole table as a +zero-copy `&[Entry]` (bytemuck slice cast). The **term dictionary** stores the +variable-length term bytes in a blob, addressed by a **fixed-width offset/length index** +(itself a POD table) → O(1) access to any term's bytes without scanning. + +**Rationale:** Handover bias: "direct section lookup", "O(1) offset computation". +Fixed-width tables are exactly the zero-copy POD slices bytemuck gives us. Only term +strings are inherently variable; isolating them behind a fixed-width index keeps every +*metadata* access O(1) while paying variable-width cost only for the term bytes. + +**Enforced by:** ITER-0004 (STORY-0044 AC-2/3). + +## DEC-03 — Dictionary/postings coupling: separately addressable (STORY-0045) + +**Decision:** The term dictionary (lexicon) and the postings metadata table are +**separate sections** with independent header offsets (`lexicon_offset`, +`postings_table_offset`), as in the handover sketch. A lexicon entry yields an index +into the postings table; the two are not interleaved. + +**Rationale:** Separate sections can be validated, mmap'd, and evolved independently, +and keep each section a homogeneous POD table (DEC-02). Interleaving would mix +variable-width term bytes with fixed-width postings metadata, defeating O(1) seeks. +The handover sketch already gives them separate offsets. + +**Enforced by:** ITER-0004 (STORY-0045 AC-2). + +## DEC-04 — Mmap readiness scope (STORY-0046) + +**Decision:** **mmap-friendly in v1** (plain little-endian POD, no heap pointers, no +relocations, viewable zero-copy): segment header, field table, term dictionary +(index + bytes), postings metadata table, postings data blocks, block metadata. +**Deferred / build-time or in-memory only in v1:** optional stored-fields section and +optional columnar section (their slots are reserved in the header but their v1 content +is minimal/empty — full content is Phase 3). + +**Rationale:** The hot read path (header → section tables → postings → blocks) must be +zero-copy mmap for the performance goals. Stored/columnar are optional and off the hot +retrieval path, so they can lag without constraining v1. Reserving header slots now +(DEC-05) keeps the format forward-compatible. + +**Enforced by:** ITER-0004 (STORY-0046 AC-2, SCENARIO-0047), ITER-0005 (mmap loading). + +## DEC-05 — Segment header / offset strategy (STORY-0090) + +**Decision:** A **fixed-layout, little-endian POD header** (bytemuck `Pod`, +alignment-1 byte-field layout consistent with the ID types) as the first bytes of the +segment. Fields: `magic` (u32), `version` (u32), `format_flags` (u32), then the +section offsets **`field_table_offset`, `lexicon_offset`, `postings_table_offset`, +`postings_data_offset`, `block_meta_offset`, `stored_fields_offset`, +`columnar_offset`, `footer_offset` (all u64 LE, per DEC-01)**. **Offsets are absolute +from segment start.** +Endianness is fixed little-endian on every host. A trailing **footer** carries the +optional checksum (DEC-10). + +**Rationale:** Extends the handover sketch (adds `magic`, `format_flags`, +`stored_fields_offset`, `footer_offset`). Absolute offsets are the cheapest to +validate — each must be `<= segment_len` and sections must be non-overlapping/ordered +— and they let a reader jump directly to any section. `format_flags` marks which +optional sections are present (DEC-10). Little-endian POD = the bytemuck zero-copy +premise. + +**Enforced by:** ITER-0004 (STORY-0090 AC-2, SCENARIO-0025). + +## DEC-06 — Block-aware capability scope (STORY-0081) + +**Decision:** Block-aware accessors (`block_max_score()`, `block_end_doc()`, block +boundaries) live on a **dedicated public trait** (e.g. `BlockCursor`) that is +**separate from** the base cursor trait — public so the format's block metadata is +reachable and a future WAND/MaxScore scorer can consume it, but not forced on basic +cursor users. **v1 exposes the data/API surface only; block-skipping *execution* +(WAND pruning) is Phase 3.** + +**Rationale:** Handover makes block metadata "a first-class citizen". Exposing it via a +separate trait honors the architectural split (traversal API vs execution) without +committing v1 to the pruning algorithm. Keeping it off the base trait avoids boxing in +non-block cursors. + +**Verification:** trait visibility/boundary enforced in code (ITER-0003, +STORY-0081 AC-2). **RESOLVED — public dedicated `BlockCursor` trait (confirmed).** + +**Enforced by:** ITER-0003. + +## DEC-07 — Segment validation strategy: three lazy modes (STORY-0082) + +**Decision:** A `ValidationMode` enum with **`HeaderOnly`**, **`Structural`** (default), +and **`Full`**: +- `HeaderOnly` — verify magic + version (reject unsupported) + header self-consistency. Cheapest open. +- `Structural` — additionally verify every offset is in-bounds and sections are ordered/non-overlapping. The default for `SegmentView::open()`. +- `Full` — additionally verify the footer checksum (DEC-10) and per-section structural invariants. + +Validation is otherwise **lazy**: per-access reads use bytemuck `try_*` casts, which +bounds-check length at the point of use regardless of mode, so traversal is always +memory-safe even under `HeaderOnly`. + +**Rationale:** Balances startup latency against safety (handover: "easy validation of +offset ranges"). `Structural` is cheap (a handful of comparisons) and catches the +common corruption/truncation cases, so it's the safe default; `Full` is opt-in for +untrusted inputs. Because `try_cast` validates every slice access anyway, even the +cheapest mode cannot cause UB. + +**Verification:** `ValidationMode` + `open_with_validation()` (ITER-0004, +STORY-0082 AC-2, SCENARIO-0020). **RESOLVED — three modes; `Structural` default; `Full` +adds the DEC-10 footer checksum (not surfaced for change; recorded default stands).** + +**Enforced by:** ITER-0004. + +## DEC-08 — View borrowing model: fully borrowed (STORY-0083) + +**Decision:** All segment section views **borrow `&[u8]` directly** (zero-copy). View +types hold a byte slice plus validated offsets/lengths; there are **no eagerly-decoded +owned fields**. The only work at view-construction time is bounds validation (returns +indices, not copies). + +**Rationale:** Directly follows the bytemuck zero-copy premise and the handover's +"low-cost borrowed views" + minimal-allocation principle. Any eager copy would +reintroduce allocation on the hot path. Decoded values (e.g. a `u32` from an offset +table) are produced on demand by cheap LE reads, not cached. + +**Enforced by:** ITER-0004 (STORY-0083 AC-2). + +## DEC-09 — Builder vs read-only type separation: strict (STORY-0084) + +**Decision:** **Strict separation.** Writer/builder types (`SegmentBuilder` and +friends) that accumulate and serialize a segment are **distinct** from the read-only +view types (`SegmentView` and section views). Builder types never appear in query/read +paths; view types are never mutable post-construction. + +**Rationale:** Segments are immutable; conflating builder and reader invites misuse +(e.g. mutating a "view"). The handover's clean split — on-disk representation ↔ +traversal views ↔ orchestration — is naturally expressed as writer-produces-bytes, +reader-views-bytes. Strict separation also lets the reader stay `no_std`/zero-copy +while the builder may freely allocate. + +**Enforced by:** ITER-0004 (STORY-0084 AC-2). + +## DEC-10 — Versioning & checking scope (STORY-0047) + +**Decision (minimal but real):** +- **Version:** `version: u32` in the header; readers **reject unsupported (future) versions cleanly** with a structured error. Backward compatibility is promised for a *bounded* set of versions; dropping support means rebuild-via-tooling (handover model). +- **Feature flags:** `format_flags: u32` bitfield marks which optional sections are present (stored_fields, columnar), so a reader knows whether those offsets are meaningful. +- **Checksum:** a **single footer checksum** over the segment body (algorithm: a fast non-cryptographic hash — candidate: the same rapidhash family already vendored, or crc32c). Validated only in `Full` mode (DEC-07). **No per-section checksums in v1.** +- **Magic:** a 4-byte magic constant as the first header field for quick format identification. + +**Rationale:** Handover wants explicit versioning + clean rejection from the start, but +warns against over-engineering. Version + flags + one optional checksum + magic is the +minimal set that supports clean rejection, optional-section detection, and integrity +checking, without the cost/complexity of per-section checksums. + +**Verification:** version-rejection + flag handling + checksum (ITER-0004, +STORY-0047 AC-2/3). **RESOLVED — include a single footer checksum in v1 (confirmed); +algorithm finalized in ITER-0004 (rapidhash-family or crc32c).** + +**Enforced by:** ITER-0004. + +--- + +## Decisions epic anchor (STORY-0078) + +All eight-plus must-resolve decisions above are recorded with a decision, a rationale, +and a verification method (the enforcing iteration + AC/scenario). Dependent +implementation (ITER-0002 codecs, ITER-0003 cursors, ITER-0004 segment format) may now +proceed against a stable decision point. Decisions flagged **[SURFACE]** were raised to +the human for confirmation; the recorded choice is the confirmed/default position. + +## Phase 3 forward-compatibility (does any v1 decision box in Phase 3?) + +Phase 3 adds **WAND / MaxScore block-skipping execution**, **columnar field content**, +and **real-corpus loading**. The v1 decisions are designed to *enable* these without a +format break: + +1. **Block metadata is first-class and mmap-friendly in v1** (DEC-04), even though + block-skipping *execution* is Phase 3 (DEC-06). The per-block `max_score` and doc + ranges live in the format now, so Phase 3 WAND reads them with **no format change** — + this is the handover's explicit "first-class, not bolted on later" intent. +2. **The block-aware API surface ships in v1** as a dedicated `BlockCursor` trait + (DEC-06) and the MaxScoreScorer block-structure contract (ITER-0003, SCENARIO-0002). + Phase 3 pruning is a new *consumer* of an existing surface, not an API break. +3. **Optional-section slots are reserved now** (DEC-05 header carries + `stored_fields_offset`/`columnar_offset`; DEC-10 `format_flags` marks presence). + Phase 3 fills the columnar slot and sets its flag — **no version bump required**; + old readers see the flag clear and ignore it. +4. **Explicit versioning + rebuild path** (DEC-01, DEC-10): any change that *cannot* be + done via a reserved slot is a clean `version` bump with clean rejection by old + readers and a tooling rebuild — never in-hot-path legacy support. +5. **Zero-copy bytemuck POD** generalizes: Phase 3's columnar and block structures are + also flat POD tables, so the same `try_*` validated casts apply. No serialization + rework. + +**No segment-size limit:** DEC-01 was resolved to **u64 offsets**, so the previously +identified 4 GiB cap is gone — there is no v1 decision that forces a future +format-break for segment size. The only residual bound is `SegmentLocalDocId` = u32 +(≤ 2³² docs per segment), which is independent of offset width and far beyond any +realistic single-segment doc count; it does not constrain Phase 3. + +**Forward constraint placed on ITER-0005 (block-metadata schema, STORY-0086):** the +block-metadata section's v1 schema **must** carry, per block, at least `max_score` and +the doc-range needed for WAND/MaxScore skipping, so Phase 3 can prune without a format +change. Recorded here so the ITER-0005 doc-range decision honors it. + +**Completed work check:** ITER-0000 (wind tunnel) is additive measurement infra and +boxes nothing; its STORY-0105 real-corpus path is explicitly a Phase 3+ plug-in against +stable output types. ITER-0001 ID types (`BlockId`, etc.) are precisely what Phase 3 +WAND consumes — defining them now *helps* Phase 3. + +## Decision → enforcement traceability + +| Decision | Story | Enforced by | Proof | +|---|---|---|---| +| DEC-01 offset width u64 | STORY-0043 | ITER-0004 | header uses u64 offset type | +| DEC-02 fixed-width tables | STORY-0044 | ITER-0004 | O(1) entry seek | +| DEC-03 separate sections | STORY-0045 | ITER-0004 | independent offsets | +| DEC-04 mmap scope | STORY-0046 | ITER-0004/0005 | SCENARIO-0047 | +| DEC-05 header/offset strategy | STORY-0090 | ITER-0004 | SCENARIO-0025 | +| DEC-06 block-aware trait | STORY-0081 | ITER-0003 | trait visibility | +| DEC-07 validation modes | STORY-0082 | ITER-0004 | SCENARIO-0020 | +| DEC-08 fully borrowed views | STORY-0083 | ITER-0004 | view accessors | +| DEC-09 strict builder/reader | STORY-0084 | ITER-0004 | type boundary | +| DEC-10 versioning/checks | STORY-0047 | ITER-0004 | version-rejection | From 67e83441248f8dd59f21161e4e4555b28b236ab4 Mon Sep 17 00:00:00 2001 From: "Norman Nunley, Jr" Date: Fri, 29 May 2026 22:43:49 -0400 Subject: [PATCH 4/9] test(leit_core): prove validated try_* reads for segment ID types (STORY-0112 AC-2) ITER-0001 audit corrective: SCENARIO-0005 now also exercises try_from_bytes/ try_cast_slice (Ok on well-formed, Err on malformed) per AC-2's validated-read obligation. --- crates/leit_core/src/segment_ids.rs | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/crates/leit_core/src/segment_ids.rs b/crates/leit_core/src/segment_ids.rs index 6256026..9eec37a 100644 --- a/crates/leit_core/src/segment_ids.rs +++ b/crates/leit_core/src/segment_ids.rs @@ -128,6 +128,33 @@ mod tests { assert_eq!(view, ids.as_slice()); } + #[test] + fn test_validated_reads_use_try_cast_variants() { + // SCENARIO-0005 (AC-2 validated-read obligation): reads from untrusted + // segment bytes go through the fallible `try_*` casts, which return `Err` + // on a malformed slice instead of panicking. + + // Correctly-sized 4-byte slice -> Ok. + let raw = 0x1234_5678_u32.to_le_bytes(); + let ok: &BlockId = bytemuck::try_from_bytes(&raw).expect("4-byte slice is a valid BlockId"); + assert_eq!(ok.get(), 0x1234_5678); + + // Wrong-length slice -> Err, never a panic. + let too_short = [0_u8; 3]; + assert!(bytemuck::try_from_bytes::(&too_short).is_err()); + + // try_cast_slice yields a zero-copy &[Id] view for an exact multiple... + let ids = [SegmentLocalDocId::new(5), SegmentLocalDocId::new(6)]; + let bytes: &[u8] = bytemuck::cast_slice(&ids); + let view: &[SegmentLocalDocId] = + bytemuck::try_cast_slice(bytes).expect("8 bytes round-trips to 2 ids"); + assert_eq!(view, ids.as_slice()); + + // ...and rejects a length that is not a whole number of ids. + let ragged = [0_u8; 7]; + assert!(bytemuck::try_cast_slice::(&ragged).is_err()); + } + #[test] fn test_unaligned_view_from_offset() { // [u8; 4] storage is alignment-1, so views work from any byte offset From 1d204c575772236ad8a8f4b430d7ae0c283ec07f Mon Sep 17 00:00:00 2001 From: "Norman Nunley, Jr" Date: Wed, 15 Jul 2026 07:46:22 -0400 Subject: [PATCH 5/9] fix: address phase 2 review feedback --- crates/leit_core/src/segment_ids.rs | 2 +- docs/2026-05-30-phase2-architectural-decisions.md | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/crates/leit_core/src/segment_ids.rs b/crates/leit_core/src/segment_ids.rs index 9eec37a..78cbfa0 100644 --- a/crates/leit_core/src/segment_ids.rs +++ b/crates/leit_core/src/segment_ids.rs @@ -25,7 +25,7 @@ macro_rules! segment_id { /// /// Fixed-width 4-byte little-endian value; viewable in place from mmap'd /// segment bytes via `bytemuck` (`Pod`). - #[derive(Clone, Copy, Default, PartialEq, Eq, Hash, Pod, Zeroable)] + #[derive(Clone, Copy, PartialEq, Eq, Hash, Pod, Zeroable)] #[repr(transparent)] pub struct $name([u8; 4]); diff --git a/docs/2026-05-30-phase2-architectural-decisions.md b/docs/2026-05-30-phase2-architectural-decisions.md index ee83e6a..00a2e41 100644 --- a/docs/2026-05-30-phase2-architectural-decisions.md +++ b/docs/2026-05-30-phase2-architectural-decisions.md @@ -4,9 +4,9 @@ without wind-tunnel measurement*; the code that enforces it is implemented in the iteration noted under "Enforced by" (the deferred `· deferred:ITER-NNNN` ACs). -**Grounding:** `docs/leit_kernel_handover.md` §"Segment Architecture" (the format -sketch, versioning bias, and Open Questions) and the ITER-0001 serialization choice -(bytemuck zero-copy, little-endian — see `docs/superpowers/iterations/requirements/EPIC-009.md`). +**Grounding:** the Phase 2 handover's segment-architecture sketch, versioning bias, +and open questions, plus the ITER-0001 serialization choice (bytemuck zero-copy, +little-endian). **Cross-cutting premise:** segment-resident structures are **zero-copy POD** — `#[repr(transparent)]`/`#[repr(C)]` over little-endian byte fields, viewed in place From 46adc0aaa32b27876daabcf3a5bae2f5c8c7406a Mon Sep 17 00:00:00 2001 From: "Norman Nunley, Jr" Date: Fri, 29 May 2026 22:43:49 -0400 Subject: [PATCH 6/9] feat(leit_postings): compressed postings codecs (DeltaVarint + BlockDelta) [ITER-0002] MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Codec layer for ITER-0002. A Codec trait with two implementations over a stable v1 block format, plus the layout decisions (DEC-11 fixed 128-doc blocks, DEC-12 layout) and a new TermFreq segment-resident type. - DeltaVarint (CodecId 0) + BlockDelta (CodecId 1, 128-doc independently-decodable blocks with validated first/last-doc header range). - Hand-rolled LEB128 varint into a type-enforced [u8;5]; no_std + alloc; no new deps. - API speaks named segment-resident types SegmentLocalDocId + TermFreq (no anonymous u32 drift); EntityId stays the in-memory abstraction, lowered at the segment boundary. - Decode into caller-provided &mut Vec<..> — scratch-ownership-agnostic (TODO(ITER-0003) / STORY-0079). Doc-sorted precondition enforced via checked_sub (deterministic panic). - CodecId marker per list; segment-format reservation deferred:ITER-0004. Stories: STORY-0002/0003/0004/0005(AC1-2)/0009 done; STORY-0087/0088 decided. Proof: SCENARIO-0006 (36 leit_postings tests). PAR spec + quality reviewed. Co-Authored-By: Claude Opus 4.8 (1M context) --- Cargo.lock | 1 + crates/leit_core/src/lib.rs | 2 +- crates/leit_core/src/segment_ids.rs | 16 +- crates/leit_postings/src/codec.rs | 1162 +++++++++++++++++ crates/leit_postings/src/lib.rs | 2 + ...26-05-30-phase2-architectural-decisions.md | 70 + 6 files changed, 1249 insertions(+), 4 deletions(-) create mode 100644 crates/leit_postings/src/codec.rs diff --git a/Cargo.lock b/Cargo.lock index dde280a..977a982 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -663,6 +663,7 @@ dependencies = [ "criterion", "leit_core", "leit_index", + "leit_postings", "leit_text", "leit_wind_tunnel", ] diff --git a/crates/leit_core/src/lib.rs b/crates/leit_core/src/lib.rs index c2f5973..aef0021 100644 --- a/crates/leit_core/src/lib.rs +++ b/crates/leit_core/src/lib.rs @@ -21,7 +21,7 @@ use core::hash::Hash; use core::ops::{Add, AddAssign, Mul, MulAssign, Sub, SubAssign}; pub mod segment_ids; -pub use segment_ids::{BlockId, FilterExprId, SegmentLocalDocId, SegmentOrd}; +pub use segment_ids::{BlockId, FilterExprId, SegmentLocalDocId, SegmentOrd, TermFreq}; /// Unique identifier for a field in an index. #[derive(Clone, Copy, Debug, Default, PartialEq, Eq, PartialOrd, Ord, Hash)] diff --git a/crates/leit_core/src/segment_ids.rs b/crates/leit_core/src/segment_ids.rs index 78cbfa0..f18acce 100644 --- a/crates/leit_core/src/segment_ids.rs +++ b/crates/leit_core/src/segment_ids.rs @@ -1,11 +1,16 @@ // Copyright 2026 the Leit Authors // SPDX-License-Identifier: Apache-2.0 OR MIT -//! Segment-resident core ID types with a stable, zero-copy serialized representation. +//! Segment-resident core ID and value types with a stable, zero-copy serialized +//! representation. //! //! Unlike the in-memory index identifiers ([`FieldId`](crate::FieldId), -//! [`TermId`](crate::TermId), [`SegmentId`](crate::SegmentId)), the types in this -//! module are designed to appear **directly in mmap'd segment bytes**. Each is a +//! [`TermId`](crate::TermId), [`SegmentId`](crate::SegmentId)) and the polymorphic +//! in-memory document identifier ([`EntityId`](crate::EntityId)), the types in this +//! module are designed to appear **directly in mmap'd segment bytes**. The generic +//! `EntityId` is *lowered* to the concrete [`SegmentLocalDocId`] when a segment is +//! written; segment-layer code (postings codecs, readers) speaks these named types, +//! never anonymous `u32`. Each is a //! `#[repr(transparent)]` newtype over a 4-byte little-endian array, so a `&[u8]` //! slice taken from a memory-mapped buffer can be viewed in place as `&[Id]` with //! no allocation and no deserialization pass (see the Phase 2 architectural @@ -85,6 +90,11 @@ segment_id!( SegmentLocalDocId, "Document identifier local to a single segment (segment-relative doc ID)." ); +segment_id!( + TermFreq, + "Term frequency: occurrences of a term in a document. A segment-resident value \ + type (not an identifier) carried alongside `SegmentLocalDocId` in postings." +); #[cfg(test)] mod tests { diff --git a/crates/leit_postings/src/codec.rs b/crates/leit_postings/src/codec.rs new file mode 100644 index 0000000..1f6b9af --- /dev/null +++ b/crates/leit_postings/src/codec.rs @@ -0,0 +1,1162 @@ +// Copyright 2026 the Leit Authors +// SPDX-License-Identifier: Apache-2.0 OR MIT + +//! Compressed postings codecs for efficient storage and traversal. +//! +//! This module provides codec implementations for encoding and decoding postings lists. +//! Multiple codec strategies are supported to balance decode cost and memory footprint: +//! +//! - **`DeltaVarint`**: Single-block encoding using delta-encoded doc IDs and varint-encoded TFs. +//! - **`BlockDelta`**: Multi-block encoding with 128-doc blocks, each independently decodable. +//! +//! ## Codec ID marker +//! +//! Encoded postings are prefixed by a 1-byte `CodecId` to support multiple codec implementations. +//! See DEC-12 in the architectural decisions for the full specification. + +use alloc::vec::Vec; +use core::fmt; +use leit_core::{SegmentLocalDocId, TermFreq}; + +/// Fixed block size for `BlockDelta` codec: 128 documents per block. +/// +/// This constant is defined in one place (DEC-11) so that a future codec may tune it +/// without a format break. The block count is encoded per block, not assumed by readers. +pub const BLOCK_DOC_COUNT: usize = 128; + +/// Codec identifier for selecting among multiple postings codec implementations. +/// +/// Each postings list is prefixed by a single byte codec marker (1-byte `CodecId`). +/// Decoders read this marker to dispatch to the correct codec implementation. +#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord)] +#[repr(u8)] +pub enum CodecId { + /// Delta-encoded doc IDs + varint-encoded TFs in a single stream (no block structure). + DeltaVarint = 0, + /// Block-based codec: 128-doc blocks, each independently decodable. + BlockDelta = 1, +} + +impl CodecId { + /// Convert a byte to a `CodecId`, returning `None` if the byte is not a valid marker. + pub fn from_u8(byte: u8) -> Option { + match byte { + 0 => Some(Self::DeltaVarint), + 1 => Some(Self::BlockDelta), + _ => None, + } + } + + /// Convert this `CodecId` to a byte. + pub const fn to_u8(self) -> u8 { + self as u8 + } +} + +/// Errors that can occur during codec operations. +#[derive(Clone, Debug, PartialEq, Eq)] +pub enum CodecError { + /// The byte stream is truncated or incomplete. + Truncated, + /// The codec ID marker is not recognized. + BadMarker(u8), + /// Invalid block count in block header. + InvalidBlockCount, + /// Invalid varint encoding. + InvalidVarint, + /// A block header's `first_doc`/`last_doc` range does not match its decoded doc stream. + BlockHeaderMismatch, +} + +impl fmt::Display for CodecError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Self::Truncated => write!(f, "byte stream truncated"), + Self::BadMarker(byte) => write!(f, "unrecognized codec ID: {byte}"), + Self::InvalidBlockCount => write!(f, "invalid block count"), + Self::InvalidVarint => write!(f, "invalid varint encoding"), + Self::BlockHeaderMismatch => { + write!(f, "block header doc-range does not match decoded stream") + } + } + } +} + +#[cfg(feature = "std")] +impl core::error::Error for CodecError {} + +/// Core codec interface for encoding and decoding postings. +/// +/// A codec is responsible for: +/// - Encoding a sequence of (`SegmentLocalDocId`, `TermFreq`) pairs into a compressed byte stream. +/// - Decoding the byte stream back into the original pairs. +/// +/// ## Encode input +/// +/// The input to `encode()` is a slice of `(SegmentLocalDocId, TermFreq)` tuples. +/// Doc IDs must be doc-sorted (ascending order) for delta encoding to be effective. +/// +/// ## Decode output +/// +/// The decode API writes into **caller-provided output buffers** (`&mut Vec` +/// and `&mut Vec`) rather than allocating owned decode structures. This design keeps +/// the codec layer scratch-ownership-agnostic; see DEC-12 and the TODO comment below. +/// +/// The caller is responsible for clearing or reusing these buffers. +/// +/// ## Codec marker +/// +/// The encoded byte stream includes a 1-byte `CodecId` prefix. +/// The `encode()` method includes this prefix in the returned bytes. +/// The `decode()` method expects the bytes to start with the marker. +pub trait Codec { + /// Return the codec ID for this implementation. + fn id(&self) -> CodecId; + + /// Encode a sequence of (`SegmentLocalDocId`, `TermFreq`) tuples into a compressed byte stream. + /// + /// The returned bytes include a 1-byte codec ID prefix. + /// + /// The v1 codec layer encodes only `(SegmentLocalDocId, TermFreq)`. Posting **positions** + /// (`Posting::positions`) are intentionally out of scope here — positions/TF + /// layering is decided in ITER-0003 (STORY-0080); a future codec or a parallel + /// positions section will carry them without changing this format. + /// + /// # Arguments + /// + /// - `postings`: slice of (`SegmentLocalDocId`, `TermFreq`) pairs, must be doc-sorted (ascending). + /// + /// # Panics + /// + /// May panic if postings are not doc-sorted. + fn encode(&self, postings: &[(SegmentLocalDocId, TermFreq)]) -> Vec; + + /// Decode a byte stream into doc IDs and term frequencies. + /// + /// This method writes decoded values into caller-provided output buffers. + /// The input bytes must start with a valid codec ID marker. + /// + /// The method validates the codec ID and delegates to the appropriate codec + /// decoder, which writes exactly `len(postings)` values into `out_docs` and + /// `out_tfs` in doc-ascending order. + /// + /// # Arguments + /// + /// - `bytes`: compressed postings with a 1-byte codec ID prefix. + /// - `out_docs`: output buffer for decoded doc IDs (will be cleared and refilled). + /// - `out_tfs`: output buffer for decoded term frequencies (will be cleared and refilled). + /// + /// # Returns + /// + /// `Ok(())` on success, or a `CodecError` if decoding fails. + /// + /// # TODO(ITER-0003): `DecodeScratch` wrapper + /// + /// The decode-scratch ownership model (e.g., a `DecodeScratch` struct holding + /// these buffers and reusable across multiple cursors) is designed and built + /// in ITER-0003 (STORY-0079). This codec layer intentionally does NOT define + /// a decode-scratch type so that decision is not boxed in here. + fn decode( + &self, + bytes: &[u8], + out_docs: &mut Vec, + out_tfs: &mut Vec, + ) -> Result<(), CodecError>; +} + +/// Helper to encode a u32 as LEB128 varint. +/// +/// Writes 1–5 bytes into `out` (a `u32` LEB128 is at most 5 bytes, so a fixed +/// `[u8; 5]` buffer can never overflow), returns the number of bytes written. +fn encode_varint(value: u32, out: &mut [u8; 5]) -> usize { + let mut v = value; + let mut len = 0; + loop { + let mut byte = (v & 0x7f) as u8; + v >>= 7; + if v != 0 { + byte |= 0x80; + } + out[len] = byte; + len += 1; + if v == 0 { + break; + } + } + len +} + +/// Helper to decode a u32 from LEB128 varint. +/// +/// Returns `(value, bytes_consumed)` on success, or `CodecError::InvalidVarint` on failure. +fn decode_varint(bytes: &[u8]) -> Result<(u32, usize), CodecError> { + let mut value = 0_u32; + let mut shift = 0; + let mut pos = 0; + + loop { + if pos >= bytes.len() { + return Err(CodecError::Truncated); + } + + let byte = bytes[pos]; + pos += 1; + + value |= ((byte & 0x7f) as u32) << shift; + + if (byte & 0x80) == 0 { + return Ok((value, pos)); + } + + shift += 7; + if shift >= 32 { + return Err(CodecError::InvalidVarint); + } + } +} + +/// Delta-encoding codec: single-block varint encoding. +/// +/// This codec encodes the postings list as a single block with: +/// - Delta-encoded doc IDs (deltas from previous doc, first delta from 0). +/// - Varint-encoded term frequencies (parallel to doc stream). +/// +/// This is a simpler, single-block alternative to `BlockDelta` for small postings lists. +#[derive(Clone, Copy, Debug)] +pub struct DeltaVarintCodec; + +impl Codec for DeltaVarintCodec { + fn id(&self) -> CodecId { + CodecId::DeltaVarint + } + + fn encode(&self, postings: &[(SegmentLocalDocId, TermFreq)]) -> Vec { + let mut result = Vec::with_capacity(postings.len() * 5 + 10); + result.push(CodecId::DeltaVarint.to_u8()); + + let mut buf = [0_u8; 5]; + let mut prev_doc = 0_u32; + + for (doc_id, tf) in postings { + // Postings MUST be doc-sorted ascending (the documented precondition). + // `checked_sub` turns a violation into a deterministic panic rather than + // a silently-wrapping delta that would corrupt the encoded stream. + let doc_id_u32 = doc_id.get(); + let delta = doc_id_u32 + .checked_sub(prev_doc) + .expect("postings must be doc-sorted ascending"); + let bytes_written = encode_varint(delta, &mut buf); + result.extend_from_slice(&buf[..bytes_written]); + + let bytes_written = encode_varint(tf.get(), &mut buf); + result.extend_from_slice(&buf[..bytes_written]); + + prev_doc = doc_id_u32; + } + + result + } + + fn decode( + &self, + bytes: &[u8], + out_docs: &mut Vec, + out_tfs: &mut Vec, + ) -> Result<(), CodecError> { + out_docs.clear(); + out_tfs.clear(); + + if bytes.is_empty() { + return Err(CodecError::Truncated); + } + + let marker = bytes[0]; + if marker != CodecId::DeltaVarint.to_u8() { + return Err(CodecError::BadMarker(marker)); + } + + let mut pos = 1; + let mut prev_doc = 0_u32; + + while pos < bytes.len() { + let (delta, delta_len) = decode_varint(&bytes[pos..])?; + pos += delta_len; + + if pos >= bytes.len() { + return Err(CodecError::Truncated); + } + + let doc_id = prev_doc + .checked_add(delta) + .ok_or(CodecError::InvalidVarint)?; + let (tf, tf_len) = decode_varint(&bytes[pos..])?; + pos += tf_len; + + out_docs.push(SegmentLocalDocId::new(doc_id)); + out_tfs.push(TermFreq::new(tf)); + prev_doc = doc_id; + } + + Ok(()) + } +} + +/// Block-delta codec: multi-block encoding with 128-doc blocks. +/// +/// This codec divides postings into fixed-size blocks of `BLOCK_DOC_COUNT` documents. +/// Each block is independently decodable, enabling selective decode and block-aware +/// traversal for future pruning (Phase 3). +/// +/// ## Block format +/// +/// The block format (not Rust code): +/// +/// ```text +/// block := block_header doc_id_stream tf_stream +/// block_header := varint(doc_count) varint(first_doc) varint(last_doc) varint(doc_bytes_len) +/// doc_id_stream := varint(first_delta) varint(delta)* # deltas from previous doc +/// tf_stream := varint(tf)* # one per doc, parallel to doc stream +/// ``` +/// +/// The `doc_count` in the header is used by readers but is implicit from input length. +/// The `first_doc` is stored as an absolute value (delta from 0), so blocks are self-contained. +/// The `doc_bytes_len` allows readers to skip to the TF stream without decoding doc deltas. +#[derive(Clone, Copy, Debug)] +pub struct BlockDeltaCodec; + +impl Codec for BlockDeltaCodec { + fn id(&self) -> CodecId { + CodecId::BlockDelta + } + + fn encode(&self, postings: &[(SegmentLocalDocId, TermFreq)]) -> Vec { + let mut result = Vec::with_capacity(postings.len() * 5 + 100); + result.push(CodecId::BlockDelta.to_u8()); + + let mut buf = [0_u8; 5]; + + for block_chunk in postings.chunks(BLOCK_DOC_COUNT) { + let doc_count = block_chunk.len(); + let first_doc = block_chunk[0].0.get(); + let last_doc = block_chunk[block_chunk.len() - 1].0.get(); + + // Encode block header: doc_count, first_doc, last_doc, doc_bytes_len. + // We'll first encode doc stream in a temporary buffer to get its length. + + // SAFETY: doc_count is usize from chunk size, bounded by BLOCK_DOC_COUNT (128). + #[expect( + clippy::cast_possible_truncation, + reason = "doc_count bounded by BLOCK_DOC_COUNT" + )] + let bytes = encode_varint(doc_count as u32, &mut buf); + result.extend_from_slice(&buf[..bytes]); + + let bytes = encode_varint(first_doc, &mut buf); + result.extend_from_slice(&buf[..bytes]); + + let bytes = encode_varint(last_doc, &mut buf); + result.extend_from_slice(&buf[..bytes]); + + // Encode doc ID stream in a temporary buffer to determine its length. + let mut doc_stream = Vec::new(); + let mut prev_doc = 0_u32; + for (doc_id, _) in block_chunk { + // Doc-sorted precondition (see DeltaVarintCodec::encode): a violation + // panics deterministically rather than wrapping into a corrupt delta. + let doc_id_u32 = doc_id.get(); + let delta = doc_id_u32 + .checked_sub(prev_doc) + .expect("postings must be doc-sorted ascending"); + let bytes = encode_varint(delta, &mut buf); + doc_stream.extend_from_slice(&buf[..bytes]); + prev_doc = doc_id_u32; + } + + // Now encode doc_bytes_len. + // SAFETY: doc_bytes_len is per-block, bounded by BLOCK_DOC_COUNT * varint_max_bytes. + let doc_bytes_len = doc_stream.len(); + #[expect( + clippy::cast_possible_truncation, + reason = "doc_bytes_len bounded by BLOCK_DOC_COUNT * 5" + )] + let bytes = encode_varint(doc_bytes_len as u32, &mut buf); + result.extend_from_slice(&buf[..bytes]); + + // Append the doc stream. + result.extend_from_slice(&doc_stream); + + // Encode TF values. + for (_, tf) in block_chunk { + let bytes = encode_varint(tf.get(), &mut buf); + result.extend_from_slice(&buf[..bytes]); + } + } + + result + } + + fn decode( + &self, + bytes: &[u8], + out_docs: &mut Vec, + out_tfs: &mut Vec, + ) -> Result<(), CodecError> { + out_docs.clear(); + out_tfs.clear(); + + if bytes.is_empty() { + return Err(CodecError::Truncated); + } + + let marker = bytes[0]; + if marker != CodecId::BlockDelta.to_u8() { + return Err(CodecError::BadMarker(marker)); + } + + let mut pos = 1; + + while pos < bytes.len() { + // Decode block header. + let (doc_count, bytes_read) = decode_varint(&bytes[pos..])?; + pos += bytes_read; + let doc_count = doc_count as usize; + + if doc_count == 0 { + return Err(CodecError::InvalidBlockCount); + } + + let (first_doc, bytes_read) = decode_varint(&bytes[pos..])?; + pos += bytes_read; + + let (last_doc, bytes_read) = decode_varint(&bytes[pos..])?; + pos += bytes_read; + + let (doc_bytes_len, bytes_read) = decode_varint(&bytes[pos..])?; + pos += bytes_read; + let doc_bytes_len = doc_bytes_len as usize; + + if pos + doc_bytes_len > bytes.len() { + return Err(CodecError::Truncated); + } + + // Decode doc ID stream. + let doc_stream_start = pos; + let doc_stream_end = pos + doc_bytes_len; + let doc_stream = &bytes[doc_stream_start..doc_stream_end]; + + let mut doc_pos = 0; + let mut prev_doc = 0_u32; + let block_doc_start = out_docs.len(); + + for _ in 0..doc_count { + if doc_pos >= doc_stream.len() { + return Err(CodecError::Truncated); + } + let (delta, bytes_read) = decode_varint(&doc_stream[doc_pos..])?; + doc_pos += bytes_read; + + let doc_id = prev_doc + .checked_add(delta) + .ok_or(CodecError::InvalidVarint)?; + out_docs.push(SegmentLocalDocId::new(doc_id)); + prev_doc = doc_id; + } + + // Validate the block header's doc-range against the decoded stream. + // This makes `first_doc`/`last_doc` (carried for ITER-0003 block-skip and + // ITER-0005 WAND doc-range) self-checking rather than dead bytes, and + // detects corruption. `prev_doc` now holds the block's last decoded doc. + if out_docs[block_doc_start].get() != first_doc || prev_doc != last_doc { + return Err(CodecError::BlockHeaderMismatch); + } + + pos = doc_stream_end; + + // Decode TF stream (parallel to doc stream). + for _ in 0..doc_count { + if pos >= bytes.len() { + return Err(CodecError::Truncated); + } + let (tf, bytes_read) = decode_varint(&bytes[pos..])?; + pos += bytes_read; + out_tfs.push(TermFreq::new(tf)); + } + } + + Ok(()) + } +} + +/// Decode a postings byte stream using any codec. +/// +/// This free function dispatches on the codec ID marker and calls the appropriate +/// codec's decode method. +/// +/// # Arguments +/// +/// - `bytes`: the encoded postings (must start with a valid `CodecId` marker). +/// - `out_docs`: output buffer for doc IDs. +/// - `out_tfs`: output buffer for term frequencies. +/// +/// # Returns +/// +/// `Ok(())` on success, or a `CodecError` on failure. +pub fn decode_any( + bytes: &[u8], + out_docs: &mut Vec, + out_tfs: &mut Vec, +) -> Result<(), CodecError> { + if bytes.is_empty() { + return Err(CodecError::Truncated); + } + + let marker = bytes[0]; + match CodecId::from_u8(marker) { + Some(CodecId::DeltaVarint) => DeltaVarintCodec.decode(bytes, out_docs, out_tfs), + Some(CodecId::BlockDelta) => BlockDeltaCodec.decode(bytes, out_docs, out_tfs), + None => Err(CodecError::BadMarker(marker)), + } +} + +#[cfg(test)] +mod tests { + use super::*; + use alloc::vec; + use alloc::vec::Vec; + + // ===== Varint Tests ===== + + #[test] + fn test_varint_encode_decode_zero() { + let mut buf = [0_u8; 5]; + let len = encode_varint(0, &mut buf); + assert_eq!(len, 1); + assert_eq!(buf[0], 0); + + let (decoded, bytes_read) = decode_varint(&buf[..len]).unwrap(); + assert_eq!(decoded, 0); + assert_eq!(bytes_read, 1); + } + + #[test] + fn test_varint_encode_decode_small() { + let mut buf = [0_u8; 5]; + let len = encode_varint(42, &mut buf); + assert_eq!(len, 1); + + let (decoded, bytes_read) = decode_varint(&buf[..len]).unwrap(); + assert_eq!(decoded, 42); + assert_eq!(bytes_read, 1); + } + + #[test] + fn test_varint_encode_decode_large() { + let mut buf = [0_u8; 5]; + let len = encode_varint(16384, &mut buf); + assert_eq!(len, 3); + + let (decoded, bytes_read) = decode_varint(&buf[..len]).unwrap(); + assert_eq!(decoded, 16384); + assert_eq!(bytes_read, 3); + } + + #[test] + fn test_varint_encode_decode_max() { + let mut buf = [0_u8; 5]; + let len = encode_varint(u32::MAX, &mut buf); + assert_eq!(len, 5); + + let (decoded, bytes_read) = decode_varint(&buf[..len]).unwrap(); + assert_eq!(decoded, u32::MAX); + assert_eq!(bytes_read, 5); + } + + #[test] + fn test_varint_decode_truncated() { + let bytes = [0x80]; // Incomplete varint. + let result = decode_varint(&bytes); + assert_eq!(result, Err(CodecError::Truncated)); + } + + // ===== DeltaVarint Codec Tests ===== + + #[test] + fn test_delta_varint_round_trip_empty() { + let codec = DeltaVarintCodec; + let postings: &[(SegmentLocalDocId, TermFreq)] = &[]; + + let encoded = codec.encode(postings); + assert_eq!(encoded.len(), 1); // Only the marker byte. + + let mut docs = Vec::new(); + let mut tfs = Vec::new(); + codec.decode(&encoded, &mut docs, &mut tfs).unwrap(); + + assert_eq!(docs.len(), 0); + assert_eq!(tfs.len(), 0); + } + + #[test] + fn test_delta_varint_round_trip_single() { + let codec = DeltaVarintCodec; + let postings = [(SegmentLocalDocId::new(100), TermFreq::new(5))]; + + let encoded = codec.encode(&postings); + + let mut docs = Vec::new(); + let mut tfs = Vec::new(); + codec.decode(&encoded, &mut docs, &mut tfs).unwrap(); + + assert_eq!(docs, vec![SegmentLocalDocId::new(100)]); + assert_eq!(tfs, vec![TermFreq::new(5)]); + } + + #[test] + fn test_delta_varint_round_trip_multiple() { + let codec = DeltaVarintCodec; + let postings = [ + (SegmentLocalDocId::new(10), TermFreq::new(1)), + (SegmentLocalDocId::new(20), TermFreq::new(2)), + (SegmentLocalDocId::new(35), TermFreq::new(3)), + (SegmentLocalDocId::new(100), TermFreq::new(10)), + ]; + + let encoded = codec.encode(&postings); + + let mut docs = Vec::new(); + let mut tfs = Vec::new(); + codec.decode(&encoded, &mut docs, &mut tfs).unwrap(); + + assert_eq!( + docs, + vec![ + SegmentLocalDocId::new(10), + SegmentLocalDocId::new(20), + SegmentLocalDocId::new(35), + SegmentLocalDocId::new(100), + ] + ); + assert_eq!( + tfs, + vec![ + TermFreq::new(1), + TermFreq::new(2), + TermFreq::new(3), + TermFreq::new(10), + ] + ); + } + + #[test] + fn test_delta_varint_compression() { + let codec = DeltaVarintCodec; + let postings = [ + (SegmentLocalDocId::new(100), TermFreq::new(5)), + (SegmentLocalDocId::new(105), TermFreq::new(3)), + (SegmentLocalDocId::new(110), TermFreq::new(7)), + (SegmentLocalDocId::new(200), TermFreq::new(2)), + ]; + + let encoded = codec.encode(&postings); + let uncompressed = postings.len() * 8; // 4 bytes doc + 4 bytes tf. + + // Encoded should be smaller than uncompressed. + // With deltas and varints, this should be significantly smaller. + assert!( + encoded.len() < uncompressed, + "encoded {} >= uncompressed {}", + encoded.len(), + uncompressed + ); + } + + #[test] + fn test_delta_varint_bad_marker() { + let codec = DeltaVarintCodec; + let bad_bytes = [5_u8, 100, 5]; // Bad marker. + + let mut docs = Vec::new(); + let mut tfs = Vec::new(); + let result = codec.decode(&bad_bytes, &mut docs, &mut tfs); + + assert_eq!(result, Err(CodecError::BadMarker(5))); + } + + // ===== BlockDelta Codec Tests ===== + + #[test] + fn test_block_delta_round_trip_empty() { + let codec = BlockDeltaCodec; + let postings: &[(SegmentLocalDocId, TermFreq)] = &[]; + + let encoded = codec.encode(postings); + assert_eq!(encoded.len(), 1); // Only marker. + + let mut docs = Vec::new(); + let mut tfs = Vec::new(); + codec.decode(&encoded, &mut docs, &mut tfs).unwrap(); + + assert_eq!(docs.len(), 0); + assert_eq!(tfs.len(), 0); + } + + #[test] + fn test_block_delta_round_trip_single_block() { + let codec = BlockDeltaCodec; + let postings: Vec<(SegmentLocalDocId, TermFreq)> = (0..50) + .map(|i| (SegmentLocalDocId::new(i * 10), TermFreq::new(i % 10 + 1))) + .collect(); + + let encoded = codec.encode(&postings); + + let mut docs = Vec::new(); + let mut tfs = Vec::new(); + codec.decode(&encoded, &mut docs, &mut tfs).unwrap(); + + let expected_docs: Vec = postings.iter().map(|(d, _)| *d).collect(); + let expected_tfs: Vec = postings.iter().map(|(_, t)| *t).collect(); + + assert_eq!(docs, expected_docs); + assert_eq!(tfs, expected_tfs); + } + + #[test] + fn test_block_delta_round_trip_multiple_blocks() { + let codec = BlockDeltaCodec; + let postings: Vec<(SegmentLocalDocId, TermFreq)> = (0..300) + .map(|i| (SegmentLocalDocId::new(i * 5), TermFreq::new(i % 7 + 1))) + .collect(); + + let encoded = codec.encode(&postings); + + let mut docs = Vec::new(); + let mut tfs = Vec::new(); + codec.decode(&encoded, &mut docs, &mut tfs).unwrap(); + + let expected_docs: Vec = postings.iter().map(|(d, _)| *d).collect(); + let expected_tfs: Vec = postings.iter().map(|(_, t)| *t).collect(); + + assert_eq!(docs, expected_docs); + assert_eq!(tfs, expected_tfs); + } + + #[test] + fn test_block_delta_compression() { + let codec = BlockDeltaCodec; + let postings: Vec<(SegmentLocalDocId, TermFreq)> = (0..200) + .map(|i| (SegmentLocalDocId::new(i * 3), TermFreq::new(i % 5 + 1))) + .collect(); + + let encoded = codec.encode(&postings); + let uncompressed = postings.len() * 8; + + assert!( + encoded.len() < uncompressed, + "encoded {} >= uncompressed {}", + encoded.len(), + uncompressed + ); + } + + #[test] + fn test_block_delta_independent_decode() { + let codec = BlockDeltaCodec; + // Create a large postings list (> BLOCK_DOC_COUNT). + let postings: Vec<(SegmentLocalDocId, TermFreq)> = (0..300) + .map(|i| (SegmentLocalDocId::new(i * 2), TermFreq::new(i % 3 + 1))) + .collect(); + + let encoded = codec.encode(&postings); + + // Verify full decode. + let mut all_docs = Vec::new(); + let mut all_tfs = Vec::new(); + codec.decode(&encoded, &mut all_docs, &mut all_tfs).unwrap(); + + assert_eq!(all_docs.len(), 300); + assert_eq!(all_tfs.len(), 300); + + // Verify that blocks are self-contained by decoding and checking structure. + // (A full independent block decode would require a separate `decode_block` method, + // which is deferred but noted in AC-2 proof; for now we verify the format is correct + // by full decode and structural checks.) + let expected_docs: Vec = postings.iter().map(|(d, _)| *d).collect(); + let expected_tfs: Vec = postings.iter().map(|(_, t)| *t).collect(); + + assert_eq!(all_docs, expected_docs); + assert_eq!(all_tfs, expected_tfs); + } + + #[test] + fn test_block_delta_bad_marker() { + let codec = BlockDeltaCodec; + let bad_bytes = [99_u8]; // Bad marker. + + let mut docs = Vec::new(); + let mut tfs = Vec::new(); + let result = codec.decode(&bad_bytes, &mut docs, &mut tfs); + + assert_eq!(result, Err(CodecError::BadMarker(99))); + } + + // ===== decode_any Tests ===== + + #[test] + fn test_decode_any_delta_varint() { + let codec = DeltaVarintCodec; + let postings = [ + (SegmentLocalDocId::new(10), TermFreq::new(1)), + (SegmentLocalDocId::new(20), TermFreq::new(2)), + ]; + let encoded = codec.encode(&postings); + + let mut docs = Vec::new(); + let mut tfs = Vec::new(); + decode_any(&encoded, &mut docs, &mut tfs).unwrap(); + + assert_eq!( + docs, + vec![SegmentLocalDocId::new(10), SegmentLocalDocId::new(20)] + ); + assert_eq!(tfs, vec![TermFreq::new(1), TermFreq::new(2)]); + } + + #[test] + fn test_decode_any_block_delta() { + let codec = BlockDeltaCodec; + let postings = [ + (SegmentLocalDocId::new(10), TermFreq::new(1)), + (SegmentLocalDocId::new(20), TermFreq::new(2)), + ]; + let encoded = codec.encode(&postings); + + let mut docs = Vec::new(); + let mut tfs = Vec::new(); + decode_any(&encoded, &mut docs, &mut tfs).unwrap(); + + assert_eq!( + docs, + vec![SegmentLocalDocId::new(10), SegmentLocalDocId::new(20)] + ); + assert_eq!(tfs, vec![TermFreq::new(1), TermFreq::new(2)]); + } + + #[test] + fn test_decode_any_bad_marker() { + let bad_bytes = [99_u8]; + + let mut docs = Vec::new(); + let mut tfs = Vec::new(); + let result = decode_any(&bad_bytes, &mut docs, &mut tfs); + + assert_eq!(result, Err(CodecError::BadMarker(99))); + } + + // ===== Integration: Size Reduction Tests ===== + + #[test] + fn test_delta_varint_size_reduction_zipfian() { + // Simulate a Zipfian-like distribution (long list with concentrated doc IDs). + let codec = DeltaVarintCodec; + let postings: Vec<(SegmentLocalDocId, TermFreq)> = (0..100) + .map(|i| { + let doc_id = (i * 50) as u32; // Sparse docs, large deltas within blocks. + let tf = (1 + (i % 10)) as u32; + (SegmentLocalDocId::new(doc_id), TermFreq::new(tf)) + }) + .collect(); + + let encoded = codec.encode(&postings); + let uncompressed_bytes = postings.len() * 8; + + assert!(encoded.len() < uncompressed_bytes); + } + + #[test] + fn test_block_delta_size_reduction_zipfian() { + let codec = BlockDeltaCodec; + let postings: Vec<(SegmentLocalDocId, TermFreq)> = (0..200) + .map(|i| { + let doc_id = (i * 50) as u32; + let tf = (1 + (i % 10)) as u32; + (SegmentLocalDocId::new(doc_id), TermFreq::new(tf)) + }) + .collect(); + + let encoded = codec.encode(&postings); + let uncompressed_bytes = postings.len() * 8; + + assert!(encoded.len() < uncompressed_bytes); + } + + #[test] + fn test_block_delta_exactly_128_docs() { + let codec = BlockDeltaCodec; + let postings: Vec<(SegmentLocalDocId, TermFreq)> = (0..128) + .map(|i| { + ( + SegmentLocalDocId::new(i as u32), + TermFreq::new((i % 7 + 1) as u32), + ) + }) + .collect(); + + let encoded = codec.encode(&postings); + let mut docs = Vec::new(); + let mut tfs = Vec::new(); + codec.decode(&encoded, &mut docs, &mut tfs).unwrap(); + + assert_eq!(docs.len(), 128, "Should have exactly 128 docs"); + let expected_docs: Vec = + (0..128).map(|i| SegmentLocalDocId::new(i as u32)).collect(); + assert_eq!(docs, expected_docs); + } + + #[test] + fn test_block_delta_129_docs() { + let codec = BlockDeltaCodec; + let postings: Vec<(SegmentLocalDocId, TermFreq)> = (0..129) + .map(|i| { + ( + SegmentLocalDocId::new(i as u32), + TermFreq::new((i % 7 + 1) as u32), + ) + }) + .collect(); + + let encoded = codec.encode(&postings); + let mut docs = Vec::new(); + let mut tfs = Vec::new(); + codec.decode(&encoded, &mut docs, &mut tfs).unwrap(); + + assert_eq!(docs.len(), 129, "Should have 129 docs in 2 blocks"); + let expected_docs: Vec = + (0..129).map(|i| SegmentLocalDocId::new(i as u32)).collect(); + assert_eq!(docs, expected_docs); + } + + #[test] + fn test_delta_varint_large_tf_values() { + let codec = DeltaVarintCodec; + // Test with TF values that require multi-byte varints (> 127) + let postings = [ + (SegmentLocalDocId::new(10), TermFreq::new(200)), + (SegmentLocalDocId::new(20), TermFreq::new(300)), + (SegmentLocalDocId::new(35), TermFreq::new(16384)), + (SegmentLocalDocId::new(100), TermFreq::new(32768)), + ]; + + let encoded = codec.encode(&postings); + let mut docs = Vec::new(); + let mut tfs = Vec::new(); + codec.decode(&encoded, &mut docs, &mut tfs).unwrap(); + + assert_eq!( + docs, + vec![ + SegmentLocalDocId::new(10), + SegmentLocalDocId::new(20), + SegmentLocalDocId::new(35), + SegmentLocalDocId::new(100), + ] + ); + assert_eq!( + tfs, + vec![ + TermFreq::new(200), + TermFreq::new(300), + TermFreq::new(16384), + TermFreq::new(32768), + ] + ); + } + + #[test] + fn test_block_delta_large_doc_gaps() { + let codec = BlockDeltaCodec; + // Simulate Zipfian: wide doc ID gaps within blocks + let postings: Vec<(SegmentLocalDocId, TermFreq)> = (0..100) + .map(|i| { + ( + SegmentLocalDocId::new((i as u32) * 10000), + TermFreq::new((i % 7 + 1) as u32), + ) + }) + .collect(); + + let encoded = codec.encode(&postings); + let mut docs = Vec::new(); + let mut tfs = Vec::new(); + codec.decode(&encoded, &mut docs, &mut tfs).unwrap(); + + assert_eq!(docs.len(), 100); + let expected_docs: Vec = (0..100) + .map(|i| SegmentLocalDocId::new((i as u32) * 10000)) + .collect(); + assert_eq!(docs, expected_docs); + } + + #[test] + fn test_block_delta_block_independence_cross_boundary() { + // Verify that block boundaries don't affect doc reconstruction. + // This tests that the first_doc in each block is stored as an absolute value + // (delta from 0), not as a delta from the previous block. + let codec = BlockDeltaCodec; + + // Create a 256-doc list spanning 2 blocks. + // Block 1: docs 0..127 (even IDs) + // Block 2: docs 128..255 (even IDs) + let postings: Vec<(SegmentLocalDocId, TermFreq)> = (0..256) + .map(|i| (SegmentLocalDocId::new((i as u32) * 2), TermFreq::new(1))) + .collect(); + + let encoded = codec.encode(&postings); + let mut docs = Vec::new(); + let mut tfs = Vec::new(); + codec.decode(&encoded, &mut docs, &mut tfs).unwrap(); + + // Verify all docs were reconstructed correctly across block boundaries. + assert_eq!(docs.len(), 256); + for (i, &doc) in docs.iter().enumerate() { + assert_eq!( + doc.get(), + u32::try_from(i).unwrap() * 2, + "Doc at index {i} mismatch" + ); + } + } + + #[test] + fn test_block_delta_non_unit_deltas() { + // Test block boundaries with varying delta sizes. + // This specifically checks that block-boundary docs have the right doc ID. + let codec = BlockDeltaCodec; + + // Docs: 0, 100, 200, ..., 12700 (128 docs per block) + let postings: Vec<(SegmentLocalDocId, TermFreq)> = (0..256) + .map(|i| { + ( + SegmentLocalDocId::new((i as u32) * 100), + TermFreq::new((i % 5 + 1) as u32), + ) + }) + .collect(); + + let encoded = codec.encode(&postings); + let mut docs = Vec::new(); + let mut tfs = Vec::new(); + codec.decode(&encoded, &mut docs, &mut tfs).unwrap(); + + assert_eq!(docs.len(), 256); + // Verify critical boundary points + assert_eq!(docs[127].get(), 127 * 100, "Last doc of block 1"); + assert_eq!( + docs[128].get(), + 128 * 100, + "First doc of block 2 (critical boundary)" + ); + assert_eq!(docs[255].get(), 255 * 100, "Last doc of block 2"); + } + + #[test] + fn test_varint_over_long_six_bytes() { + // Manually construct a 6-byte varint encoding (invalid). + // After reading the 5th byte, shift=28. On the 6th byte, shift becomes 35 >= 32. + let codec = DeltaVarintCodec; + let mut bad_bytes = Vec::new(); + bad_bytes.push(CodecId::DeltaVarint.to_u8()); + // Encode a valid posting first (doc_id=10, tf=1) + bad_bytes.extend_from_slice(&[10_u8, 1_u8]); + // Append over-long varint: all bytes with MSB set + bad_bytes.extend_from_slice(&[0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0x01_u8]); + + let mut docs = Vec::new(); + let mut tfs = Vec::new(); + let result = codec.decode(&bad_bytes, &mut docs, &mut tfs); + + // Must reject over-long varint, not silently accept + assert_eq!(result, Err(CodecError::InvalidVarint)); + } + + #[test] + fn test_delta_varint_malformed_truncated_tf() { + // Encode: doc_delta(10) + incomplete TF (0x80 without continuation). + let codec = DeltaVarintCodec; + let mut bad_bytes = Vec::new(); + bad_bytes.push(CodecId::DeltaVarint.to_u8()); + // First posting: varint(10) for delta, then 0x80 (MSB set, more bytes expected) + bad_bytes.extend_from_slice(&[10_u8, 0x80_u8]); + + let mut docs = Vec::new(); + let mut tfs = Vec::new(); + let result = codec.decode(&bad_bytes, &mut docs, &mut tfs); + + // Should detect truncation, not crash + assert_eq!(result, Err(CodecError::Truncated)); + } + + #[test] + fn test_block_delta_doc_bytes_len_bounds_check() { + // Create a block header claiming more doc bytes than available. + let codec = BlockDeltaCodec; + let mut bad_bytes = Vec::new(); + bad_bytes.push(CodecId::BlockDelta.to_u8()); + // Block header: doc_count=2, first=10, last=20, doc_bytes_len=100 (but only 2 bytes follow) + bad_bytes.extend_from_slice(&[2_u8, 10_u8, 20_u8, 100_u8]); + bad_bytes.extend_from_slice(&[10_u8, 1_u8]); // Only 2 bytes of doc stream + + let mut docs = Vec::new(); + let mut tfs = Vec::new(); + let result = codec.decode(&bad_bytes, &mut docs, &mut tfs); + + // Must reject, not read past end + assert_eq!(result, Err(CodecError::Truncated)); + } + + #[test] + #[should_panic(expected = "doc-sorted")] + fn test_delta_varint_unsorted_input_panics() { + // The doc-sorted precondition is enforced: an out-of-order doc id panics + // deterministically instead of silently producing a corrupt (wrapped) delta. + let codec = DeltaVarintCodec; + let _ = codec.encode(&[ + (SegmentLocalDocId::new(10), TermFreq::new(1)), + (SegmentLocalDocId::new(5), TermFreq::new(1)), + ]); + } + + #[test] + #[should_panic(expected = "doc-sorted")] + fn test_block_delta_unsorted_input_panics() { + let codec = BlockDeltaCodec; + let _ = codec.encode(&[ + (SegmentLocalDocId::new(10), TermFreq::new(1)), + (SegmentLocalDocId::new(5), TermFreq::new(1)), + ]); + } + + #[test] + fn test_block_delta_corrupt_header_doc_range_rejected() { + // A block header whose first_doc/last_doc disagree with the doc stream is + // rejected (the header range is validated, not ignored). + let codec = BlockDeltaCodec; + let encoded = codec.encode(&[ + (SegmentLocalDocId::new(3), TermFreq::new(1)), + (SegmentLocalDocId::new(7), TermFreq::new(2)), + (SegmentLocalDocId::new(11), TermFreq::new(3)), + ]); + + // Header layout after the 1-byte marker: varint(doc_count) varint(first_doc) ... + // doc_count=3 at index 1, first_doc=3 at index 2. Corrupt first_doc 3 -> 4. + let mut corrupt = encoded.clone(); + assert_eq!(corrupt[2], 3, "expected first_doc varint at index 2"); + corrupt[2] = 4; + + let mut docs = Vec::new(); + let mut tfs = Vec::new(); + assert_eq!( + codec.decode(&corrupt, &mut docs, &mut tfs), + Err(CodecError::BlockHeaderMismatch) + ); + } +} diff --git a/crates/leit_postings/src/lib.rs b/crates/leit_postings/src/lib.rs index fb2b71f..7513046 100644 --- a/crates/leit_postings/src/lib.rs +++ b/crates/leit_postings/src/lib.rs @@ -17,6 +17,8 @@ use alloc::vec::Vec; use leit_core::{EntityId, TermId}; +pub mod codec; + /// A single posting: term occurrence in a document with ``Id``. #[derive(Clone, Debug, PartialEq, Eq)] pub struct Posting { diff --git a/docs/2026-05-30-phase2-architectural-decisions.md b/docs/2026-05-30-phase2-architectural-decisions.md index 00a2e41..6b52519 100644 --- a/docs/2026-05-30-phase2-architectural-decisions.md +++ b/docs/2026-05-30-phase2-architectural-decisions.md @@ -257,3 +257,73 @@ WAND consumes — defining them now *helps* Phase 3. | DEC-08 fully borrowed views | STORY-0083 | ITER-0004 | view accessors | | DEC-09 strict builder/reader | STORY-0084 | ITER-0004 | type boundary | | DEC-10 versioning/checks | STORY-0047 | ITER-0004 | version-rejection | +| DEC-11 block boundary strategy | STORY-0087 | ITER-0002 (codec) / ITER-0004 (writer) | block codec conformance | +| DEC-12 v1 postings/block layout | STORY-0088 | ITER-0002 | SCENARIO-0006 | + +--- + +## DEC-11 — Postings block boundary: fixed document-count blocks (STORY-0087) — RESOLVED + +**Decision:** v1 postings blocks are bounded by a **fixed document count** of **128 documents +per block** (the last block may be short). Boundaries are by doc-count, **not** fixed byte size +and **not** an adaptive/merge heuristic. + +**Rationale (measured against ITER-0000 evidence):** +- The ITER-0000 wind-tunnel corpus is Zipfian (a few very long postings lists, a long tail of + short ones). A **fixed byte-size** block would split a list at unpredictable doc positions, + forcing partial-integer state across block boundaries and making per-block doc-range/skip + metadata (the Phase 3 WAND constraint recorded for ITER-0005) awkward to compute. A fixed + **doc-count** block gives every block a clean `[first_doc, last_doc]` range and a known item + count, which is exactly what skip-lists and block-max metadata need. +- 128 docs/block is the conventional Lucene/PForDelta block size and matches the decode-batch + granularity that keeps the decode scratch small and cache-resident. It is large enough to + amortize per-block header cost over the long lists that dominate query latency in the baseline + (`single_term` ≈ 136 µs/1k), small enough that selective block decode (the deferred ITER-0003 + `advance_to` skip) skips meaningful work. +- Short lists (< 128 docs — the Zipfian tail) occupy a single block; no padding, no waste. + +**Consequence / forward-compat:** Each block carries its document count and first/last doc in its +header, so ITER-0005's block-metadata sidecar can reference blocks by ordinal and attach +`max_score` + doc-range without re-deriving boundaries. The boundary constant lives in one place +(`BLOCK_DOC_COUNT`) so a future codec may tune it without a format-break (the count is encoded +per block, not assumed by readers). + +## DEC-12 — v1 postings/block layout for compressed traversal (STORY-0088) — RESOLVED + +**Decision:** A postings list serializes as a sequence of independently-decodable **blocks**. +Each block is laid out as: + +``` +block := block_header doc_id_stream tf_stream +block_header := varint(doc_count) varint(first_doc) varint(last_doc) varint(doc_bytes_len) +doc_id_stream := varint(first_delta) varint(delta)* # deltas from previous doc id +tf_stream := varint(tf)* # one per doc, parallel to doc stream +``` + +- **Doc IDs** are **delta-encoded** then **LEB128 varint**-encoded (deltas are non-negative + because postings are doc-sorted). The first doc in a block is stored as a delta from 0 (i.e. its + absolute value), so each block is self-contained and decodable without the previous block + (DEC-11 enables this). +- **Term frequencies** are LEB128 varint-encoded, one per doc, in a stream parallel to the doc-id + stream — so a doc-only cursor can decode the doc stream and skip the TF stream using + `doc_bytes_len`. +- **`doc_bytes_len`** in the header lets a reader locate the TF stream (and the next block) without + decoding the doc stream — the basis for the deferred ITER-0003 selective/skip decode. +- **Codec marker:** a postings list is prefixed by a 1-byte `CodecId` (DEC: `0 = DeltaVarint` + single-block, `1 = BlockDelta` doc-count blocks). This is the per-list marker; reserving the + *field in the segment format* is deferred to ITER-0004 (STORY-0002 AC-3). + +**Traversal-semantics contract (uncompressed ↔ compressed API stability — STORY-0088 AC-2 is the +e2e proof, deferred to ITER-0003):** the codec **API speaks named segment-resident types** +`SegmentLocalDocId` and `TermFreq` (the u32 values are delta-encoded and varint-encoded at the +byte level, unchanged). The generic `EntityId` is lowered to `SegmentLocalDocId` at the +segment-write boundary (ITER-0004). Decode produces exactly the `(SegmentLocalDocId, TermFreq)` +sequence, in doc-ascending order, that the in-memory `PostingsList` will hold — byte-for-byte +equal round trip (SCENARIO-0006). The codec writes decoded values into **caller-provided output +buffers** (`&mut Vec` and `&mut Vec`); it does **not** own a +decode-scratch type. The decode-scratch ownership wrapper and the cursor adaptor over this API +are decided and built in ITER-0003 (STORY-0079) — the codec layer is deliberately +scratch-ownership-agnostic so that decision is not boxed in here. + +**Decided in ITER-0002 against the ITER-0000 wind-tunnel baseline; enforced by the codec +implementation (SCENARIO-0006 round-trip) and measured by SCENARIO-0070 (codec comparison).** From 615f05b76e1d110b14757338f774440a67b85c63 Mon Sep 17 00:00:00 2001 From: "Norman Nunley, Jr" Date: Wed, 15 Jul 2026 07:49:50 -0400 Subject: [PATCH 7/9] fix: make codec decode errors transactional --- Cargo.lock | 1 - crates/leit_postings/src/codec.rs | 280 ++++++++++++++++++++---------- 2 files changed, 189 insertions(+), 92 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 977a982..dde280a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -663,7 +663,6 @@ dependencies = [ "criterion", "leit_core", "leit_index", - "leit_postings", "leit_text", "leit_wind_tunnel", ] diff --git a/crates/leit_postings/src/codec.rs b/crates/leit_postings/src/codec.rs index 1f6b9af..15c1a19 100644 --- a/crates/leit_postings/src/codec.rs +++ b/crates/leit_postings/src/codec.rs @@ -102,7 +102,8 @@ impl core::error::Error for CodecError {} /// and `&mut Vec`) rather than allocating owned decode structures. This design keeps /// the codec layer scratch-ownership-agnostic; see DEC-12 and the TODO comment below. /// -/// The caller is responsible for clearing or reusing these buffers. +/// The caller may reuse these buffers. Decoders clear them before decoding and +/// leave both buffers empty if decoding fails. /// /// ## Codec marker /// @@ -148,7 +149,8 @@ pub trait Codec { /// /// # Returns /// - /// `Ok(())` on success, or a `CodecError` if decoding fails. + /// `Ok(())` on success, or a `CodecError` if decoding fails. On error, both + /// output buffers are empty. /// /// # TODO(ITER-0003): `DecodeScratch` wrapper /// @@ -202,6 +204,10 @@ fn decode_varint(bytes: &[u8]) -> Result<(u32, usize), CodecError> { let byte = bytes[pos]; pos += 1; + if shift == 28 && (byte & 0x7f) > 0x0f { + return Err(CodecError::InvalidVarint); + } + value |= ((byte & 0x7f) as u32) << shift; if (byte & 0x80) == 0 { @@ -266,38 +272,46 @@ impl Codec for DeltaVarintCodec { out_docs.clear(); out_tfs.clear(); - if bytes.is_empty() { - return Err(CodecError::Truncated); - } + let result = (|| { + if bytes.is_empty() { + return Err(CodecError::Truncated); + } - let marker = bytes[0]; - if marker != CodecId::DeltaVarint.to_u8() { - return Err(CodecError::BadMarker(marker)); - } + let marker = bytes[0]; + if marker != CodecId::DeltaVarint.to_u8() { + return Err(CodecError::BadMarker(marker)); + } - let mut pos = 1; - let mut prev_doc = 0_u32; + let mut pos = 1; + let mut prev_doc = 0_u32; - while pos < bytes.len() { - let (delta, delta_len) = decode_varint(&bytes[pos..])?; - pos += delta_len; + while pos < bytes.len() { + let (delta, delta_len) = decode_varint(&bytes[pos..])?; + pos += delta_len; - if pos >= bytes.len() { - return Err(CodecError::Truncated); + if pos >= bytes.len() { + return Err(CodecError::Truncated); + } + + let doc_id = prev_doc + .checked_add(delta) + .ok_or(CodecError::InvalidVarint)?; + let (tf, tf_len) = decode_varint(&bytes[pos..])?; + pos += tf_len; + + out_docs.push(SegmentLocalDocId::new(doc_id)); + out_tfs.push(TermFreq::new(tf)); + prev_doc = doc_id; } - let doc_id = prev_doc - .checked_add(delta) - .ok_or(CodecError::InvalidVarint)?; - let (tf, tf_len) = decode_varint(&bytes[pos..])?; - pos += tf_len; + Ok(()) + })(); - out_docs.push(SegmentLocalDocId::new(doc_id)); - out_tfs.push(TermFreq::new(tf)); - prev_doc = doc_id; + if result.is_err() { + out_docs.clear(); + out_tfs.clear(); } - - Ok(()) + result } } @@ -404,86 +418,98 @@ impl Codec for BlockDeltaCodec { out_docs.clear(); out_tfs.clear(); - if bytes.is_empty() { - return Err(CodecError::Truncated); - } - - let marker = bytes[0]; - if marker != CodecId::BlockDelta.to_u8() { - return Err(CodecError::BadMarker(marker)); - } - - let mut pos = 1; - - while pos < bytes.len() { - // Decode block header. - let (doc_count, bytes_read) = decode_varint(&bytes[pos..])?; - pos += bytes_read; - let doc_count = doc_count as usize; - - if doc_count == 0 { - return Err(CodecError::InvalidBlockCount); + let result = (|| { + if bytes.is_empty() { + return Err(CodecError::Truncated); } - let (first_doc, bytes_read) = decode_varint(&bytes[pos..])?; - pos += bytes_read; + let marker = bytes[0]; + if marker != CodecId::BlockDelta.to_u8() { + return Err(CodecError::BadMarker(marker)); + } - let (last_doc, bytes_read) = decode_varint(&bytes[pos..])?; - pos += bytes_read; + let mut pos = 1; - let (doc_bytes_len, bytes_read) = decode_varint(&bytes[pos..])?; - pos += bytes_read; - let doc_bytes_len = doc_bytes_len as usize; + while pos < bytes.len() { + // Decode block header. + let (doc_count, bytes_read) = decode_varint(&bytes[pos..])?; + pos += bytes_read; + let doc_count = doc_count as usize; - if pos + doc_bytes_len > bytes.len() { - return Err(CodecError::Truncated); - } + if doc_count == 0 || doc_count > BLOCK_DOC_COUNT { + return Err(CodecError::InvalidBlockCount); + } - // Decode doc ID stream. - let doc_stream_start = pos; - let doc_stream_end = pos + doc_bytes_len; - let doc_stream = &bytes[doc_stream_start..doc_stream_end]; + let (first_doc, bytes_read) = decode_varint(&bytes[pos..])?; + pos += bytes_read; - let mut doc_pos = 0; - let mut prev_doc = 0_u32; - let block_doc_start = out_docs.len(); + let (last_doc, bytes_read) = decode_varint(&bytes[pos..])?; + pos += bytes_read; - for _ in 0..doc_count { - if doc_pos >= doc_stream.len() { - return Err(CodecError::Truncated); + let (doc_bytes_len, bytes_read) = decode_varint(&bytes[pos..])?; + pos += bytes_read; + let doc_bytes_len = doc_bytes_len as usize; + + let doc_stream_end = pos + .checked_add(doc_bytes_len) + .filter(|end| *end <= bytes.len()) + .ok_or(CodecError::Truncated)?; + + // Decode doc ID stream. + let doc_stream_start = pos; + let doc_stream = &bytes[doc_stream_start..doc_stream_end]; + + let mut doc_pos = 0; + let mut prev_doc = 0_u32; + let block_doc_start = out_docs.len(); + + for _ in 0..doc_count { + if doc_pos >= doc_stream.len() { + return Err(CodecError::Truncated); + } + let (delta, bytes_read) = decode_varint(&doc_stream[doc_pos..])?; + doc_pos += bytes_read; + + let doc_id = prev_doc + .checked_add(delta) + .ok_or(CodecError::InvalidVarint)?; + out_docs.push(SegmentLocalDocId::new(doc_id)); + prev_doc = doc_id; } - let (delta, bytes_read) = decode_varint(&doc_stream[doc_pos..])?; - doc_pos += bytes_read; - let doc_id = prev_doc - .checked_add(delta) - .ok_or(CodecError::InvalidVarint)?; - out_docs.push(SegmentLocalDocId::new(doc_id)); - prev_doc = doc_id; - } + if doc_pos != doc_stream.len() { + return Err(CodecError::InvalidVarint); + } - // Validate the block header's doc-range against the decoded stream. - // This makes `first_doc`/`last_doc` (carried for ITER-0003 block-skip and - // ITER-0005 WAND doc-range) self-checking rather than dead bytes, and - // detects corruption. `prev_doc` now holds the block's last decoded doc. - if out_docs[block_doc_start].get() != first_doc || prev_doc != last_doc { - return Err(CodecError::BlockHeaderMismatch); - } + // Validate the block header's doc-range against the decoded stream. + // This makes `first_doc`/`last_doc` (carried for ITER-0003 block-skip and + // ITER-0005 WAND doc-range) self-checking rather than dead bytes, and + // detects corruption. `prev_doc` now holds the block's last decoded doc. + if out_docs[block_doc_start].get() != first_doc || prev_doc != last_doc { + return Err(CodecError::BlockHeaderMismatch); + } - pos = doc_stream_end; + pos = doc_stream_end; - // Decode TF stream (parallel to doc stream). - for _ in 0..doc_count { - if pos >= bytes.len() { - return Err(CodecError::Truncated); + // Decode TF stream (parallel to doc stream). + for _ in 0..doc_count { + if pos >= bytes.len() { + return Err(CodecError::Truncated); + } + let (tf, bytes_read) = decode_varint(&bytes[pos..])?; + pos += bytes_read; + out_tfs.push(TermFreq::new(tf)); } - let (tf, bytes_read) = decode_varint(&bytes[pos..])?; - pos += bytes_read; - out_tfs.push(TermFreq::new(tf)); } - } - Ok(()) + Ok(()) + })(); + + if result.is_err() { + out_docs.clear(); + out_tfs.clear(); + } + result } } @@ -500,12 +526,16 @@ impl Codec for BlockDeltaCodec { /// /// # Returns /// -/// `Ok(())` on success, or a `CodecError` on failure. +/// `Ok(())` on success, or a `CodecError` on failure. On error, both output +/// buffers are empty. pub fn decode_any( bytes: &[u8], out_docs: &mut Vec, out_tfs: &mut Vec, ) -> Result<(), CodecError> { + out_docs.clear(); + out_tfs.clear(); + if bytes.is_empty() { return Err(CodecError::Truncated); } @@ -578,6 +608,12 @@ mod tests { assert_eq!(result, Err(CodecError::Truncated)); } + #[test] + fn test_varint_decode_rejects_five_byte_overflow() { + let bytes = [0x80, 0x80, 0x80, 0x80, 0x10]; + assert_eq!(decode_varint(&bytes), Err(CodecError::InvalidVarint)); + } + // ===== DeltaVarint Codec Tests ===== #[test] @@ -647,6 +683,34 @@ mod tests { ); } + #[test] + fn test_delta_varint_error_clears_partial_output() { + let codec = DeltaVarintCodec; + let bytes = [CodecId::DeltaVarint.to_u8(), 1, 1, 0x80]; + let mut docs = vec![SegmentLocalDocId::new(99)]; + let mut tfs = vec![TermFreq::new(99)]; + + assert_eq!( + codec.decode(&bytes, &mut docs, &mut tfs), + Err(CodecError::Truncated) + ); + assert!(docs.is_empty()); + assert!(tfs.is_empty()); + } + + #[test] + fn test_decode_any_bad_marker_clears_output() { + let mut docs = vec![SegmentLocalDocId::new(99)]; + let mut tfs = vec![TermFreq::new(99)]; + + assert_eq!( + decode_any(&[u8::MAX], &mut docs, &mut tfs), + Err(CodecError::BadMarker(u8::MAX)) + ); + assert!(docs.is_empty()); + assert!(tfs.is_empty()); + } + #[test] fn test_delta_varint_compression() { let codec = DeltaVarintCodec; @@ -1158,5 +1222,39 @@ mod tests { codec.decode(&corrupt, &mut docs, &mut tfs), Err(CodecError::BlockHeaderMismatch) ); + assert!(docs.is_empty()); + assert!(tfs.is_empty()); + } + + #[test] + fn test_block_delta_rejects_count_above_block_size() { + let codec = BlockDeltaCodec; + let bytes = [CodecId::BlockDelta.to_u8(), 129, 1, 1, 1, 1, 1]; + let mut docs = Vec::new(); + let mut tfs = Vec::new(); + + assert_eq!( + codec.decode(&bytes, &mut docs, &mut tfs), + Err(CodecError::InvalidBlockCount) + ); + } + + #[test] + fn test_block_delta_rejects_unconsumed_doc_stream_bytes() { + let codec = BlockDeltaCodec; + let encoded = codec.encode(&[ + (SegmentLocalDocId::new(10), TermFreq::new(2)), + (SegmentLocalDocId::new(20), TermFreq::new(3)), + ]); + let mut corrupt = encoded.clone(); + corrupt[4] = 3; + corrupt.insert(8, 0); + let mut docs = Vec::new(); + let mut tfs = Vec::new(); + + assert_eq!( + codec.decode(&corrupt, &mut docs, &mut tfs), + Err(CodecError::InvalidVarint) + ); } } From d20bdca72b72c67f2a78be7acacf3a92c69df956 Mon Sep 17 00:00:00 2001 From: "Norman Nunley, Jr" Date: Fri, 29 May 2026 22:43:49 -0400 Subject: [PATCH 8/9] feat(wind-tunnel): codec comparison benchmark + tradeoff guidance [ITER-0002 T6-T7] MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit SCENARIO-0070 (process-level): Criterion benchmark comparing DeltaVarint vs BlockDelta over the deterministic wind-tunnel corpus (1K/10K, multi-field, Zipfian). Measures encode time, decode time, and compressed size vs the 8-byte/posting baseline, with a lossless sanity gate. Baseline: DeltaVarint ~25%, BlockDelta ~26-27% of uncompressed; DeltaVarint decode ~4-11% faster. - crates/leit_wind_tunnel_index/benches/codec_compare.rs (+ [[bench]], leit_postings/ leit_core dev-deps). Criterion stays out of all primary crates (SCENARIO-0061/0069 pass). - leit_index: PostingEntry made public + InMemoryIndex::postings_by_term() accessor, the minimal surface needed to extract doc-sorted (SegmentLocalDocId, TermFreq) postings (lowering stands in for the ITER-0004 segment-write boundary). - docs/2026-05-30-codec-tradeoffs.md — STORY-0006 AC-3 decode-cost vs memory guidance. Stories: STORY-0006 (benchmark + guidance). Proof: SCENARIO-0070. PAR reviewed. Co-Authored-By: Claude Opus 4.8 (1M context) --- crates/leit_index/src/lib.rs | 2 +- crates/leit_index/src/memory.rs | 20 +- crates/leit_wind_tunnel_index/Cargo.toml | 5 + .../benches/codec_compare.rs | 300 ++++++++++++++++++ docs/2026-05-30-codec-tradeoffs.md | 50 +++ 5 files changed, 373 insertions(+), 4 deletions(-) create mode 100644 crates/leit_wind_tunnel_index/benches/codec_compare.rs create mode 100644 docs/2026-05-30-codec-tradeoffs.md diff --git a/crates/leit_index/src/lib.rs b/crates/leit_index/src/lib.rs index c1736d0..b0999b1 100644 --- a/crates/leit_index/src/lib.rs +++ b/crates/leit_index/src/lib.rs @@ -30,6 +30,6 @@ mod segment; pub use builder::{InMemoryIndexBuilder, IndexBuilder}; pub use error::{IndexError, SegmentError}; pub use leit_core::{FilterEvaluator, FilterSlotId, NoFilter}; -pub use memory::InMemoryIndex; +pub use memory::{InMemoryIndex, PostingEntry}; pub use search::{ExecutionStats, ExecutionWorkspace, SearchScorer}; pub use segment::{SectionKind, SegmentView}; diff --git a/crates/leit_index/src/memory.rs b/crates/leit_index/src/memory.rs index ae442ed..8ff56eb 100644 --- a/crates/leit_index/src/memory.rs +++ b/crates/leit_index/src/memory.rs @@ -24,10 +24,16 @@ pub(crate) struct TermEntry { pub(crate) term: String, } +/// A single posting: a document ID and its term frequency for a term. +/// +/// Postings are aggregated per term and stored in doc-sorted order (ascending doc ID). +/// This type is public primarily for codec benchmarking and analysis. #[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] -pub(crate) struct PostingEntry { - pub(crate) doc_id: u32, - pub(crate) term_freq: u32, +pub struct PostingEntry { + /// Document identifier (segment-local, u32). + pub doc_id: u32, + /// Term frequency (raw count of term occurrences in the document's field). + pub term_freq: u32, } #[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] @@ -135,6 +141,14 @@ impl InMemoryIndex { &self.postings } + /// Return all postings indexed by term ID. + /// + /// Postings are doc-sorted (ascending doc ID) within each term's list. + /// This method is primarily used for codec benchmarking and analysis. + pub fn postings_by_term(&self) -> &BTreeMap> { + &self.postings + } + fn avg_field_doc_length(&self, field: FieldId) -> f32 { let Some(stats) = self.field_stats.get(&field) else { return 0.0; diff --git a/crates/leit_wind_tunnel_index/Cargo.toml b/crates/leit_wind_tunnel_index/Cargo.toml index 99fc83e..2eb0a3e 100644 --- a/crates/leit_wind_tunnel_index/Cargo.toml +++ b/crates/leit_wind_tunnel_index/Cargo.toml @@ -20,6 +20,7 @@ bench = false criterion = { workspace = true } leit_core = { features = ["std"], workspace = true } leit_index = { features = ["std"], workspace = true } +leit_postings = { features = ["std"], workspace = true } leit_text = { features = ["std"], workspace = true } leit_wind_tunnel = { workspace = true } @@ -27,5 +28,9 @@ leit_wind_tunnel = { workspace = true } name = "indexing" harness = false +[[bench]] +name = "codec_compare" +harness = false + [lints] workspace = true diff --git a/crates/leit_wind_tunnel_index/benches/codec_compare.rs b/crates/leit_wind_tunnel_index/benches/codec_compare.rs new file mode 100644 index 0000000..895fe53 --- /dev/null +++ b/crates/leit_wind_tunnel_index/benches/codec_compare.rs @@ -0,0 +1,300 @@ +// Copyright 2026 the Leit Authors +// SPDX-License-Identifier: Apache-2.0 OR MIT + +//! Codec comparison benchmarks: encode/decode latency and compressed size. +//! +//! This benchmark indexes a deterministic wind-tunnel corpus and extracts all postings, +//! then measures and reports: +//! - Encode time per codec +//! - Decode time per codec +//! - Compressed size ratio vs 8-byte uncompressed baseline +//! +//! Run with `cargo bench -p leit_wind_tunnel_index --bench codec_compare`. + +#![expect( + missing_docs, + reason = "criterion_group! generates an undocumented public `benches` fn" +)] + +use criterion::{BenchmarkId, Criterion, criterion_group, criterion_main}; +use leit_core::{FieldId, SegmentLocalDocId, TermFreq}; +use leit_index::InMemoryIndexBuilder; +use leit_postings::codec::{BlockDeltaCodec, Codec, CodecId, DeltaVarintCodec}; +use leit_text::{Analyzer, FieldAnalyzers, UnicodeNormalizer, WhitespaceTokenizer}; +use leit_wind_tunnel::{CorpusGenerator, corpus::GeneratedDoc}; + +/// Fixed seed so every benchmark run indexes byte-identical corpora. +const SEED: u64 = 42; +/// `title` field, matching the corpus generator's field 1. +const TITLE: FieldId = FieldId::new(1); +/// `body` field, matching the corpus generator's field 2. +const BODY: FieldId = FieldId::new(2); + +/// Postings list type: `Vec` of (`SegmentLocalDocId`, `TermFreq`) tuples per term. +type PostingsList = Vec>; + +/// Build the field analyzers used for indexing (whitespace tokenizer + unicode +/// normalizer on both fields), matching the wind-tunnel integration tests. +fn make_analyzers() -> FieldAnalyzers { + let mut analyzers = FieldAnalyzers::new(); + analyzers.set( + TITLE, + Analyzer::new(WhitespaceTokenizer::new()).with_normalizer(UnicodeNormalizer::new()), + ); + analyzers.set( + BODY, + Analyzer::new(WhitespaceTokenizer::new()).with_normalizer(UnicodeNormalizer::new()), + ); + analyzers +} + +/// Index a corpus and extract all postings as `Vec<(doc_id, term_freq)>` tuples. +fn extract_postings(corpus: &[GeneratedDoc]) -> PostingsList { + let mut builder = InMemoryIndexBuilder::new(make_analyzers()); + builder.register_field_alias(TITLE, "title"); + builder.register_field_alias(BODY, "body"); + + for doc in corpus { + builder + .index_document( + doc.id, + &[(TITLE, doc.title.as_str()), (BODY, doc.body.as_str())], + ) + .expect("indexing should succeed"); + } + + let index = builder.build_index(); + + // Extract all postings as (SegmentLocalDocId, TermFreq) tuples. + // Postings are already doc-sorted from the index. + let mut all_postings = Vec::new(); + for posting_list in index.postings_by_term().values() { + let postings: Vec<(SegmentLocalDocId, TermFreq)> = posting_list + .iter() + .map(|p| (SegmentLocalDocId::new(p.doc_id), TermFreq::new(p.term_freq))) + .collect(); + if !postings.is_empty() { + all_postings.push(postings); + } + } + all_postings +} + +/// Measure compression ratio and emit a summary table. +fn report_compression_detailed( + corpus_size: u32, + total_postings: usize, + codec_sizes: &[(&str, usize)], +) { + // Baseline: 8 bytes per posting (u32 doc_id + u32 term_freq). + let uncompressed_baseline = total_postings * 8; + + eprintln!( + "\n=== Codec Compression Summary (corpus: {} docs, {} total postings) ===", + corpus_size, total_postings + ); + eprintln!("Baseline (uncompressed): 8 bytes per posting"); + eprintln!(" Total uncompressed: {} bytes", uncompressed_baseline); + + for (name, bytes) in codec_sizes { + let ratio = if uncompressed_baseline > 0 { + (*bytes as f64) / (uncompressed_baseline as f64) * 100.0 + } else { + 0.0 + }; + let avg_bytes_per_posting = if total_postings > 0 { + *bytes as f64 / total_postings as f64 + } else { + 0.0 + }; + eprintln!( + " {}: {} bytes ({:.1}% of baseline, {:.2} bytes/posting)", + name, bytes, ratio, avg_bytes_per_posting + ); + } +} + +fn bench_codec_encode_decode(c: &mut Criterion) { + let generator = CorpusGenerator::new(SEED); + + // Prepare corpora and postings once outside the benchmark loop. + #[expect( + clippy::useless_vec, + reason = "vec literal is the idiomatic way to construct this list" + )] + let corpora = vec![ + (1_000_u32, generator.generate(1_000)), + (10_000_u32, generator.generate(10_000)), + ]; + + let all_postings: Vec<(u32, PostingsList)> = corpora + .iter() + .map(|(size, corpus)| (*size, extract_postings(corpus))) + .collect(); + + // Report compression sizes upfront. + for (corpus_size, postings_list) in &all_postings { + let mut total_postings = 0; + let mut sizes = Vec::new(); + + // Encode all postings with each codec and measure total size. + let delta_varint_codec = DeltaVarintCodec; + let block_delta_codec = BlockDeltaCodec; + + let mut dv_total_size = 0; + let mut bd_total_size = 0; + + for postings in postings_list { + if !postings.is_empty() { + total_postings += postings.len(); + + let dv_encoded = delta_varint_codec.encode(postings); + dv_total_size += dv_encoded.len(); + + let bd_encoded = block_delta_codec.encode(postings); + bd_total_size += bd_encoded.len(); + } + } + + sizes.push(("DeltaVarint", dv_total_size)); + sizes.push(("BlockDelta", bd_total_size)); + + report_compression_detailed(*corpus_size, total_postings, &sizes); + } + + let mut group = c.benchmark_group("codec_encode"); + for (corpus_size_label, corpus_size, postings_list) in all_postings.iter().map(|(s, p)| { + let label = if *s == 1_000 { "1k" } else { "10k" }; + (label, *s, p) + }) { + let delta_varint_codec = DeltaVarintCodec; + let block_delta_codec = BlockDeltaCodec; + + group.bench_with_input( + BenchmarkId::from_parameter(format!("deltavarint/{}", corpus_size_label)), + &corpus_size, + |b, _| { + b.iter(|| { + let mut total_bytes = 0; + for postings in postings_list { + if !postings.is_empty() { + let encoded = delta_varint_codec.encode(postings); + total_bytes += encoded.len(); + } + } + criterion::black_box(total_bytes); + }); + }, + ); + + group.bench_with_input( + BenchmarkId::from_parameter(format!("blockdelta/{}", corpus_size_label)), + &corpus_size, + |b, _| { + b.iter(|| { + let mut total_bytes = 0; + for postings in postings_list { + if !postings.is_empty() { + let encoded = block_delta_codec.encode(postings); + total_bytes += encoded.len(); + } + } + criterion::black_box(total_bytes); + }); + }, + ); + } + group.finish(); + + let mut group = c.benchmark_group("codec_decode"); + for (corpus_size_label, corpus_size, postings_list) in all_postings.iter().map(|(s, p)| { + let label = if *s == 1_000 { "1k" } else { "10k" }; + (label, *s, p) + }) { + let delta_varint_codec = DeltaVarintCodec; + let block_delta_codec = BlockDeltaCodec; + + // Pre-encode for decode benchmarks. + let dv_encoded: Vec> = postings_list + .iter() + .map(|p| { + if p.is_empty() { + vec![CodecId::DeltaVarint.to_u8()] + } else { + delta_varint_codec.encode(p) + } + }) + .collect(); + + let bd_encoded: Vec> = postings_list + .iter() + .map(|p| { + if p.is_empty() { + vec![CodecId::BlockDelta.to_u8()] + } else { + block_delta_codec.encode(p) + } + }) + .collect(); + + group.bench_with_input( + BenchmarkId::from_parameter(format!("deltavarint/{}", corpus_size_label)), + &corpus_size, + |b, _| { + b.iter(|| { + let mut decoded_count = 0; + for (encoded, original_postings) in dv_encoded.iter().zip(postings_list) { + let mut docs = Vec::new(); + let mut tfs = Vec::new(); + delta_varint_codec + .decode(encoded, &mut docs, &mut tfs) + .expect("decode should succeed"); + + // Sanity gate: verify decode matches original (compare typed values). + assert_eq!(docs.len(), original_postings.len(), "doc count mismatch"); + assert_eq!(tfs.len(), original_postings.len(), "tf count mismatch"); + for (i, (doc_id, tf)) in original_postings.iter().enumerate() { + assert_eq!(docs[i], *doc_id, "doc mismatch at index {i}"); + assert_eq!(tfs[i], *tf, "tf mismatch at index {i}"); + } + + decoded_count += docs.len(); + } + criterion::black_box(decoded_count); + }); + }, + ); + + group.bench_with_input( + BenchmarkId::from_parameter(format!("blockdelta/{}", corpus_size_label)), + &corpus_size, + |b, _| { + b.iter(|| { + let mut decoded_count = 0; + for (encoded, original_postings) in bd_encoded.iter().zip(postings_list) { + let mut docs = Vec::new(); + let mut tfs = Vec::new(); + block_delta_codec + .decode(encoded, &mut docs, &mut tfs) + .expect("decode should succeed"); + + // Sanity gate: verify decode matches original (compare typed values). + assert_eq!(docs.len(), original_postings.len(), "doc count mismatch"); + assert_eq!(tfs.len(), original_postings.len(), "tf count mismatch"); + for (i, (doc_id, tf)) in original_postings.iter().enumerate() { + assert_eq!(docs[i], *doc_id, "doc mismatch at index {i}"); + assert_eq!(tfs[i], *tf, "tf mismatch at index {i}"); + } + + decoded_count += docs.len(); + } + criterion::black_box(decoded_count); + }); + }, + ); + } + group.finish(); +} + +criterion_group!(benches, bench_codec_encode_decode); +criterion_main!(benches); diff --git a/docs/2026-05-30-codec-tradeoffs.md b/docs/2026-05-30-codec-tradeoffs.md new file mode 100644 index 0000000..711fd8e --- /dev/null +++ b/docs/2026-05-30-codec-tradeoffs.md @@ -0,0 +1,50 @@ +# Codec Tradeoff Analysis: DeltaVarint vs BlockDelta + +**Status:** Evidence-based analysis from SCENARIO-0070 codec benchmarks (ITER-0002, STORY-0006 AC-3). + +**Measurement:** `cargo bench -p leit_wind_tunnel_index --bench codec_compare` on the deterministic wind-tunnel corpus (SEED=42, 1K and 10K documents, Zipfian term distribution). + +--- + +## Summary + +Both codecs compress postings lists to ~25–27% of the uncompressed baseline (8 bytes per posting). + +- **DeltaVarint**: single-block, varint-encoded deltas, ~2.03–2.05 bytes/posting +- **BlockDelta**: 128-doc blocks with per-block headers, ~2.10–2.19 bytes/posting + +**Decode latency:** +- 1K corpus: DeltaVarint ~285 µs, BlockDelta ~297 µs (encode/decode times are comparable; BlockDelta slower due to per-block overhead) +- 10K corpus: DeltaVarint ~1.34 ms, BlockDelta ~1.48 ms + +**Encode latency:** +- 1K corpus: DeltaVarint ~188 µs, BlockDelta ~311 µs (~1.65× slower) +- 10K corpus: DeltaVarint ~1.67 ms, BlockDelta ~2.54 ms (~1.52× slower) + +--- + +## Tradeoff Rationale + +### DeltaVarint: Decode speed, simplicity +- **Single stream**: no block metadata to parse, minimal decode latency. +- **Simplest codec**: delta encoding + varints, lowest complexity on the read path. +- **Trade-off**: no block structure means future block-aware features (selective decode, skip, WAND doc-range pruning) require full decode. +- **Encode cost**: low; varint encoding is linear and fast. + +### BlockDelta: Block-aware future evolution +- **128-doc blocks**: each block independently decodable; enables Phase 3 features (selective block skip, WAND pruning with block-level doc ranges). +- **Per-block header overhead**: first_doc, last_doc, doc_bytes_len increase encoded size slightly vs DeltaVarint. +- **Trade-off**: decode is ~4–11% slower due to per-block header parsing; blocks do not improve memory footprint (compression ratio is similar). +- **Encode cost**: higher; block boundaries and per-block headers add work. + +--- + +## Conclusion + +**For Phase 2 (v1)**: DeltaVarint is sufficient and simpler; it achieves the same compression ratio with lower latency. + +**For Phase 3+ (selective decode, block-aware WAND)**: BlockDelta's block structure is necessary to enable those features without full decode. The ~4–11% decode-latency cost is acceptable when the alternative is a format migration. + +The benchmark confirms that **compression efficiency is not the differentiator**—both codecs perform similarly. The decision is **architectural**: DeltaVarint for speed/simplicity in v1, BlockDelta for extensibility in v2+. + +**Current production choice**: DeltaVarint is the default; BlockDelta is implemented and tested in parallel for Phase 3 integration. From 3085a5774e87f870a8ee05daef49cf40537ef3a5 Mon Sep 17 00:00:00 2001 From: "Norman Nunley, Jr" Date: Wed, 15 Jul 2026 07:52:20 -0400 Subject: [PATCH 9/9] fix: isolate codec benchmark measurement plumbing --- Cargo.lock | 1 + crates/leit_index/Cargo.toml | 1 + crates/leit_index/src/lib.rs | 2 +- crates/leit_index/src/memory.rs | 27 ++- crates/leit_wind_tunnel_index/Cargo.toml | 2 +- .../benches/codec_compare.rs | 196 +++++++++--------- docs/2026-05-30-codec-tradeoffs.md | 25 +-- 7 files changed, 128 insertions(+), 126 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index dde280a..977a982 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -663,6 +663,7 @@ dependencies = [ "criterion", "leit_core", "leit_index", + "leit_postings", "leit_text", "leit_wind_tunnel", ] diff --git a/crates/leit_index/Cargo.toml b/crates/leit_index/Cargo.toml index b6a08ed..6234711 100644 --- a/crates/leit_index/Cargo.toml +++ b/crates/leit_index/Cargo.toml @@ -20,6 +20,7 @@ leit_text.workspace = true [features] default = ["std"] +bench-internals = [] std = [ "leit_core/std", "leit_text/std", diff --git a/crates/leit_index/src/lib.rs b/crates/leit_index/src/lib.rs index b0999b1..c1736d0 100644 --- a/crates/leit_index/src/lib.rs +++ b/crates/leit_index/src/lib.rs @@ -30,6 +30,6 @@ mod segment; pub use builder::{InMemoryIndexBuilder, IndexBuilder}; pub use error::{IndexError, SegmentError}; pub use leit_core::{FilterEvaluator, FilterSlotId, NoFilter}; -pub use memory::{InMemoryIndex, PostingEntry}; +pub use memory::InMemoryIndex; pub use search::{ExecutionStats, ExecutionWorkspace, SearchScorer}; pub use segment::{SectionKind, SegmentView}; diff --git a/crates/leit_index/src/memory.rs b/crates/leit_index/src/memory.rs index 8ff56eb..7520266 100644 --- a/crates/leit_index/src/memory.rs +++ b/crates/leit_index/src/memory.rs @@ -27,13 +27,12 @@ pub(crate) struct TermEntry { /// A single posting: a document ID and its term frequency for a term. /// /// Postings are aggregated per term and stored in doc-sorted order (ascending doc ID). -/// This type is public primarily for codec benchmarking and analysis. #[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] -pub struct PostingEntry { +pub(crate) struct PostingEntry { /// Document identifier (segment-local, u32). - pub doc_id: u32, + pub(crate) doc_id: u32, /// Term frequency (raw count of term occurrences in the document's field). - pub term_freq: u32, + pub(crate) term_freq: u32, } #[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] @@ -141,12 +140,20 @@ impl InMemoryIndex { &self.postings } - /// Return all postings indexed by term ID. - /// - /// Postings are doc-sorted (ascending doc ID) within each term's list. - /// This method is primarily used for codec benchmarking and analysis. - pub fn postings_by_term(&self) -> &BTreeMap> { - &self.postings + /// Snapshot postings as primitive tuples for out-of-crate benchmarks. + #[cfg(feature = "bench-internals")] + #[doc(hidden)] + pub fn benchmark_postings(&self) -> Vec> { + self.postings + .values() + .filter(|postings| !postings.is_empty()) + .map(|postings| { + postings + .iter() + .map(|posting| (posting.doc_id, posting.term_freq)) + .collect() + }) + .collect() } fn avg_field_doc_length(&self, field: FieldId) -> f32 { diff --git a/crates/leit_wind_tunnel_index/Cargo.toml b/crates/leit_wind_tunnel_index/Cargo.toml index 2eb0a3e..d736383 100644 --- a/crates/leit_wind_tunnel_index/Cargo.toml +++ b/crates/leit_wind_tunnel_index/Cargo.toml @@ -19,7 +19,7 @@ bench = false [dev-dependencies] criterion = { workspace = true } leit_core = { features = ["std"], workspace = true } -leit_index = { features = ["std"], workspace = true } +leit_index = { features = ["bench-internals", "std"], workspace = true } leit_postings = { features = ["std"], workspace = true } leit_text = { features = ["std"], workspace = true } leit_wind_tunnel = { workspace = true } diff --git a/crates/leit_wind_tunnel_index/benches/codec_compare.rs b/crates/leit_wind_tunnel_index/benches/codec_compare.rs index 895fe53..81f4460 100644 --- a/crates/leit_wind_tunnel_index/benches/codec_compare.rs +++ b/crates/leit_wind_tunnel_index/benches/codec_compare.rs @@ -16,10 +16,10 @@ reason = "criterion_group! generates an undocumented public `benches` fn" )] -use criterion::{BenchmarkId, Criterion, criterion_group, criterion_main}; +use criterion::{BenchmarkId, Criterion, Throughput, criterion_group, criterion_main}; use leit_core::{FieldId, SegmentLocalDocId, TermFreq}; use leit_index::InMemoryIndexBuilder; -use leit_postings::codec::{BlockDeltaCodec, Codec, CodecId, DeltaVarintCodec}; +use leit_postings::codec::{BlockDeltaCodec, Codec, DeltaVarintCodec}; use leit_text::{Analyzer, FieldAnalyzers, UnicodeNormalizer, WhitespaceTokenizer}; use leit_wind_tunnel::{CorpusGenerator, corpus::GeneratedDoc}; @@ -33,6 +33,22 @@ const BODY: FieldId = FieldId::new(2); /// Postings list type: `Vec` of (`SegmentLocalDocId`, `TermFreq`) tuples per term. type PostingsList = Vec>; +struct CorpusCase { + label: &'static str, + doc_count: u32, + postings: PostingsList, +} + +impl CorpusCase { + fn total_postings(&self) -> usize { + self.postings.iter().map(Vec::len).sum() + } + + fn max_postings_len(&self) -> usize { + self.postings.iter().map(Vec::len).max().unwrap_or(0) + } +} + /// Build the field analyzers used for indexing (whitespace tokenizer + unicode /// normalizer on both fields), matching the wind-tunnel integration tests. fn make_analyzers() -> FieldAnalyzers { @@ -67,17 +83,36 @@ fn extract_postings(corpus: &[GeneratedDoc]) -> PostingsList { // Extract all postings as (SegmentLocalDocId, TermFreq) tuples. // Postings are already doc-sorted from the index. - let mut all_postings = Vec::new(); - for posting_list in index.postings_by_term().values() { - let postings: Vec<(SegmentLocalDocId, TermFreq)> = posting_list - .iter() - .map(|p| (SegmentLocalDocId::new(p.doc_id), TermFreq::new(p.term_freq))) - .collect(); - if !postings.is_empty() { - all_postings.push(postings); + index + .benchmark_postings() + .into_iter() + .map(|posting_list| { + posting_list + .into_iter() + .map(|(doc_id, term_freq)| { + (SegmentLocalDocId::new(doc_id), TermFreq::new(term_freq)) + }) + .collect() + }) + .collect() +} + +fn validate_decodes(codec: &C, encoded: &[Vec], postings: &PostingsList) { + let max_len = postings.iter().map(Vec::len).max().unwrap_or(0); + let mut docs = Vec::with_capacity(max_len); + let mut tfs = Vec::with_capacity(max_len); + + for (bytes, expected) in encoded.iter().zip(postings) { + codec + .decode(bytes, &mut docs, &mut tfs) + .expect("decode should succeed"); + assert_eq!(docs.len(), expected.len(), "doc count mismatch"); + assert_eq!(tfs.len(), expected.len(), "tf count mismatch"); + for (index, (doc_id, tf)) in expected.iter().enumerate() { + assert_eq!(docs[index], *doc_id, "doc mismatch at index {index}"); + assert_eq!(tfs[index], *tf, "tf mismatch at index {index}"); } } - all_postings } /// Measure compression ratio and emit a summary table. @@ -118,23 +153,23 @@ fn bench_codec_encode_decode(c: &mut Criterion) { let generator = CorpusGenerator::new(SEED); // Prepare corpora and postings once outside the benchmark loop. - #[expect( - clippy::useless_vec, - reason = "vec literal is the idiomatic way to construct this list" - )] - let corpora = vec![ - (1_000_u32, generator.generate(1_000)), - (10_000_u32, generator.generate(10_000)), + let corpora = [ + ("1k", 1_000_u32, generator.generate(1_000)), + ("10k", 10_000_u32, generator.generate(10_000)), ]; - let all_postings: Vec<(u32, PostingsList)> = corpora + let all_postings: Vec = corpora .iter() - .map(|(size, corpus)| (*size, extract_postings(corpus))) + .map(|(label, doc_count, corpus)| CorpusCase { + label, + doc_count: *doc_count, + postings: extract_postings(corpus), + }) .collect(); // Report compression sizes upfront. - for (corpus_size, postings_list) in &all_postings { - let mut total_postings = 0; + for corpus in &all_postings { + let total_postings = corpus.total_postings(); let mut sizes = Vec::new(); // Encode all postings with each codec and measure total size. @@ -144,43 +179,35 @@ fn bench_codec_encode_decode(c: &mut Criterion) { let mut dv_total_size = 0; let mut bd_total_size = 0; - for postings in postings_list { - if !postings.is_empty() { - total_postings += postings.len(); + for postings in &corpus.postings { + let dv_encoded = delta_varint_codec.encode(postings); + dv_total_size += dv_encoded.len(); - let dv_encoded = delta_varint_codec.encode(postings); - dv_total_size += dv_encoded.len(); - - let bd_encoded = block_delta_codec.encode(postings); - bd_total_size += bd_encoded.len(); - } + let bd_encoded = block_delta_codec.encode(postings); + bd_total_size += bd_encoded.len(); } sizes.push(("DeltaVarint", dv_total_size)); sizes.push(("BlockDelta", bd_total_size)); - report_compression_detailed(*corpus_size, total_postings, &sizes); + report_compression_detailed(corpus.doc_count, total_postings, &sizes); } let mut group = c.benchmark_group("codec_encode"); - for (corpus_size_label, corpus_size, postings_list) in all_postings.iter().map(|(s, p)| { - let label = if *s == 1_000 { "1k" } else { "10k" }; - (label, *s, p) - }) { + for corpus in &all_postings { let delta_varint_codec = DeltaVarintCodec; let block_delta_codec = BlockDeltaCodec; + group.throughput(Throughput::Elements(corpus.total_postings() as u64)); group.bench_with_input( - BenchmarkId::from_parameter(format!("deltavarint/{}", corpus_size_label)), - &corpus_size, + BenchmarkId::from_parameter(format!("deltavarint/{}", corpus.label)), + &corpus.doc_count, |b, _| { b.iter(|| { let mut total_bytes = 0; - for postings in postings_list { - if !postings.is_empty() { - let encoded = delta_varint_codec.encode(postings); - total_bytes += encoded.len(); - } + for postings in &corpus.postings { + let encoded = delta_varint_codec.encode(postings); + total_bytes += encoded.len(); } criterion::black_box(total_bytes); }); @@ -188,16 +215,14 @@ fn bench_codec_encode_decode(c: &mut Criterion) { ); group.bench_with_input( - BenchmarkId::from_parameter(format!("blockdelta/{}", corpus_size_label)), - &corpus_size, + BenchmarkId::from_parameter(format!("blockdelta/{}", corpus.label)), + &corpus.doc_count, |b, _| { b.iter(|| { let mut total_bytes = 0; - for postings in postings_list { - if !postings.is_empty() { - let encoded = block_delta_codec.encode(postings); - total_bytes += encoded.len(); - } + for postings in &corpus.postings { + let encoded = block_delta_codec.encode(postings); + total_bytes += encoded.len(); } criterion::black_box(total_bytes); }); @@ -207,57 +232,40 @@ fn bench_codec_encode_decode(c: &mut Criterion) { group.finish(); let mut group = c.benchmark_group("codec_decode"); - for (corpus_size_label, corpus_size, postings_list) in all_postings.iter().map(|(s, p)| { - let label = if *s == 1_000 { "1k" } else { "10k" }; - (label, *s, p) - }) { + for corpus in &all_postings { let delta_varint_codec = DeltaVarintCodec; let block_delta_codec = BlockDeltaCodec; // Pre-encode for decode benchmarks. - let dv_encoded: Vec> = postings_list + let dv_encoded: Vec> = corpus + .postings .iter() - .map(|p| { - if p.is_empty() { - vec![CodecId::DeltaVarint.to_u8()] - } else { - delta_varint_codec.encode(p) - } - }) + .map(|postings| delta_varint_codec.encode(postings)) .collect(); - let bd_encoded: Vec> = postings_list + let bd_encoded: Vec> = corpus + .postings .iter() - .map(|p| { - if p.is_empty() { - vec![CodecId::BlockDelta.to_u8()] - } else { - block_delta_codec.encode(p) - } - }) + .map(|postings| block_delta_codec.encode(postings)) .collect(); + validate_decodes(&delta_varint_codec, &dv_encoded, &corpus.postings); + validate_decodes(&block_delta_codec, &bd_encoded, &corpus.postings); + group.throughput(Throughput::Elements(corpus.total_postings() as u64)); + group.bench_with_input( - BenchmarkId::from_parameter(format!("deltavarint/{}", corpus_size_label)), - &corpus_size, + BenchmarkId::from_parameter(format!("deltavarint/{}", corpus.label)), + &corpus.doc_count, |b, _| { + let max_len = corpus.max_postings_len(); + let mut docs = Vec::with_capacity(max_len); + let mut tfs = Vec::with_capacity(max_len); b.iter(|| { let mut decoded_count = 0; - for (encoded, original_postings) in dv_encoded.iter().zip(postings_list) { - let mut docs = Vec::new(); - let mut tfs = Vec::new(); + for encoded in &dv_encoded { delta_varint_codec .decode(encoded, &mut docs, &mut tfs) .expect("decode should succeed"); - - // Sanity gate: verify decode matches original (compare typed values). - assert_eq!(docs.len(), original_postings.len(), "doc count mismatch"); - assert_eq!(tfs.len(), original_postings.len(), "tf count mismatch"); - for (i, (doc_id, tf)) in original_postings.iter().enumerate() { - assert_eq!(docs[i], *doc_id, "doc mismatch at index {i}"); - assert_eq!(tfs[i], *tf, "tf mismatch at index {i}"); - } - decoded_count += docs.len(); } criterion::black_box(decoded_count); @@ -266,26 +274,18 @@ fn bench_codec_encode_decode(c: &mut Criterion) { ); group.bench_with_input( - BenchmarkId::from_parameter(format!("blockdelta/{}", corpus_size_label)), - &corpus_size, + BenchmarkId::from_parameter(format!("blockdelta/{}", corpus.label)), + &corpus.doc_count, |b, _| { + let max_len = corpus.max_postings_len(); + let mut docs = Vec::with_capacity(max_len); + let mut tfs = Vec::with_capacity(max_len); b.iter(|| { let mut decoded_count = 0; - for (encoded, original_postings) in bd_encoded.iter().zip(postings_list) { - let mut docs = Vec::new(); - let mut tfs = Vec::new(); + for encoded in &bd_encoded { block_delta_codec .decode(encoded, &mut docs, &mut tfs) .expect("decode should succeed"); - - // Sanity gate: verify decode matches original (compare typed values). - assert_eq!(docs.len(), original_postings.len(), "doc count mismatch"); - assert_eq!(tfs.len(), original_postings.len(), "tf count mismatch"); - for (i, (doc_id, tf)) in original_postings.iter().enumerate() { - assert_eq!(docs[i], *doc_id, "doc mismatch at index {i}"); - assert_eq!(tfs[i], *tf, "tf mismatch at index {i}"); - } - decoded_count += docs.len(); } criterion::black_box(decoded_count); diff --git a/docs/2026-05-30-codec-tradeoffs.md b/docs/2026-05-30-codec-tradeoffs.md index 711fd8e..0f8904a 100644 --- a/docs/2026-05-30-codec-tradeoffs.md +++ b/docs/2026-05-30-codec-tradeoffs.md @@ -2,7 +2,7 @@ **Status:** Evidence-based analysis from SCENARIO-0070 codec benchmarks (ITER-0002, STORY-0006 AC-3). -**Measurement:** `cargo bench -p leit_wind_tunnel_index --bench codec_compare` on the deterministic wind-tunnel corpus (SEED=42, 1K and 10K documents, Zipfian term distribution). +**Measurement:** `cargo bench -p leit_wind_tunnel_index --bench codec_compare` on the deterministic wind-tunnel corpus (SEED=42, 1K and 10K documents, Zipfian term distribution). Encode measures codec work plus allocation of each encoded output `Vec`. Decode reuses preallocated doc/TF buffers; correctness validation runs before, not inside, the timed loop. Criterion reports throughput in postings per second. --- @@ -13,38 +13,31 @@ Both codecs compress postings lists to ~25–27% of the uncompressed baseline (8 - **DeltaVarint**: single-block, varint-encoded deltas, ~2.03–2.05 bytes/posting - **BlockDelta**: 128-doc blocks with per-block headers, ~2.10–2.19 bytes/posting -**Decode latency:** -- 1K corpus: DeltaVarint ~285 µs, BlockDelta ~297 µs (encode/decode times are comparable; BlockDelta slower due to per-block overhead) -- 10K corpus: DeltaVarint ~1.34 ms, BlockDelta ~1.48 ms - -**Encode latency:** -- 1K corpus: DeltaVarint ~188 µs, BlockDelta ~311 µs (~1.65× slower) -- 10K corpus: DeltaVarint ~1.67 ms, BlockDelta ~2.54 ms (~1.52× slower) +Latency results from the earlier allocation-and-assertion-inclusive loop are intentionally omitted. Fresh results from the corrected loop must be recorded before latency is used as decision evidence. --- ## Tradeoff Rationale -### DeltaVarint: Decode speed, simplicity -- **Single stream**: no block metadata to parse, minimal decode latency. +### DeltaVarint: Simplicity +- **Single stream**: no block metadata to parse. - **Simplest codec**: delta encoding + varints, lowest complexity on the read path. - **Trade-off**: no block structure means future block-aware features (selective decode, skip, WAND doc-range pruning) require full decode. -- **Encode cost**: low; varint encoding is linear and fast. +- **Encode path**: linear delta and varint encoding. ### BlockDelta: Block-aware future evolution - **128-doc blocks**: each block independently decodable; enables Phase 3 features (selective block skip, WAND pruning with block-level doc ranges). - **Per-block header overhead**: first_doc, last_doc, doc_bytes_len increase encoded size slightly vs DeltaVarint. -- **Trade-off**: decode is ~4–11% slower due to per-block header parsing; blocks do not improve memory footprint (compression ratio is similar). -- **Encode cost**: higher; block boundaries and per-block headers add work. +- **Trade-off**: per-block headers add parsing and encoding work; blocks do not improve memory footprint (compression ratio is similar). --- ## Conclusion -**For Phase 2 (v1)**: DeltaVarint is sufficient and simpler; it achieves the same compression ratio with lower latency. +**For Phase 2 (v1)**: DeltaVarint is sufficient and simpler; it achieves a similar compression ratio without block metadata. -**For Phase 3+ (selective decode, block-aware WAND)**: BlockDelta's block structure is necessary to enable those features without full decode. The ~4–11% decode-latency cost is acceptable when the alternative is a format migration. +**For Phase 3+ (selective decode, block-aware WAND)**: BlockDelta's block structure enables those features without full decode. -The benchmark confirms that **compression efficiency is not the differentiator**—both codecs perform similarly. The decision is **architectural**: DeltaVarint for speed/simplicity in v1, BlockDelta for extensibility in v2+. +The compression measurements indicate that **compression efficiency is not the differentiator**. The decision is currently **architectural**: DeltaVarint for simplicity in v1, BlockDelta for extensibility in v2+; corrected latency results may refine that tradeoff. **Current production choice**: DeltaVarint is the default; BlockDelta is implemented and tested in parallel for Phase 3 integration.