diff --git a/crates/cli/src/proxy/pipeline.rs b/crates/cli/src/proxy/pipeline.rs index 2ffd771..8024033 100644 --- a/crates/cli/src/proxy/pipeline.rs +++ b/crates/cli/src/proxy/pipeline.rs @@ -214,7 +214,7 @@ mod tests { "[{}]", (0..60) .map(|i| format!( - r#"{{"id":{i},"state":"open","title":"r{i}","body":"{}"}}"#, + r#"{{"id":{i},"state":"open","title":"r{i}","body":"{} record {i}"}}"#, "long boilerplate detail text repeated for bulk ".repeat(8) )) .collect::>() diff --git a/crates/ffi/src/lib.rs b/crates/ffi/src/lib.rs index b1abb21..67ed782 100644 --- a/crates/ffi/src/lib.rs +++ b/crates/ffi/src/lib.rs @@ -509,7 +509,7 @@ mod tests { let out: Value = serde_json::from_str(&rewrite_request(session, Some(request.as_bytes()))).unwrap(); assert_eq!(out["stats"]["blocks_rewritten"], 1); - assert_eq!(out["stats"]["transforms"][0], "columnar"); + assert_eq!(out["stats"]["transforms"][0], "nested"); let tool = out["request"]["messages"][1]["content"].as_str().unwrap(); assert!(tool.len() < ls.len(), "the tool output must shrink"); assert_eq!( diff --git a/crates/optimize/src/doc.rs b/crates/optimize/src/doc.rs index c90f55a..fd3cc89 100644 --- a/crates/optimize/src/doc.rs +++ b/crates/optimize/src/doc.rs @@ -1,7 +1,10 @@ use serde_json::{Map, Value}; +use std::sync::Arc; use crate::column::string_token; use crate::nested::{self, Nested}; +use crate::norm::Normalize; +use crate::tokens::{ByteCounter, TokenCounter}; use crate::transform::{Encoded, Transform}; const HEADER: &str = "SWDOC"; @@ -10,7 +13,23 @@ const MAP_MIN: usize = 8; // Readable document codec for a top-level object: scalar fields stay literal, and each nested record // collection (array of objects, or a map of objects/scalars/arrays) becomes an embedded readable // table, so object-map responses (registries, lockfiles) compress inline instead of only offloading. -pub struct Doc; +pub struct Doc { + counter: Arc, +} + +impl Default for Doc { + fn default() -> Self { + Self { + counter: Arc::new(ByteCounter), + } + } +} + +impl Doc { + pub fn with_counter(counter: Arc) -> Self { + Self { counter } + } +} impl Transform for Doc { fn id(&self) -> &'static str { @@ -32,16 +51,27 @@ impl Transform for Doc { for (k, v) in obj { if let Some((shape, keycol, records)) = tablify(v) { // Normalize (pull inner collections into side tables) when it beats a plain table. - let plain = Nested + let plain = Nested::with_counter(self.counter.clone()) + .try_encode(&Value::Array(records.clone())) + .map(|e| e.wire); + let normed = Normalize::with_counter(self.counter.clone()) .try_encode(&Value::Array(records.clone())) .map(|e| e.wire); - let normed = crate::norm::encode(&records); let table = match (normed, plain) { - (Some(a), Some(b)) => Some(if a.len() < b.len() { a } else { b }), + (Some(a), Some(b)) => Some( + if (self.counter.count(&a), a.len()) < (self.counter.count(&b), b.len()) { + a + } else { + b + }, + ), (a, b) => a.or(b), }; if let Some(tw) = table - && tw.len() < serde_json::to_string(v).map(|s| s.len()).unwrap_or(0) + && self.counter.count(&tw) + < serde_json::to_string(v) + .map(|s| self.counter.count(&s)) + .unwrap_or(0) { let nlines = tw.split('\n').count(); let kc = keycol.map(|c| string_token(&c)).unwrap_or_default(); @@ -193,10 +223,25 @@ fn refold(shape: char, keycol: Option<&str>, items: &[Value]) -> Option { #[cfg(test)] mod tests { use super::*; + use std::sync::Arc; + + struct NormalizePreferred; + + impl TokenCounter for NormalizePreferred { + fn count(&self, text: &str) -> usize { + if text.starts_with("SWNORM") { + 1 + } else if text.starts_with("SWNEST") { + 100 + } else { + text.len() + } + } + } fn round_trip(json: &str) { let value: Value = serde_json::from_str(json).unwrap(); - if let Some(enc) = Doc.try_encode(&value) { + if let Some(enc) = Doc::default().try_encode(&value) { assert_eq!(decode(&enc.wire).unwrap(), value, "wire:\n{}", enc.wire); } } @@ -211,7 +256,7 @@ mod tests { #[test] fn tablifies_a_map_of_objects_and_round_trips() { let value: Value = serde_json::from_str(&map_of_objects(10)).unwrap(); - let enc = Doc.try_encode(&value).unwrap(); + let enc = Doc::default().try_encode(&value).unwrap(); assert!(enc.wire.contains("SWDOC") && enc.wire.contains("index.js")); assert_eq!(decode(&enc.wire).unwrap(), value); } @@ -227,6 +272,44 @@ mod tests { )); } + #[test] + fn counter_selects_the_embedded_table_and_admission_gate() { + let packages: Vec = (0..2) + .map(|i| { + serde_json::json!({ + "id": i, + "name": format!("pkg-{i}"), + "dependencies": [{"name": format!("dep-{i}"), "req": "^1"}], + }) + }) + .collect(); + let value = serde_json::json!({"packages": packages}); + let records = value["packages"].as_array().unwrap().clone(); + let plain = Nested::default() + .try_encode(&Value::Array(records.clone())) + .unwrap() + .wire; + let normed = Normalize::default() + .try_encode(&Value::Array(records)) + .unwrap() + .wire; + assert!( + normed.len() > plain.len(), + "the fixture must diverge from a byte-length choice: norm={} nested={}", + normed.len(), + plain.len() + ); + + let wire = Doc::with_counter(Arc::new(NormalizePreferred)) + .try_encode(&value) + .unwrap() + .wire; + assert!( + wire.contains("SWNORM"), + "Doc must use its counter for both table selection and table admission" + ); + } + #[test] fn tablifies_a_map_of_scalars_and_arrays() { let times: Vec = (0..10) @@ -242,6 +325,6 @@ mod tests { #[test] fn refuses_a_plain_struct_with_no_collection() { let value: Value = serde_json::from_str(r#"{"name":"pkg","version":"1.0.0"}"#).unwrap(); - assert!(Doc.try_encode(&value).is_none()); + assert!(Doc::default().try_encode(&value).is_none()); } } diff --git a/crates/optimize/src/lib.rs b/crates/optimize/src/lib.rs index b9fcd24..ada9335 100644 --- a/crates/optimize/src/lib.rs +++ b/crates/optimize/src/lib.rs @@ -69,6 +69,9 @@ use offload::{OffloadStore, Store}; use tokens::{ByteCounter, TokenCounter}; use transform::{Encoded, TextProposer, Transform}; +type StructuredCandidate = (String, &'static str, Certificate, f64, usize); +type StructuredSearch = (Option, Option); + pub enum Outcome { Compressed { wire: String, @@ -134,6 +137,7 @@ const RELEVANCE_MAX_INLINE: usize = 24; const PROSE_SUMMARY_MIN_BYTES: usize = 1000; const PROSE_SUMMARY_FRACTION: f64 = 0.5; +const BUILTIN_TRANSFORM_COUNT: usize = 4; pub struct Optimizer { transforms: Vec>, @@ -159,13 +163,9 @@ impl Default for Optimizer { impl Optimizer { pub fn new(gate: NetCostGate, zone: Zone) -> Self { + let counter: Arc = Arc::new(ByteCounter); Self { - transforms: vec![ - Box::new(Columnar::default()), - Box::new(Normalize), - Box::new(Nested), - Box::new(Doc), - ], + transforms: Self::built_in_transforms(counter.clone()), // Built-in text codecs are unreadable inline; only host-registered proposers ship here, // and a host owns its codec's readability. See compress_text. text_proposers: Vec::new(), @@ -173,7 +173,7 @@ impl Optimizer { gate, zone, store: Arc::new(Store::default()), - counter: Arc::new(ByteCounter), + counter, embedder: Arc::new(distilled::DistilledEmbedder), prose_mode: false, prose_shrinker: None, @@ -181,6 +181,17 @@ impl Optimizer { } } + // Keep all built-in structured codecs on the same token counter. Custom transforms remain + // appended after this fixed prefix and retain their own construction/configuration. + fn built_in_transforms(counter: Arc) -> Vec> { + vec![ + Box::new(Columnar::with_counter(counter.clone())), + Box::new(Normalize::with_counter(counter.clone())), + Box::new(Nested::with_counter(counter.clone())), + Box::new(Doc::with_counter(counter)), + ] + } + // Pick how recoverable eviction competes with inline: Off, Auto (cost model), or Always. pub fn set_offload_mode(&mut self, mode: OffloadMode) { self.offload_mode = mode; @@ -245,8 +256,14 @@ impl Optimizer { // Switch the pipeline from the default byte proxy to a real token counter, so codec selection // and every gate decision optimize token cost. pub fn with_counter(mut self, counter: Arc) -> Self { - // Index 0 is the built-in readable table codec; rebuild only it so a with_transform addition survives. - self.transforms[0] = Box::new(Columnar::with_counter(counter.clone())); + // Rebuild the fixed built-in prefix without disturbing host transforms appended after it. + for (slot, transform) in Self::built_in_transforms(counter.clone()) + .into_iter() + .enumerate() + { + self.transforms[slot] = transform; + } + debug_assert!(self.transforms.len() >= BUILTIN_TRANSFORM_COUNT); self.counter = counter; self } @@ -424,21 +441,46 @@ impl Optimizer { }; } + let (best, refusal) = self.best_structured(raw, &value); + if let Some((wire, id, certificate, usd, _)) = best { + return self.ship_inline_or_offload(raw, wire, id, certificate, usd); + } + + let outcome = self.try_offload(raw); + if matches!( + &outcome, + Outcome::KeptVerbatim { + reason: KeptReason::NotApplicable + } + ) && let Some(reason) = refusal + { + Outcome::KeptVerbatim { reason } + } else { + outcome + } + } + + // Best-of-N for structured codecs: prove, price, and detector-check every applicable wire, + // then use the actual configured token counter to pick the cheapest. Stable transform-id and + // byte-length ties keep the selected wire deterministic for prompt-cache reuse. + fn best_structured(&self, raw: &str, value: &Value) -> StructuredSearch { + let canonical = atom::canonicalize(value); + let mut best: Option = None; + let mut refusal = None; for transform in &self.transforms { - let Some(encoded) = transform.try_encode(&value) else { + let Some(encoded) = transform.try_encode(value) else { continue; }; let certificate = match admit( - &value, + value, &encoded, |v| transform.try_encode(v), !transform.trusted(), ) { Ok(certificate) => certificate, - Err(refusal) => { - return Outcome::KeptVerbatim { - reason: KeptReason::Refused(transform.id(), refusal), - }; + Err(reason) => { + refusal.get_or_insert(KeptReason::Refused(transform.id(), reason)); + continue; } }; // Price the counterfactual against raw (what the model would be billed), not canonical, @@ -450,24 +492,25 @@ impl Optimizer { continue; }; - if !detectorgate::detector_findings(&atom::canonicalize(&value), &encoded.wire) - .is_empty() - { - return Outcome::KeptVerbatim { - reason: KeptReason::DetectorFired, - }; + if !detectorgate::detector_findings(&canonical, &encoded.wire).is_empty() { + refusal.get_or_insert(KeptReason::DetectorFired); + continue; } - return self.ship_inline_or_offload( - raw, - encoded.wire, - transform.id(), - certificate, - usd, - ); + let units = self.counter.count(&encoded.wire); + let id = transform.id(); + let wire_bytes = encoded.wire.len(); + let better = match &best { + Some((best_wire, best_id, _, _, best_units)) => { + (units, id, wire_bytes) < (*best_units, *best_id, best_wire.len()) + } + None => true, + }; + if better { + best = Some((encoded.wire, id, certificate, usd, units)); + } } - - self.try_offload(raw) + (best, refusal) } // Readable inline text codecs ship first: grouped (grep by file, listings by directory) and tree diff --git a/crates/optimize/src/lock.rs b/crates/optimize/src/lock.rs index 51f7e84..4492baa 100644 --- a/crates/optimize/src/lock.rs +++ b/crates/optimize/src/lock.rs @@ -22,7 +22,7 @@ pub fn encode(raw: &str) -> Option { if records.len() < 3 { return None; } - let table = Nested.try_encode(&Value::Array(records))?.wire; + let table = Nested::default().try_encode(&Value::Array(records))?.wire; let wire = format!( "{HEADER} {} {}\n{preamble}{table}", trailing as u8, diff --git a/crates/optimize/src/nested.rs b/crates/optimize/src/nested.rs index 77277da..2b894ba 100644 --- a/crates/optimize/src/nested.rs +++ b/crates/optimize/src/nested.rs @@ -1,6 +1,8 @@ use serde_json::{Map, Value}; +use std::sync::Arc; use crate::column::string_token; +use crate::tokens::{ByteCounter, TokenCounter}; use crate::transform::{Encoded, Transform}; const HEADER: &str = "SWNEST"; @@ -9,7 +11,25 @@ const SEP: char = '.'; // Row-major readable table for nested/ragged record arrays: flatten scalar paths to dotted columns, // keep every value literal so the model reads it, union schema with an empty cell for an absent field. // A column that is present and identical in every record is stated once as a constant, not per row. -pub struct Nested; +pub struct Nested { + // Nested currently has one readable wire shape, but keeping the configured counter with it + // makes callers compose it consistently with the other structured codecs. + _counter: Arc, +} + +impl Default for Nested { + fn default() -> Self { + Self { + _counter: Arc::new(ByteCounter), + } + } +} + +impl Nested { + pub fn with_counter(counter: Arc) -> Self { + Self { _counter: counter } + } +} impl Transform for Nested { fn id(&self) -> &'static str { @@ -267,7 +287,7 @@ mod tests { fn round_trip(json: &str) { let value: Value = serde_json::from_str(json).unwrap(); - let encoded = Nested.try_encode(&value).unwrap(); + let encoded = Nested::default().try_encode(&value).unwrap(); assert_eq!( decode(&encoded.wire).unwrap(), value, @@ -302,7 +322,7 @@ mod tests { r#"[{"n":1,"repo":"cli/cli","kind":"pr"},{"n":2,"repo":"cli/cli","kind":"pr"}]"#, ) .unwrap(); - let wire = Nested.try_encode(&value).unwrap().wire; + let wire = Nested::default().try_encode(&value).unwrap().wire; assert!( wire.starts_with("SWNEST 2 2 1"), "two constants, one varying: {wire}" @@ -314,13 +334,13 @@ mod tests { fn string_values_are_literal_and_readable() { let value: Value = serde_json::from_str(r#"[{"u":{"login":"alice"}},{"u":{"login":"bob"}}]"#).unwrap(); - let wire = Nested.try_encode(&value).unwrap().wire; + let wire = Nested::default().try_encode(&value).unwrap().wire; assert!(wire.contains("alice") && wire.contains("bob") && !wire.contains("\"alice\"")); } #[test] fn keys_with_the_separator_are_refused() { let value: Value = serde_json::from_str(r#"[{"a.b":1},{"a.b":2}]"#).unwrap(); - assert!(Nested.try_encode(&value).is_none()); + assert!(Nested::default().try_encode(&value).is_none()); } } diff --git a/crates/optimize/src/norm.rs b/crates/optimize/src/norm.rs index 4a9e84f..167de8c 100644 --- a/crates/optimize/src/norm.rs +++ b/crates/optimize/src/norm.rs @@ -1,9 +1,11 @@ use std::collections::{HashMap, HashSet}; +use std::sync::Arc; use serde_json::{Map, Value}; use crate::column::string_token; use crate::nested::{self, Nested}; +use crate::tokens::{ByteCounter, TokenCounter}; use crate::transform::{Encoded, Transform}; const HEADER: &str = "SWNORM"; @@ -12,7 +14,27 @@ const HEADER: &str = "SWNORM"; // release file-lists) are pulled into side tables keyed by a detected unique field, so repeating // groups compress into their own table instead of a per-row json blob. The model reads it as a // foreign-key join, which it can do (unlike a positional join). Reversible; self-verified in admit. -pub struct Normalize; +pub struct Normalize { + counter: Arc, +} + +impl Default for Normalize { + fn default() -> Self { + Self { + counter: Arc::new(ByteCounter), + } + } +} + +impl Normalize { + pub fn with_counter(counter: Arc) -> Self { + Self { counter } + } + + fn encode(&self, items: &[Value]) -> Option { + encode_with_counter(items, self.counter.clone()) + } +} impl Transform for Normalize { fn id(&self) -> &'static str { @@ -25,7 +47,7 @@ impl Transform for Normalize { fn try_encode(&self, value: &Value) -> Option { let items = value.as_array()?; - let wire = encode(items)?; + let wire = self.encode(items)?; let decoded = decode(&wire)?; if crate::atom::canonicalize(&decoded) != crate::atom::canonicalize(value) { return None; @@ -89,6 +111,10 @@ fn pick_sidefk(rows: &[(Value, Value)]) -> String { } pub fn encode(items: &[Value]) -> Option { + encode_with_counter(items, Arc::new(ByteCounter)) +} + +fn encode_with_counter(items: &[Value], counter: Arc) -> Option { if items.len() < 2 { return None; } @@ -137,7 +163,9 @@ pub fn encode(items: &[Value]) -> Option { return None; } - let main_wire = Nested.try_encode(&Value::Array(main))?.wire; + let main_wire = Nested::with_counter(counter.clone()) + .try_encode(&Value::Array(main))? + .wire; let mut parts = vec![ format!("{HEADER}\t{}\t{}", string_token(&fk), order.len()), format!("@main\t{}", main_wire.split('\n').count()), @@ -154,7 +182,9 @@ pub fn encode(items: &[Value]) -> Option { Value::Object(r) }) .collect(); - let side_wire = Nested.try_encode(&Value::Array(side_records))?.wire; + let side_wire = Nested::with_counter(counter.clone()) + .try_encode(&Value::Array(side_records))? + .wire; parts.push(format!( "@side\t{}\t{}\t{}\t{}", string_token(k), diff --git a/crates/optimize/src/proxy.rs b/crates/optimize/src/proxy.rs index 8ef3119..b626407 100644 --- a/crates/optimize/src/proxy.rs +++ b/crates/optimize/src/proxy.rs @@ -501,12 +501,14 @@ mod tests { "x".repeat(20_000) } - // A records array whose long bodies the content-table preview truncates, so offload beats inline. + // Long per-row bodies exceed the readable preview cell limit, so the covering content table + // makes offload cheaper than any inline wire. (A constant body is now correctly hoisted by + // Nested and stays inline under best-of-N.) fn offloadable() -> String { let rows: Vec = (0..60) .map(|i| { format!( - r#"{{"id":{i},"state":"{}","title":"record {i}","body":"{}"}}"#, + r#"{{"id":{i},"state":"{}","title":"record {i}","body":"{} record {i}"}}"#, if i % 2 == 0 { "open" } else { "closed" }, "long boilerplate detail text repeated for bulk ".repeat(8) ) diff --git a/crates/optimize/tests/admission.rs b/crates/optimize/tests/admission.rs index b43adcb..02ab2a7 100644 --- a/crates/optimize/tests/admission.rs +++ b/crates/optimize/tests/admission.rs @@ -21,7 +21,7 @@ fn big_object() -> String { } #[test] -fn faithful_columnar_transform_is_admitted_and_smaller() { +fn faithful_best_structured_transform_is_admitted_and_smaller() { let mut opt = Optimizer::default(); match opt.compress_block(UNIFORM) { Outcome::Compressed { @@ -30,7 +30,7 @@ fn faithful_columnar_transform_is_admitted_and_smaller() { certificate, saved_usd, } => { - assert_eq!(transform, "columnar"); + assert_eq!(transform, "nested"); assert!(certificate.wire_bytes < certificate.canonical_bytes); assert!(wire.contains("7001") && wire.contains("FAIL")); assert_eq!(certificate.clmh_before, certificate.clmh_after); diff --git a/crates/optimize/tests/fuzz.rs b/crates/optimize/tests/fuzz.rs index a76447a..f54db4a 100644 --- a/crates/optimize/tests/fuzz.rs +++ b/crates/optimize/tests/fuzz.rs @@ -452,7 +452,7 @@ fn doc_round_trips_or_refuses_on_random_documents() { obj.insert("list".into(), Value::Array(arr)); } let value = Value::Object(obj); - if let Some(enc) = Doc.try_encode(&value) { + if let Some(enc) = Doc::default().try_encode(&value) { assert_eq!( canonicalize(&doc_decode(&enc.wire).unwrap()), canonicalize(&value), @@ -482,7 +482,7 @@ fn nested_round_trips_or_refuses_on_random_object_arrays() { }) .collect(); let value = Value::Array(items); - if let Some(encoded) = Nested.try_encode(&value) { + if let Some(encoded) = Nested::default().try_encode(&value) { let decoded = nested_decode(&encoded.wire).expect("wire decodes"); assert_eq!( canonicalize(&decoded), diff --git a/crates/optimize/tests/offload_mode.rs b/crates/optimize/tests/offload_mode.rs index e74206b..fa986a1 100644 --- a/crates/optimize/tests/offload_mode.rs +++ b/crates/optimize/tests/offload_mode.rs @@ -19,7 +19,9 @@ fn record_array() -> String { "id": i, "state": if i % 2 == 0 { "open" } else { "closed" }, "title": format!("change number {i} to the pipeline"), - "body": "x".repeat(300), + // Keep this field unique per row: a fully constant body is now hoisted by + // Nested and intentionally stays inline under best-of-N selection. + "body": format!("{} record {i}", "x".repeat(300)), }) }) .collect(); diff --git a/crates/optimize/tests/structured_search.rs b/crates/optimize/tests/structured_search.rs new file mode 100644 index 0000000..a2ae101 --- /dev/null +++ b/crates/optimize/tests/structured_search.rs @@ -0,0 +1,210 @@ +use std::sync::Arc; + +use secondwind_optimize::columnar::Columnar; +use secondwind_optimize::nested::Nested; +use secondwind_optimize::tokens::TokenCounter; +use secondwind_optimize::transform::{Encoded, Transform}; +use secondwind_optimize::{KeptReason, OffloadMode, Optimizer, Outcome}; +use serde_json::Value; + +struct Candidate { + id: &'static str, + tag: &'static str, + padding: usize, + corrupts: bool, + forges_grep_claim: bool, +} + +impl Transform for Candidate { + fn id(&self) -> &'static str { + self.id + } + + fn try_encode(&self, value: &Value) -> Option { + value.as_object()?; + let mut wire = format!("{} {}", self.tag, serde_json::to_string(value).ok()?); + wire.push_str(&" ".repeat(self.padding)); + if self.forges_grep_claim { + wire.push_str("\nsrc/forged.rs:1:made up"); + } + Some(Encoded { + wire, + decoded: if self.corrupts { + Value::Null + } else { + value.clone() + }, + }) + } +} + +// A deliberately non-native counter lets this test distinguish ranking from the byte-based +// admission gate: the longer candidate is the cheaper one in the configured billing unit. +struct TokenPreference; + +impl TokenCounter for TokenPreference { + fn count(&self, text: &str) -> usize { + match () { + _ if text.contains("cheap-token-wire") => 1, + _ if text.contains("costly-token-wire") => 100, + _ => text.len(), + } + } +} + +fn raw() -> String { + let value = serde_json::json!({ + "kind": "report", + "payload": "the value must remain visible ".repeat(200), + }); + let compact = serde_json::to_string(&value).unwrap(); + let pad = "\n".repeat(300); + format!("{{{pad}\"kind\"{pad}:{pad}\"report\"{pad},\"payload\"{pad}:{pad}{compact}{pad}}}") +} + +fn optimizer() -> Optimizer { + let mut optimizer = Optimizer::default(); + optimizer.set_offload_mode(OffloadMode::Off); + optimizer +} + +#[test] +fn continues_after_an_admission_refusal_to_a_valid_structured_candidate() { + let mut optimizer = optimizer() + .with_transform(Box::new(Candidate { + id: "broken", + tag: "broken-wire", + padding: 0, + corrupts: true, + forges_grep_claim: false, + })) + .with_transform(Box::new(Candidate { + id: "valid", + tag: "valid-wire", + padding: 0, + corrupts: false, + forges_grep_claim: false, + })); + + let Outcome::Compressed { transform, .. } = optimizer.compress_block(&raw()) else { + panic!("a later admitted candidate must still be considered"); + }; + assert_eq!(transform, "valid"); +} + +#[test] +fn reports_an_admission_refusal_only_after_all_structured_candidates_fail() { + let mut optimizer = optimizer().with_transform(Box::new(Candidate { + id: "broken", + tag: "broken-wire", + padding: 0, + corrupts: true, + forges_grep_claim: false, + })); + + let Outcome::KeptVerbatim { + reason: KeptReason::Refused(id, _), + } = optimizer.compress_block(&raw()) + else { + panic!("the admission refusal belongs only to an exhausted structured search"); + }; + assert_eq!(id, "broken"); +} + +#[test] +fn continues_after_a_detector_rejection_to_a_valid_structured_candidate() { + let mut optimizer = optimizer() + .with_transform(Box::new(Candidate { + id: "forged", + tag: "forged-wire", + padding: 0, + corrupts: false, + forges_grep_claim: true, + })) + .with_transform(Box::new(Candidate { + id: "valid", + tag: "valid-wire", + padding: 0, + corrupts: false, + forges_grep_claim: false, + })); + + let Outcome::Compressed { transform, .. } = optimizer.compress_block(&raw()) else { + panic!("a later detector-clean candidate must still be considered"); + }; + assert_eq!(transform, "valid"); +} + +#[test] +fn ranks_structured_candidates_by_the_configured_token_counter() { + let mut optimizer = optimizer() + .with_transform(Box::new(Candidate { + id: "a-costly", + tag: "costly-token-wire", + padding: 0, + corrupts: false, + forges_grep_claim: false, + })) + .with_transform(Box::new(Candidate { + id: "z-cheap", + tag: "cheap-token-wire", + padding: 200, + corrupts: false, + forges_grep_claim: false, + })) + .with_counter(Arc::new(TokenPreference)); + + let Outcome::Compressed { transform, .. } = optimizer.compress_block(&raw()) else { + panic!("expected a structured candidate"); + }; + assert_eq!( + transform, "z-cheap", + "the configured counter, not bytes, ranks wires" + ); +} + +#[test] +fn breaks_equal_token_ties_by_transform_id() { + let mut optimizer = optimizer() + .with_transform(Box::new(Candidate { + id: "z-later", + tag: "tie-wire", + padding: 0, + corrupts: false, + forges_grep_claim: false, + })) + .with_transform(Box::new(Candidate { + id: "a-earlier", + tag: "tie-wire", + padding: 0, + corrupts: false, + forges_grep_claim: false, + })); + + let Outcome::Compressed { transform, .. } = optimizer.compress_block(&raw()) else { + panic!("expected a structured candidate"); + }; + assert_eq!(transform, "a-earlier", "ties must be stable across runs"); +} + +#[test] +fn chooses_the_smaller_builtin_wire_when_formats_overlap() { + let raw = include_str!("../../../bench/compression/corpus/04-small-array.json"); + let value: Value = serde_json::from_str(raw).unwrap(); + let columnar = Columnar::default().try_encode(&value).unwrap().wire; + let nested = Nested::default().try_encode(&value).unwrap().wire; + assert!( + nested.len() < columnar.len(), + "the fixture must make nested beat the first registered codec" + ); + + let mut optimizer = optimizer(); + let Outcome::Compressed { + transform, wire, .. + } = optimizer.compress_block(raw) + else { + panic!("expected an inline structured candidate"); + }; + assert_eq!(transform, "nested"); + assert_eq!(wire, nested); +}