feat(optimize): select the best structured transform

crates/cli/src/proxy/pipeline.rs

@@ -214,7 +214,7 @@ mod tests {
             "[{}]",
             (0..60)
                 .map(|i| format!(
-                    r#"{{"id":{i},"state":"open","title":"r{i}","body":"{}"}}"#,
+                    r#"{{"id":{i},"state":"open","title":"r{i}","body":"{} record {i}"}}"#,
                     "long boilerplate detail text repeated for bulk ".repeat(8)
                 ))
                 .collect::<Vec<_>>()

crates/ffi/src/lib.rs

@@ -509,7 +509,7 @@ mod tests {
         let out: Value =
             serde_json::from_str(&rewrite_request(session, Some(request.as_bytes()))).unwrap();
         assert_eq!(out["stats"]["blocks_rewritten"], 1);
-        assert_eq!(out["stats"]["transforms"][0], "columnar");
+        assert_eq!(out["stats"]["transforms"][0], "nested");
         let tool = out["request"]["messages"][1]["content"].as_str().unwrap();
         assert!(tool.len() < ls.len(), "the tool output must shrink");
         assert_eq!(

crates/optimize/src/doc.rs

@@ -1,7 +1,10 @@
 use serde_json::{Map, Value};
+use std::sync::Arc;
 
 use crate::column::string_token;
 use crate::nested::{self, Nested};
+use crate::norm::Normalize;
+use crate::tokens::{ByteCounter, TokenCounter};
 use crate::transform::{Encoded, Transform};
 
 const HEADER: &str = "SWDOC";
@@ -10,7 +13,23 @@ const MAP_MIN: usize = 8;
 // Readable document codec for a top-level object: scalar fields stay literal, and each nested record
 // collection (array of objects, or a map of objects/scalars/arrays) becomes an embedded readable
 // table, so object-map responses (registries, lockfiles) compress inline instead of only offloading.
-pub struct Doc;
+pub struct Doc {
+    counter: Arc<dyn TokenCounter>,
+}
+
+impl Default for Doc {
+    fn default() -> Self {
+        Self {
+            counter: Arc::new(ByteCounter),
+        }
+    }
+}
+
+impl Doc {
+    pub fn with_counter(counter: Arc<dyn TokenCounter>) -> Self {
+        Self { counter }
+    }
+}
 
 impl Transform for Doc {
     fn id(&self) -> &'static str {
@@ -32,16 +51,27 @@ impl Transform for Doc {
         for (k, v) in obj {
             if let Some((shape, keycol, records)) = tablify(v) {
                 // Normalize (pull inner collections into side tables) when it beats a plain table.
-                let plain = Nested
+                let plain = Nested::with_counter(self.counter.clone())
+                    .try_encode(&Value::Array(records.clone()))
+                    .map(|e| e.wire);
+                let normed = Normalize::with_counter(self.counter.clone())
                     .try_encode(&Value::Array(records.clone()))
                     .map(|e| e.wire);
-                let normed = crate::norm::encode(&records);
                 let table = match (normed, plain) {
-                    (Some(a), Some(b)) => Some(if a.len() < b.len() { a } else { b }),
+                    (Some(a), Some(b)) => Some(
+                        if (self.counter.count(&a), a.len()) < (self.counter.count(&b), b.len()) {
+                            a
+                        } else {
+                            b
+                        },
+                    ),
                     (a, b) => a.or(b),
                 };
                 if let Some(tw) = table
-                    && tw.len() < serde_json::to_string(v).map(|s| s.len()).unwrap_or(0)
+                    && self.counter.count(&tw)
+                        < serde_json::to_string(v)
+                            .map(|s| self.counter.count(&s))
+                            .unwrap_or(0)
                 {
                     let nlines = tw.split('\n').count();
                     let kc = keycol.map(|c| string_token(&c)).unwrap_or_default();
@@ -193,10 +223,25 @@ fn refold(shape: char, keycol: Option<&str>, items: &[Value]) -> Option<Value> {
 #[cfg(test)]
 mod tests {
     use super::*;
+    use std::sync::Arc;
+
+    struct NormalizePreferred;
+
+    impl TokenCounter for NormalizePreferred {
+        fn count(&self, text: &str) -> usize {
+            if text.starts_with("SWNORM") {
+                1
+            } else if text.starts_with("SWNEST") {
+                100
+            } else {
+                text.len()
+            }
+        }
+    }
 
     fn round_trip(json: &str) {
         let value: Value = serde_json::from_str(json).unwrap();
-        if let Some(enc) = Doc.try_encode(&value) {
+        if let Some(enc) = Doc::default().try_encode(&value) {
             assert_eq!(decode(&enc.wire).unwrap(), value, "wire:\n{}", enc.wire);
         }
     }
@@ -211,7 +256,7 @@ mod tests {
     #[test]
     fn tablifies_a_map_of_objects_and_round_trips() {
         let value: Value = serde_json::from_str(&map_of_objects(10)).unwrap();
-        let enc = Doc.try_encode(&value).unwrap();
+        let enc = Doc::default().try_encode(&value).unwrap();
         assert!(enc.wire.contains("SWDOC") && enc.wire.contains("index.js"));
         assert_eq!(decode(&enc.wire).unwrap(), value);
     }
@@ -227,6 +272,44 @@ mod tests {
         ));
     }
 
+    #[test]
+    fn counter_selects_the_embedded_table_and_admission_gate() {
+        let packages: Vec<Value> = (0..2)
+            .map(|i| {
+                serde_json::json!({
+                    "id": i,
+                    "name": format!("pkg-{i}"),
+                    "dependencies": [{"name": format!("dep-{i}"), "req": "^1"}],
+                })
+            })
+            .collect();
+        let value = serde_json::json!({"packages": packages});
+        let records = value["packages"].as_array().unwrap().clone();
+        let plain = Nested::default()
+            .try_encode(&Value::Array(records.clone()))
+            .unwrap()
+            .wire;
+        let normed = Normalize::default()
+            .try_encode(&Value::Array(records))
+            .unwrap()
+            .wire;
+        assert!(
+            normed.len() > plain.len(),
+            "the fixture must diverge from a byte-length choice: norm={} nested={}",
+            normed.len(),
+            plain.len()
+        );
+
+        let wire = Doc::with_counter(Arc::new(NormalizePreferred))
+            .try_encode(&value)
+            .unwrap()
+            .wire;
+        assert!(
+            wire.contains("SWNORM"),
+            "Doc must use its counter for both table selection and table admission"
+        );
+    }
+
     #[test]
     fn tablifies_a_map_of_scalars_and_arrays() {
         let times: Vec<String> = (0..10)
@@ -242,6 +325,6 @@ mod tests {
     #[test]
     fn refuses_a_plain_struct_with_no_collection() {
         let value: Value = serde_json::from_str(r#"{"name":"pkg","version":"1.0.0"}"#).unwrap();
-        assert!(Doc.try_encode(&value).is_none());
+        assert!(Doc::default().try_encode(&value).is_none());
     }
 }

crates/optimize/src/lib.rs

@@ -69,6 +69,9 @@ use offload::{OffloadStore, Store};
 use tokens::{ByteCounter, TokenCounter};
 use transform::{Encoded, TextProposer, Transform};
 
+type StructuredCandidate = (String, &'static str, Certificate, f64, usize);
+type StructuredSearch = (Option<StructuredCandidate>, Option<KeptReason>);
+
 pub enum Outcome {
     Compressed {
         wire: String,
@@ -134,6 +137,7 @@ const RELEVANCE_MAX_INLINE: usize = 24;
 
 const PROSE_SUMMARY_MIN_BYTES: usize = 1000;
 const PROSE_SUMMARY_FRACTION: f64 = 0.5;
+const BUILTIN_TRANSFORM_COUNT: usize = 4;
 
 pub struct Optimizer {
     transforms: Vec<Box<dyn Transform>>,
@@ -159,28 +163,35 @@ impl Default for Optimizer {
 
 impl Optimizer {
     pub fn new(gate: NetCostGate, zone: Zone) -> Self {
+        let counter: Arc<dyn TokenCounter> = Arc::new(ByteCounter);
         Self {
-            transforms: vec![
-                Box::new(Columnar::default()),
-                Box::new(Normalize),
-                Box::new(Nested),
-                Box::new(Doc),
-            ],
+            transforms: Self::built_in_transforms(counter.clone()),
             // Built-in text codecs are unreadable inline; only host-registered proposers ship here,
             // and a host owns its codec's readability. See compress_text.
             text_proposers: Vec::new(),
             proposers_enabled: true,
             gate,
             zone,
             store: Arc::new(Store::default()),
-            counter: Arc::new(ByteCounter),
+            counter,
             embedder: Arc::new(distilled::DistilledEmbedder),
             prose_mode: false,
             prose_shrinker: None,
             offload_mode: OffloadMode::Auto,
         }
     }
 
+    // Keep all built-in structured codecs on the same token counter. Custom transforms remain
+    // appended after this fixed prefix and retain their own construction/configuration.
+    fn built_in_transforms(counter: Arc<dyn TokenCounter>) -> Vec<Box<dyn Transform>> {
+        vec![
+            Box::new(Columnar::with_counter(counter.clone())),
+            Box::new(Normalize::with_counter(counter.clone())),
+            Box::new(Nested::with_counter(counter.clone())),
+            Box::new(Doc::with_counter(counter)),
+        ]
+    }
+
     // Pick how recoverable eviction competes with inline: Off, Auto (cost model), or Always.
     pub fn set_offload_mode(&mut self, mode: OffloadMode) {
         self.offload_mode = mode;
@@ -245,8 +256,14 @@ impl Optimizer {
     // Switch the pipeline from the default byte proxy to a real token counter, so codec selection
     // and every gate decision optimize token cost.
     pub fn with_counter(mut self, counter: Arc<dyn TokenCounter>) -> Self {
-        // Index 0 is the built-in readable table codec; rebuild only it so a with_transform addition survives.
-        self.transforms[0] = Box::new(Columnar::with_counter(counter.clone()));
+        // Rebuild the fixed built-in prefix without disturbing host transforms appended after it.
+        for (slot, transform) in Self::built_in_transforms(counter.clone())
+            .into_iter()
+            .enumerate()
+        {
+            self.transforms[slot] = transform;
+        }
+        debug_assert!(self.transforms.len() >= BUILTIN_TRANSFORM_COUNT);
         self.counter = counter;
         self
     }
@@ -424,21 +441,46 @@ impl Optimizer {
             };
         }
 
+        let (best, refusal) = self.best_structured(raw, &value);
+        if let Some((wire, id, certificate, usd, _)) = best {
+            return self.ship_inline_or_offload(raw, wire, id, certificate, usd);
+        }
+
+        let outcome = self.try_offload(raw);
+        if matches!(
+            &outcome,
+            Outcome::KeptVerbatim {
+                reason: KeptReason::NotApplicable
+            }
+        ) && let Some(reason) = refusal
+        {
+            Outcome::KeptVerbatim { reason }
+        } else {
+            outcome
+        }
+    }
+
+    // Best-of-N for structured codecs: prove, price, and detector-check every applicable wire,
+    // then use the actual configured token counter to pick the cheapest. Stable transform-id and
+    // byte-length ties keep the selected wire deterministic for prompt-cache reuse.
+    fn best_structured(&self, raw: &str, value: &Value) -> StructuredSearch {
+        let canonical = atom::canonicalize(value);
+        let mut best: Option<StructuredCandidate> = None;
+        let mut refusal = None;
         for transform in &self.transforms {
-            let Some(encoded) = transform.try_encode(&value) else {
+            let Some(encoded) = transform.try_encode(value) else {
                 continue;
             };
             let certificate = match admit(
-                &value,
+                value,
                 &encoded,
                 |v| transform.try_encode(v),
                 !transform.trusted(),
             ) {
                 Ok(certificate) => certificate,
-                Err(refusal) => {
-                    return Outcome::KeptVerbatim {
-                        reason: KeptReason::Refused(transform.id(), refusal),
-                    };
+                Err(reason) => {
+                    refusal.get_or_insert(KeptReason::Refused(transform.id(), reason));
+                    continue;
                 }
             };
             // Price the counterfactual against raw (what the model would be billed), not canonical,
@@ -450,24 +492,25 @@ impl Optimizer {
                 continue;
             };
 
-            if !detectorgate::detector_findings(&atom::canonicalize(&value), &encoded.wire)
-                .is_empty()
-            {
-                return Outcome::KeptVerbatim {
-                    reason: KeptReason::DetectorFired,
-                };
+            if !detectorgate::detector_findings(&canonical, &encoded.wire).is_empty() {
+                refusal.get_or_insert(KeptReason::DetectorFired);
+                continue;
             }
 
-            return self.ship_inline_or_offload(
-                raw,
-                encoded.wire,
-                transform.id(),
-                certificate,
-                usd,
-            );
+            let units = self.counter.count(&encoded.wire);
+            let id = transform.id();
+            let wire_bytes = encoded.wire.len();
+            let better = match &best {
+                Some((best_wire, best_id, _, _, best_units)) => {
+                    (units, id, wire_bytes) < (*best_units, *best_id, best_wire.len())
+                }
+                None => true,
+            };
+            if better {
+                best = Some((encoded.wire, id, certificate, usd, units));
+            }
         }
-
-        self.try_offload(raw)
+        (best, refusal)
     }
 
     // Readable inline text codecs ship first: grouped (grep by file, listings by directory) and tree

crates/optimize/src/lock.rs

@@ -22,7 +22,7 @@ pub fn encode(raw: &str) -> Option<String> {
     if records.len() < 3 {
         return None;
     }
-    let table = Nested.try_encode(&Value::Array(records))?.wire;
+    let table = Nested::default().try_encode(&Value::Array(records))?.wire;
     let wire = format!(
         "{HEADER} {} {}\n{preamble}{table}",
         trailing as u8,