Carry Fulu kzg_commitments on data_column_sidecar events

crates/application_boundary/src/lib.rs

@@ -6,7 +6,7 @@ use silver_beacon_state_data::{BeaconStateReader, SpecConfig};
 use silver_common::{
     BeaconStateEvent, BlockStage, Enr, GossipTopic, Identify, Keypair, PeerEvent, SilverSpine,
     SyncUpdate, TProducer, TRandomAccess, TRead,
-    column_util::{SidecarIdentity, block_root},
+    column_util::{SidecarIdentity, block_root, kzg_commitments_from_sidecar},
     ssz_view::{SignedBeaconBlockView, StatusView},
 };
 use silver_config::EngineConfig;
@@ -173,11 +173,16 @@ impl ApplicationBoundaryTile {
 }
 
 fn publish_data_column_sidecar(beacon: &mut BeaconApi, sidecar: TRead) {
-    match sidecar.buffer().map(|(bytes, _)| SidecarIdentity::of(bytes)) {
-        Ok(Some(column)) => {
-            beacon.publish_data_column_sidecar(&column.block_root, column.column_index, column.slot)
-        }
-        Ok(None) => tracing::warn!("published sidecar fits no layout data_column_sidecar reads"),
+    match sidecar.buffer() {
+        Ok((bytes, _)) => match SidecarIdentity::of(bytes) {
+            Some(column) => beacon.publish_data_column_sidecar(
+                &column.block_root,
+                column.column_index,
+                column.slot,
+                kzg_commitments_from_sidecar(bytes),
+            ),
+            None => tracing::warn!("published sidecar fits no layout data_column_sidecar reads"),
+        },
         Err(e) => tracing::warn!(?e, "published sidecar unavailable to data_column_sidecar"),
     }
 }

crates/application_boundary/tests/tile.rs

@@ -20,8 +20,9 @@ use silver_common::{
     TCacheProducer, TCacheRead, TProducer,
     column_util::{block_root_from_sidecar, block_root_fulu},
     ssz_view::{
-        BEACON_BLOCK_BODY_FIXED, DATA_COLUMN_SIDECAR_GLOAS_MIN, DATA_COLUMN_SIDECAR_MIN,
-        SIGNED_BEACON_BLOCK_MIN, STATUS_V2_SIZE,
+        BEACON_BLOCK_BODY_FIXED, BYTES_PER_KZG_COMMITMENT, DATA_COLUMN_SIDECAR_GLOAS_MIN,
+        DATA_COLUMN_SIDECAR_MIN, DataColumnSidecarFuluView, SIGNED_BEACON_BLOCK_MIN,
+        STATUS_V2_SIZE,
     },
     test_util::ShmemDir,
 };
@@ -197,17 +198,22 @@ fn block_bytes(slot: u64, byte: u8) -> Vec<u8> {
     block
 }
 
-/// Empty Fulu sidecar for field extraction; consensus validation is outside
-/// this fixture.
+/// Contains two distinct commitments for field extraction, without
+/// consensus-valid column or proof lists.
 fn fulu_sidecar_bytes(slot: u64, byte: u8, index: u64) -> Vec<u8> {
+    let commitments =
+        [[byte; BYTES_PER_KZG_COMMITMENT], [!byte; BYTES_PER_KZG_COMMITMENT]].concat();
     let mut sidecar = vec![0u8; DATA_COLUMN_SIDECAR_MIN];
     sidecar[0..8].copy_from_slice(&index.to_le_bytes());
-    for offset in [8, 12, 16] {
-        sidecar[offset..offset + 4]
-            .copy_from_slice(&(DATA_COLUMN_SIDECAR_MIN as u32).to_le_bytes());
+    let proofs = DATA_COLUMN_SIDECAR_MIN + commitments.len();
+    for (offset, start) in
+        [(8, DATA_COLUMN_SIDECAR_MIN), (12, DATA_COLUMN_SIDECAR_MIN), (16, proofs)]
+    {
+        sidecar[offset..offset + 4].copy_from_slice(&(start as u32).to_le_bytes());
     }
     sidecar[20..28].copy_from_slice(&slot.to_le_bytes());
     sidecar[68..100].copy_from_slice(&[byte; 32]);
+    sidecar.extend_from_slice(&commitments);
     sidecar
 }
 
@@ -245,9 +251,13 @@ fn block_relay(gossip: &mut TProducer, slot: u64, byte: u8) -> (PeerEvent, SseEv
 
 fn column_relay(gossip: &mut TProducer, slot: u64, byte: u8, index: u64) -> (PeerEvent, SseEvent) {
     let sidecar = fulu_sidecar_bytes(slot, byte, index);
+    let commitments = DataColumnSidecarFuluView::kzg_commitments(&sidecar);
+    assert_eq!(commitments.len(), 2 * BYTES_PER_KZG_COMMITMENT, "the fixture carries two");
     let topic = GossipTopic::DataColumnSidecar(index);
     let event = send_gossip(topic, byte, write_object(gossip, &sidecar));
-    (event, SseEvent::column(slot, &block_root_from_sidecar(&sidecar), index))
+    let expected =
+        SseEvent::column(slot, &block_root_from_sidecar(&sidecar), index, Some(commitments));
+    (event, expected)
 }
 
 fn column_publication(
@@ -261,42 +271,64 @@ fn column_publication(
         topic: GossipTopic::DataColumnSidecar(index),
         ssz: write_object(rpc, &gloas_sidecar_bytes(slot, byte, index)),
     };
-    (event, SseEvent::column(slot, &[byte; 32], index))
+    (event, SseEvent::column(slot, &[byte; 32], index, None))
 }
 
+/// Expected fields are matched individually; additional fields are allowed
+/// unless listed in `absent`.
 #[derive(Clone, Debug)]
 struct SseEvent {
     name: String,
     data: Value,
+    absent: Vec<&'static str>,
 }
 
 impl SseEvent {
     fn block(slot: u64, byte: u8) -> Self {
         Self {
             name: "block".to_owned(),
             data: json!({"slot": slot.to_string(), "block": format!("0x{}", hex::encode([byte; 32]))}),
+            absent: Vec::new(),
         }
     }
 
     fn block_gossip(slot: u64, block_root: &[u8; 32]) -> Self {
         Self {
             name: "block_gossip".to_owned(),
             data: json!({"slot": slot.to_string(), "block": format!("0x{}", hex::encode(block_root))}),
+            absent: Vec::new(),
         }
     }
 
-    fn column(slot: u64, block_root: &[u8; 32], index: u64) -> Self {
-        Self {
-            name: "data_column_sidecar".to_owned(),
-            data: json!({"block_root": format!("0x{}", hex::encode(block_root)), "index": index.to_string(), "slot": slot.to_string()}),
+    fn column(
+        slot: u64,
+        block_root: &[u8; 32],
+        index: u64,
+        kzg_commitments: Option<&[u8]>,
+    ) -> Self {
+        let mut data = json!({"block_root": format!("0x{}", hex::encode(block_root)), "index": index.to_string(), "slot": slot.to_string()});
+        let mut absent = Vec::new();
+        match kzg_commitments {
+            Some(commitments) => {
+                let list: Vec<_> = commitments
+                    .chunks_exact(BYTES_PER_KZG_COMMITMENT)
+                    .map(|commitment| format!("0x{}", hex::encode(commitment)))
+                    .collect();
+                data["kzg_commitments"] = json!(list);
+            }
+            None => absent.push("kzg_commitments"),
         }
+        Self { name: "data_column_sidecar".to_owned(), data, absent }
     }
 
     fn assert_matches(&self, expected: &Self) {
         assert_eq!(self.name, expected.name);
         for (key, value) in expected.data.as_object().unwrap() {
             assert_eq!(self.data.get(key), Some(value), "field {key} in {}", self.name);
         }
+        for key in &expected.absent {
+            assert!(self.data.get(key).is_none(), "field {key} present in {}", self.name);
+        }
     }
 
     fn assert_block(&self, name: &str, slot: u64, byte: u8) {
@@ -332,6 +364,7 @@ impl EventsSubscriber {
                         send.send(SseEvent {
                             name: if name.is_empty() { "message".to_owned() } else { name.clone() },
                             data: serde_json::from_str(&data).expect("event data is JSON"),
+                            absent: Vec::new(),
                         })
                         .unwrap();
                         received += 1;
@@ -702,7 +735,7 @@ fn pool_cap_gates_spine_intake() {
 /// after that intake: a request reaches the EL in the iteration that took it,
 /// not the one after.
 #[test]
-fn an_engine_request_reaches_the_el_in_the_iteration_that_takes_it() {
+fn engine_request_reaches_the_el_in_the_iteration_that_takes_it() {
     let base = ShmemDir::new().unwrap();
     let mut spine = Box::new(SilverSpine::new_with_base_dir(base.path(), None));
     let (mut el, endpoint) = FakeEl::tcp();
@@ -1015,7 +1048,7 @@ fn serves_concurrent_clients_with_no_engine_registered() {
 }
 
 #[test]
-fn an_applied_block_on_the_spine_reaches_an_events_subscriber() {
+fn applied_block_on_the_spine_reaches_an_events_subscriber() {
     let base = ShmemDir::new().unwrap();
     let mut spine = Box::new(SilverSpine::new_with_base_dir(base.path(), None));
     let mut tile = boundary_tile(&Bind::parse("127.0.0.1:0"), no_el(), [
@@ -1137,7 +1170,7 @@ fn subscriptions_select_their_topics_and_preserve_repeated_requests() {
 }
 
 #[test]
-fn a_late_subscriber_receives_only_relay_requests_published_after_it() {
+fn late_subscriber_receives_only_relay_requests_published_after_it() {
     let base = ShmemDir::new().unwrap();
     let mut spine = Box::new(SilverSpine::new_with_base_dir(base.path(), None));
     let (mut tile, mut gossip, mut rpc) =
@@ -1180,7 +1213,7 @@ fn a_late_subscriber_receives_only_relay_requests_published_after_it() {
 }
 
 #[test]
-fn an_idle_boundary_lets_the_object_rings_evict_its_consumers() {
+fn idle_boundary_lets_the_object_rings_evict_its_consumers() {
     let base = ShmemDir::new().unwrap();
     let mut spine = Box::new(SilverSpine::new_with_base_dir(base.path(), None));
     let (mut tile, mut gossip, mut rpc) =

crates/beacon_api/src/json.rs

@@ -4,6 +4,7 @@
 //! `serde_json` is reserved for bodies built once at startup (`identity.rs`).
 
 use silver_beacon_state_data::{B256, Checkpoint, Fork, Version};
+use silver_common::ssz_view::BYTES_PER_KZG_COMMITMENT;
 
 use crate::events::HeadEvent;
 
@@ -287,11 +288,13 @@ impl Json<'_> {
         self.end_object();
     }
 
+    /// `None` omits `kzg_commitments` rather than emitting an empty array.
     pub(crate) fn data_column_sidecar_event(
         &mut self,
         block_root: &[u8; 32],
         column_index: u64,
         slot: u64,
+        kzg_commitments: Option<&[u8]>,
     ) {
         self.begin_object();
         self.key("block_root");
@@ -300,6 +303,15 @@ impl Json<'_> {
         self.quoted_u64(column_index);
         self.key("slot");
         self.quoted_u64(slot);
+        if let Some(commitments) = kzg_commitments {
+            debug_assert!(commitments.len().is_multiple_of(BYTES_PER_KZG_COMMITMENT));
+            self.key("kzg_commitments");
+            self.begin_array();
+            for commitment in commitments.chunks_exact(BYTES_PER_KZG_COMMITMENT) {
+                self.hex(commitment);
+            }
+            self.end_array();
+        }
         self.end_object();
     }
 
@@ -642,11 +654,25 @@ mod tests {
     }
 
     #[test]
-    fn data_column_sidecar_event_carries_the_root_index_and_slot() {
-        let body = write(|json| json.data_column_sidecar_event(&[0x9a; 32], 3, 10));
+    fn data_column_sidecar_event_carries_the_root_index_slot_and_commitments() {
+        let commitments =
+            [[0x11; BYTES_PER_KZG_COMMITMENT], [0x22; BYTES_PER_KZG_COMMITMENT]].concat();
+        let body = write(|json| {
+            json.data_column_sidecar_event(&[0x9a; 32], 3, 10, Some(&commitments));
+        });
         let parsed: serde_json::Value = serde_json::from_str(&body).expect("valid JSON");
         assert_eq!(parsed["block_root"], format!("0x{}", hex::encode([0x9a; 32])));
         assert_eq!(parsed["index"], "3");
         assert_eq!(parsed["slot"], "10");
+        let expected = [format!("0x{}", "11".repeat(48)), format!("0x{}", "22".repeat(48))];
+        assert_eq!(parsed["kzg_commitments"], serde_json::json!(expected));
+    }
+
+    #[test]
+    fn a_column_event_without_commitments_omits_the_field() {
+        let body = write(|json| json.data_column_sidecar_event(&[0x9a; 32], 3, 10, None));
+        let parsed: serde_json::Value = serde_json::from_str(&body).expect("valid JSON");
+        assert_eq!(parsed["slot"], "10");
+        assert!(parsed.get("kzg_commitments").is_none());
     }
 }

crates/beacon_api/src/server.rs

@@ -411,9 +411,15 @@ impl BeaconApi {
         block_root: &[u8; 32],
         column_index: u64,
         slot: u64,
+        kzg_commitments: Option<&[u8]>,
     ) {
         let mut data = Vec::new();
-        Json::new(&mut data).data_column_sidecar_event(block_root, column_index, slot);
+        Json::new(&mut data).data_column_sidecar_event(
+            block_root,
+            column_index,
+            slot,
+            kzg_commitments,
+        );
         self.publish(Channel::DataColumnSidecar, "data_column_sidecar", &data);
     }
 

crates/columns/src/tile/tests/publication.rs

@@ -175,6 +175,38 @@ fn column_publications_name_the_sidecar_for_gossip_and_following_rpc() {
     }
 }
 
+#[test]
+fn fulu_columns_are_not_accepted_at_or_after_gloas_activation() {
+    let blob = BlockBlob::counting();
+    let spec = SpecConfig { fulu_fork_epoch: 0, gloas_fork_epoch: 1, ..SpecConfig::mainnet() };
+    let activation_slot = spec.gloas_fork_epoch * SLOTS_PER_EPOCH;
+    for slot in [activation_slot - 1, activation_slot, activation_slot + 1] {
+        for source in [ColumnSource::Gossip, ColumnSource::Rpc] {
+            let block = block_around(slot, &fulu_body(&blob.commitment));
+            let block_root = util::block_root_fulu(&block);
+            let sidecar = blob.fulu_sidecar(3, &block);
+            let mut rig = Rig::with_spec(CUSTODY_COLUMNS, spec.clone());
+            rig.follow(*SignedBeaconBlockView::parent_root(&block));
+            // The empty registry cannot verify signatures. Cache the signature to
+            // isolate the layout gate while exercising shape, inclusion and KZG checks.
+            rig.tile
+                .tracker
+                .set_signature(block_root, *DataColumnSidecarFuluView::block_signature(&sidecar));
+
+            rig.receive_column(source, 3, &sidecar);
+            rig.turn();
+            let out = rig.drain();
+            if slot < activation_slot {
+                assert!(out.persisted(block_root, 3), "{source:?} at slot {slot}");
+                assert!(!out.publications.is_empty(), "{source:?} at slot {slot}");
+            } else {
+                assert!(out.receipts.is_empty(), "{source:?} at slot {slot}");
+                assert!(out.publications.is_empty(), "{source:?} at slot {slot}");
+            }
+        }
+    }
+}
+
 #[test]
 fn fulu_column_publication_requires_a_resolved_proposer() {
     let blob = BlockBlob::counting();

crates/columns/src/validate.rs

@@ -235,6 +235,11 @@ impl ColumnValidator {
             return ColumnOutcome::Reject { block_root, slot, bitmask: column_bitmask };
         }
 
+        if self.spec.is_gloas_at_slot(slot) {
+            tracing::warn!(?stream_id, slot, "Fulu sidecar at or after Gloas activation");
+            return ColumnOutcome::Reject { block_root, slot, bitmask: column_bitmask };
+        }
+
         if tracker.has_any(&block_root, column_bitmask) {
             return ColumnOutcome::AlreadyHeld { block_root, column_index, slot };
         }

crates/common/src/column_util.rs

@@ -112,6 +112,15 @@ impl SidecarIdentity {
     }
 }
 
+/// Returns `None` unless the length and column offset identify Fulu.
+/// Borrows the commitment bytes without validating the remaining SSZ offsets.
+pub fn kzg_commitments_from_sidecar(sidecar: &[u8]) -> Option<&[u8]> {
+    match SidecarLayout::of(sidecar)? {
+        SidecarLayout::Fulu => Some(DataColumnSidecarFuluView::kzg_commitments(sidecar)),
+        SidecarLayout::Gloas => None,
+    }
+}
+
 fn check_sidecar_shape(column: &[u8], commits: &[u8], proofs: &[u8], max_blobs: usize) -> bool {
     if !column.len().is_multiple_of(BYTES_PER_CELL) ||
         !commits.len().is_multiple_of(BYTES_PER_KZG_COMMITMENT) ||
@@ -475,6 +484,22 @@ mod tests {
         buf
     }
 
+    #[test]
+    fn kzg_commitments_from_sidecar_reads_fulu_and_returns_none_for_other_layouts() {
+        let commitments =
+            [[0x42; BYTES_PER_KZG_COMMITMENT], [0xa7; BYTES_PER_KZG_COMMITMENT]].concat();
+        let mut sidecar = synth_sidecar(3, 2, 2, 2);
+        let start = DATA_COLUMN_SIDECAR_MIN + 2 * BYTES_PER_CELL;
+        sidecar[start..start + commitments.len()].copy_from_slice(&commitments);
+
+        assert_eq!(kzg_commitments_from_sidecar(&sidecar), Some(commitments.as_slice()));
+        assert_eq!(kzg_commitments_from_sidecar(&synth_gloas_sidecar(3, 2, 2)), None);
+        assert_eq!(kzg_commitments_from_sidecar(&sidecar[..DATA_COLUMN_SIDECAR_MIN - 1]), None);
+
+        sidecar[8..12].copy_from_slice(&0u32.to_le_bytes());
+        assert_eq!(kzg_commitments_from_sidecar(&sidecar), None);
+    }
+
     /// Every sidecar carries at least one blob; the active schedule is the
     /// upper bound, not the SSZ list limit.
     #[test]