diff --git a/crates/application_boundary/src/lib.rs b/crates/application_boundary/src/lib.rs index c1901735..a2a55b9e 100644 --- a/crates/application_boundary/src/lib.rs +++ b/crates/application_boundary/src/lib.rs @@ -6,7 +6,7 @@ use silver_beacon_state_data::{BeaconStateReader, SpecConfig}; use silver_common::{ BeaconStateEvent, BlockStage, Enr, GossipTopic, Identify, Keypair, PeerEvent, SilverSpine, SyncUpdate, TProducer, TRandomAccess, TRead, - column_util::{SidecarIdentity, block_root}, + column_util::{SidecarIdentity, block_root, kzg_commitments_from_sidecar}, ssz_view::{SignedBeaconBlockView, StatusView}, }; use silver_config::EngineConfig; @@ -173,11 +173,16 @@ impl ApplicationBoundaryTile { } fn publish_data_column_sidecar(beacon: &mut BeaconApi, sidecar: TRead) { - match sidecar.buffer().map(|(bytes, _)| SidecarIdentity::of(bytes)) { - Ok(Some(column)) => { - beacon.publish_data_column_sidecar(&column.block_root, column.column_index, column.slot) - } - Ok(None) => tracing::warn!("published sidecar fits no layout data_column_sidecar reads"), + match sidecar.buffer() { + Ok((bytes, _)) => match SidecarIdentity::of(bytes) { + Some(column) => beacon.publish_data_column_sidecar( + &column.block_root, + column.column_index, + column.slot, + kzg_commitments_from_sidecar(bytes), + ), + None => tracing::warn!("published sidecar fits no layout data_column_sidecar reads"), + }, Err(e) => tracing::warn!(?e, "published sidecar unavailable to data_column_sidecar"), } } diff --git a/crates/application_boundary/tests/tile.rs b/crates/application_boundary/tests/tile.rs index e4100345..14df2300 100644 --- a/crates/application_boundary/tests/tile.rs +++ b/crates/application_boundary/tests/tile.rs @@ -20,8 +20,9 @@ use silver_common::{ TCacheProducer, TCacheRead, TProducer, column_util::{block_root_from_sidecar, block_root_fulu}, ssz_view::{ - BEACON_BLOCK_BODY_FIXED, DATA_COLUMN_SIDECAR_GLOAS_MIN, DATA_COLUMN_SIDECAR_MIN, - SIGNED_BEACON_BLOCK_MIN, STATUS_V2_SIZE, + BEACON_BLOCK_BODY_FIXED, BYTES_PER_KZG_COMMITMENT, DATA_COLUMN_SIDECAR_GLOAS_MIN, + DATA_COLUMN_SIDECAR_MIN, DataColumnSidecarFuluView, SIGNED_BEACON_BLOCK_MIN, + STATUS_V2_SIZE, }, test_util::ShmemDir, }; @@ -197,17 +198,22 @@ fn block_bytes(slot: u64, byte: u8) -> Vec { block } -/// Empty Fulu sidecar for field extraction; consensus validation is outside -/// this fixture. +/// Contains two distinct commitments for field extraction, without +/// consensus-valid column or proof lists. fn fulu_sidecar_bytes(slot: u64, byte: u8, index: u64) -> Vec { + let commitments = + [[byte; BYTES_PER_KZG_COMMITMENT], [!byte; BYTES_PER_KZG_COMMITMENT]].concat(); let mut sidecar = vec![0u8; DATA_COLUMN_SIDECAR_MIN]; sidecar[0..8].copy_from_slice(&index.to_le_bytes()); - for offset in [8, 12, 16] { - sidecar[offset..offset + 4] - .copy_from_slice(&(DATA_COLUMN_SIDECAR_MIN as u32).to_le_bytes()); + let proofs = DATA_COLUMN_SIDECAR_MIN + commitments.len(); + for (offset, start) in + [(8, DATA_COLUMN_SIDECAR_MIN), (12, DATA_COLUMN_SIDECAR_MIN), (16, proofs)] + { + sidecar[offset..offset + 4].copy_from_slice(&(start as u32).to_le_bytes()); } sidecar[20..28].copy_from_slice(&slot.to_le_bytes()); sidecar[68..100].copy_from_slice(&[byte; 32]); + sidecar.extend_from_slice(&commitments); sidecar } @@ -245,9 +251,13 @@ fn block_relay(gossip: &mut TProducer, slot: u64, byte: u8) -> (PeerEvent, SseEv fn column_relay(gossip: &mut TProducer, slot: u64, byte: u8, index: u64) -> (PeerEvent, SseEvent) { let sidecar = fulu_sidecar_bytes(slot, byte, index); + let commitments = DataColumnSidecarFuluView::kzg_commitments(&sidecar); + assert_eq!(commitments.len(), 2 * BYTES_PER_KZG_COMMITMENT, "the fixture carries two"); let topic = GossipTopic::DataColumnSidecar(index); let event = send_gossip(topic, byte, write_object(gossip, &sidecar)); - (event, SseEvent::column(slot, &block_root_from_sidecar(&sidecar), index)) + let expected = + SseEvent::column(slot, &block_root_from_sidecar(&sidecar), index, Some(commitments)); + (event, expected) } fn column_publication( @@ -261,13 +271,16 @@ fn column_publication( topic: GossipTopic::DataColumnSidecar(index), ssz: write_object(rpc, &gloas_sidecar_bytes(slot, byte, index)), }; - (event, SseEvent::column(slot, &[byte; 32], index)) + (event, SseEvent::column(slot, &[byte; 32], index, None)) } +/// Expected fields are matched individually; additional fields are allowed +/// unless listed in `absent`. #[derive(Clone, Debug)] struct SseEvent { name: String, data: Value, + absent: Vec<&'static str>, } impl SseEvent { @@ -275,6 +288,7 @@ impl SseEvent { Self { name: "block".to_owned(), data: json!({"slot": slot.to_string(), "block": format!("0x{}", hex::encode([byte; 32]))}), + absent: Vec::new(), } } @@ -282,14 +296,29 @@ impl SseEvent { Self { name: "block_gossip".to_owned(), data: json!({"slot": slot.to_string(), "block": format!("0x{}", hex::encode(block_root))}), + absent: Vec::new(), } } - fn column(slot: u64, block_root: &[u8; 32], index: u64) -> Self { - Self { - name: "data_column_sidecar".to_owned(), - data: json!({"block_root": format!("0x{}", hex::encode(block_root)), "index": index.to_string(), "slot": slot.to_string()}), + fn column( + slot: u64, + block_root: &[u8; 32], + index: u64, + kzg_commitments: Option<&[u8]>, + ) -> Self { + let mut data = json!({"block_root": format!("0x{}", hex::encode(block_root)), "index": index.to_string(), "slot": slot.to_string()}); + let mut absent = Vec::new(); + match kzg_commitments { + Some(commitments) => { + let list: Vec<_> = commitments + .chunks_exact(BYTES_PER_KZG_COMMITMENT) + .map(|commitment| format!("0x{}", hex::encode(commitment))) + .collect(); + data["kzg_commitments"] = json!(list); + } + None => absent.push("kzg_commitments"), } + Self { name: "data_column_sidecar".to_owned(), data, absent } } fn assert_matches(&self, expected: &Self) { @@ -297,6 +326,9 @@ impl SseEvent { for (key, value) in expected.data.as_object().unwrap() { assert_eq!(self.data.get(key), Some(value), "field {key} in {}", self.name); } + for key in &expected.absent { + assert!(self.data.get(key).is_none(), "field {key} present in {}", self.name); + } } fn assert_block(&self, name: &str, slot: u64, byte: u8) { @@ -332,6 +364,7 @@ impl EventsSubscriber { send.send(SseEvent { name: if name.is_empty() { "message".to_owned() } else { name.clone() }, data: serde_json::from_str(&data).expect("event data is JSON"), + absent: Vec::new(), }) .unwrap(); received += 1; @@ -702,7 +735,7 @@ fn pool_cap_gates_spine_intake() { /// after that intake: a request reaches the EL in the iteration that took it, /// not the one after. #[test] -fn an_engine_request_reaches_the_el_in_the_iteration_that_takes_it() { +fn engine_request_reaches_the_el_in_the_iteration_that_takes_it() { let base = ShmemDir::new().unwrap(); let mut spine = Box::new(SilverSpine::new_with_base_dir(base.path(), None)); let (mut el, endpoint) = FakeEl::tcp(); @@ -1015,7 +1048,7 @@ fn serves_concurrent_clients_with_no_engine_registered() { } #[test] -fn an_applied_block_on_the_spine_reaches_an_events_subscriber() { +fn applied_block_on_the_spine_reaches_an_events_subscriber() { let base = ShmemDir::new().unwrap(); let mut spine = Box::new(SilverSpine::new_with_base_dir(base.path(), None)); let mut tile = boundary_tile(&Bind::parse("127.0.0.1:0"), no_el(), [ @@ -1137,7 +1170,7 @@ fn subscriptions_select_their_topics_and_preserve_repeated_requests() { } #[test] -fn a_late_subscriber_receives_only_relay_requests_published_after_it() { +fn late_subscriber_receives_only_relay_requests_published_after_it() { let base = ShmemDir::new().unwrap(); let mut spine = Box::new(SilverSpine::new_with_base_dir(base.path(), None)); let (mut tile, mut gossip, mut rpc) = @@ -1180,7 +1213,7 @@ fn a_late_subscriber_receives_only_relay_requests_published_after_it() { } #[test] -fn an_idle_boundary_lets_the_object_rings_evict_its_consumers() { +fn idle_boundary_lets_the_object_rings_evict_its_consumers() { let base = ShmemDir::new().unwrap(); let mut spine = Box::new(SilverSpine::new_with_base_dir(base.path(), None)); let (mut tile, mut gossip, mut rpc) = diff --git a/crates/beacon_api/src/json.rs b/crates/beacon_api/src/json.rs index e2b43ffd..5ba9f28f 100644 --- a/crates/beacon_api/src/json.rs +++ b/crates/beacon_api/src/json.rs @@ -4,6 +4,7 @@ //! `serde_json` is reserved for bodies built once at startup (`identity.rs`). use silver_beacon_state_data::{B256, Checkpoint, Fork, Version}; +use silver_common::ssz_view::BYTES_PER_KZG_COMMITMENT; use crate::events::HeadEvent; @@ -287,11 +288,13 @@ impl Json<'_> { self.end_object(); } + /// `None` omits `kzg_commitments` rather than emitting an empty array. pub(crate) fn data_column_sidecar_event( &mut self, block_root: &[u8; 32], column_index: u64, slot: u64, + kzg_commitments: Option<&[u8]>, ) { self.begin_object(); self.key("block_root"); @@ -300,6 +303,15 @@ impl Json<'_> { self.quoted_u64(column_index); self.key("slot"); self.quoted_u64(slot); + if let Some(commitments) = kzg_commitments { + debug_assert!(commitments.len().is_multiple_of(BYTES_PER_KZG_COMMITMENT)); + self.key("kzg_commitments"); + self.begin_array(); + for commitment in commitments.chunks_exact(BYTES_PER_KZG_COMMITMENT) { + self.hex(commitment); + } + self.end_array(); + } self.end_object(); } @@ -642,11 +654,25 @@ mod tests { } #[test] - fn data_column_sidecar_event_carries_the_root_index_and_slot() { - let body = write(|json| json.data_column_sidecar_event(&[0x9a; 32], 3, 10)); + fn data_column_sidecar_event_carries_the_root_index_slot_and_commitments() { + let commitments = + [[0x11; BYTES_PER_KZG_COMMITMENT], [0x22; BYTES_PER_KZG_COMMITMENT]].concat(); + let body = write(|json| { + json.data_column_sidecar_event(&[0x9a; 32], 3, 10, Some(&commitments)); + }); let parsed: serde_json::Value = serde_json::from_str(&body).expect("valid JSON"); assert_eq!(parsed["block_root"], format!("0x{}", hex::encode([0x9a; 32]))); assert_eq!(parsed["index"], "3"); assert_eq!(parsed["slot"], "10"); + let expected = [format!("0x{}", "11".repeat(48)), format!("0x{}", "22".repeat(48))]; + assert_eq!(parsed["kzg_commitments"], serde_json::json!(expected)); + } + + #[test] + fn a_column_event_without_commitments_omits_the_field() { + let body = write(|json| json.data_column_sidecar_event(&[0x9a; 32], 3, 10, None)); + let parsed: serde_json::Value = serde_json::from_str(&body).expect("valid JSON"); + assert_eq!(parsed["slot"], "10"); + assert!(parsed.get("kzg_commitments").is_none()); } } diff --git a/crates/beacon_api/src/server.rs b/crates/beacon_api/src/server.rs index ca9c7520..536fad44 100644 --- a/crates/beacon_api/src/server.rs +++ b/crates/beacon_api/src/server.rs @@ -411,9 +411,15 @@ impl BeaconApi { block_root: &[u8; 32], column_index: u64, slot: u64, + kzg_commitments: Option<&[u8]>, ) { let mut data = Vec::new(); - Json::new(&mut data).data_column_sidecar_event(block_root, column_index, slot); + Json::new(&mut data).data_column_sidecar_event( + block_root, + column_index, + slot, + kzg_commitments, + ); self.publish(Channel::DataColumnSidecar, "data_column_sidecar", &data); } diff --git a/crates/columns/src/tile/tests/publication.rs b/crates/columns/src/tile/tests/publication.rs index 8bf30cb4..384f30cd 100644 --- a/crates/columns/src/tile/tests/publication.rs +++ b/crates/columns/src/tile/tests/publication.rs @@ -175,6 +175,38 @@ fn column_publications_name_the_sidecar_for_gossip_and_following_rpc() { } } +#[test] +fn fulu_columns_are_not_accepted_at_or_after_gloas_activation() { + let blob = BlockBlob::counting(); + let spec = SpecConfig { fulu_fork_epoch: 0, gloas_fork_epoch: 1, ..SpecConfig::mainnet() }; + let activation_slot = spec.gloas_fork_epoch * SLOTS_PER_EPOCH; + for slot in [activation_slot - 1, activation_slot, activation_slot + 1] { + for source in [ColumnSource::Gossip, ColumnSource::Rpc] { + let block = block_around(slot, &fulu_body(&blob.commitment)); + let block_root = util::block_root_fulu(&block); + let sidecar = blob.fulu_sidecar(3, &block); + let mut rig = Rig::with_spec(CUSTODY_COLUMNS, spec.clone()); + rig.follow(*SignedBeaconBlockView::parent_root(&block)); + // The empty registry cannot verify signatures. Cache the signature to + // isolate the layout gate while exercising shape, inclusion and KZG checks. + rig.tile + .tracker + .set_signature(block_root, *DataColumnSidecarFuluView::block_signature(&sidecar)); + + rig.receive_column(source, 3, &sidecar); + rig.turn(); + let out = rig.drain(); + if slot < activation_slot { + assert!(out.persisted(block_root, 3), "{source:?} at slot {slot}"); + assert!(!out.publications.is_empty(), "{source:?} at slot {slot}"); + } else { + assert!(out.receipts.is_empty(), "{source:?} at slot {slot}"); + assert!(out.publications.is_empty(), "{source:?} at slot {slot}"); + } + } + } +} + #[test] fn fulu_column_publication_requires_a_resolved_proposer() { let blob = BlockBlob::counting(); diff --git a/crates/columns/src/validate.rs b/crates/columns/src/validate.rs index f4fbd056..b75e7567 100644 --- a/crates/columns/src/validate.rs +++ b/crates/columns/src/validate.rs @@ -235,6 +235,11 @@ impl ColumnValidator { return ColumnOutcome::Reject { block_root, slot, bitmask: column_bitmask }; } + if self.spec.is_gloas_at_slot(slot) { + tracing::warn!(?stream_id, slot, "Fulu sidecar at or after Gloas activation"); + return ColumnOutcome::Reject { block_root, slot, bitmask: column_bitmask }; + } + if tracker.has_any(&block_root, column_bitmask) { return ColumnOutcome::AlreadyHeld { block_root, column_index, slot }; } diff --git a/crates/common/src/column_util.rs b/crates/common/src/column_util.rs index 6e793216..288817c6 100644 --- a/crates/common/src/column_util.rs +++ b/crates/common/src/column_util.rs @@ -112,6 +112,15 @@ impl SidecarIdentity { } } +/// Returns `None` unless the length and column offset identify Fulu. +/// Borrows the commitment bytes without validating the remaining SSZ offsets. +pub fn kzg_commitments_from_sidecar(sidecar: &[u8]) -> Option<&[u8]> { + match SidecarLayout::of(sidecar)? { + SidecarLayout::Fulu => Some(DataColumnSidecarFuluView::kzg_commitments(sidecar)), + SidecarLayout::Gloas => None, + } +} + fn check_sidecar_shape(column: &[u8], commits: &[u8], proofs: &[u8], max_blobs: usize) -> bool { if !column.len().is_multiple_of(BYTES_PER_CELL) || !commits.len().is_multiple_of(BYTES_PER_KZG_COMMITMENT) || @@ -475,6 +484,22 @@ mod tests { buf } + #[test] + fn kzg_commitments_from_sidecar_reads_fulu_and_returns_none_for_other_layouts() { + let commitments = + [[0x42; BYTES_PER_KZG_COMMITMENT], [0xa7; BYTES_PER_KZG_COMMITMENT]].concat(); + let mut sidecar = synth_sidecar(3, 2, 2, 2); + let start = DATA_COLUMN_SIDECAR_MIN + 2 * BYTES_PER_CELL; + sidecar[start..start + commitments.len()].copy_from_slice(&commitments); + + assert_eq!(kzg_commitments_from_sidecar(&sidecar), Some(commitments.as_slice())); + assert_eq!(kzg_commitments_from_sidecar(&synth_gloas_sidecar(3, 2, 2)), None); + assert_eq!(kzg_commitments_from_sidecar(&sidecar[..DATA_COLUMN_SIDECAR_MIN - 1]), None); + + sidecar[8..12].copy_from_slice(&0u32.to_le_bytes()); + assert_eq!(kzg_commitments_from_sidecar(&sidecar), None); + } + /// Every sidecar carries at least one blob; the active schedule is the /// upper bound, not the SSZ list limit. #[test] diff --git a/crates/e2e/tests/checkpoint_load.rs b/crates/e2e/tests/checkpoint_load.rs index 5e4de77e..6b84920b 100644 --- a/crates/e2e/tests/checkpoint_load.rs +++ b/crates/e2e/tests/checkpoint_load.rs @@ -144,10 +144,17 @@ fn finalized_state_loads() { return; }; - // Ticker against the SSZ's genesis_time so `current_slot()` returns the - // real mainnet slot — otherwise `precheck_block` ignores blocks as future. + let mut blocks = dir.read_sorted_next_blocks(); + let state = BeaconState::from_checkpoint(&ssz, &SpecConfig::mainnet(), &[]) + .unwrap_or_else(|e| panic!("decompose checkpoint: {e}")); + let checkpoint_slot = state.slot_states.finalized_view().slot_number(); + let last_slot = blocks.last().map_or(checkpoint_slot, |(slot, _)| *slot); + + // Keep the checkpoint recent and every fixture block in the past, + // independent of the date on which the test runs. let genesis_time = u64::from_le_bytes(ssz[0..8].try_into().unwrap()); - let ticker = SlotTicker::new(genesis_time, Duration::from_secs(12), Duration::from_secs(4)); + let mut ticker = SlotTicker::new(genesis_time, Duration::from_secs(12), Duration::from_secs(4)); + ticker.set_current_slot(last_slot + 1); let gossip_p = TCache::producer("gossip_in", 1 << 20); let rpc_p = TCache::producer("rpc_in", 1 << 20); let engine_resp_p = TCache::producer("engine_resp", 1 << 24); @@ -157,8 +164,6 @@ fn finalized_state_loads() { let engine_resp_c = engine_resp_p.cache_ref().random_access("test", false).unwrap(); let replay_c = replay_p.cache_ref().random_access("test", false).unwrap(); - let state = BeaconState::from_checkpoint(&ssz, &SpecConfig::mainnet(), &[]) - .unwrap_or_else(|e| panic!("decompose checkpoint: {e}")); let mut tile = BeaconStateTile::new( ticker, Arc::new(silver_beacon_state_data::SpecConfig::mainnet()), @@ -217,8 +222,6 @@ fn finalized_state_loads() { ); } - // Collect and sort `next_block_.ssz` fixtures. - let mut blocks = dir.read_sorted_next_blocks(); if blocks.is_empty() { eprintln!( "skipping next-block apply: no next_block_*.ssz fixtures in {}", diff --git a/docs/adr/0004-sync-materialized-api.md b/docs/adr/0004-sync-materialized-api.md index d9e9c75d..240c0060 100644 --- a/docs/adr/0004-sync-materialized-api.md +++ b/docs/adr/0004-sync-materialized-api.md @@ -198,3 +198,10 @@ At 64 subscriptions, the pending-output allowance totals 32 MiB. Each subscription reserves its buffer at construction and retains the allocation until it closes. A full drain reuses the buffer from its beginning without releasing it. Partial drains can advance through the entire allocation. + +Amended 2026-09-14: Fulu `data_column_sidecar` events include the sidecar's +ordered commitment list as `kzg_commitments`. Each 48-byte commitment is +encoded as a 0x-prefixed string of 96 hexadecimal digits. This follows the +[beacon-APIs v4.0.0 event format](https://github.com/ethereum/beacon-APIs/blob/v4.0.0/apis/eventstream/index.yaml). +Gloas sidecars carry no commitments, so their events omit the field, matching +[v5.0.0-alpha.2](https://github.com/ethereum/beacon-APIs/blob/v5.0.0-alpha.2/apis/eventstream/index.yaml).