From 2192496b5c1b76040e63fe71601a783d368736d9 Mon Sep 17 00:00:00 2001 From: vladimir-ea Date: Fri, 11 Sep 2026 14:48:32 +0100 Subject: [PATCH 1/5] partial payloads: tcache range acquire and sub-reservations Producer tracks min_allocation so uncommitted reservations block ring reuse. MultiProducer serialises allocation behind a claim, initialising slot headers before the head advances. Consumers gain pinned sub-slot ranges, fixed retention boundaries, per-part sub-reservations with claim/validate/accept states, and gossip frame descriptors over pinned segments. Gossip decompression computes the message id before the offset commits the reservation. Co-Authored-By: Claude Fable 5 --- crates/common/src/spine/tcache.rs | 2 + .../common/src/spine/tcache/gossip_frame.rs | 299 ++++++++++++ .../src/spine/tcache/gossip_frame/acquired.rs | 141 ++++++ .../src/spine/tcache/gossip_frame/tests.rs | 445 ++++++++++++++++++ 4 files changed, 887 insertions(+) create mode 100644 crates/common/src/spine/tcache/gossip_frame.rs create mode 100644 crates/common/src/spine/tcache/gossip_frame/acquired.rs create mode 100644 crates/common/src/spine/tcache/gossip_frame/tests.rs diff --git a/crates/common/src/spine/tcache.rs b/crates/common/src/spine/tcache.rs index ff148ab3..944c25e7 100644 --- a/crates/common/src/spine/tcache.rs +++ b/crates/common/src/spine/tcache.rs @@ -16,6 +16,7 @@ pub use consumer::{ AcquiredRange, AcquiredRead, AcquiredWithOffset, Consumer, RandomAccessConsumer, TCacheRead, }; use flux::{Timer, timing::Nanos, tracing}; + pub use producer::{MultiProducer, Producer, Reservation, TCacheProducer}; pub use sub_reservation::{ AcquiredSubReservation, PendingSubReservation, SubLayout, SubReservation, SubReservationError, @@ -46,6 +47,7 @@ const fn lag_threshold(len: u32) -> u64 { mod cache_frame; mod consumer; +mod gossip_frame; mod metrics; mod producer; mod sub_reservation; diff --git a/crates/common/src/spine/tcache/gossip_frame.rs b/crates/common/src/spine/tcache/gossip_frame.rs new file mode 100644 index 00000000..ff30daec --- /dev/null +++ b/crates/common/src/spine/tcache/gossip_frame.rs @@ -0,0 +1,299 @@ +use std::{io::Write, ops::Range, time::Instant}; + +use super::{ + AcquiredRange, AcquiredRead, Producer, RandomAccessConsumer, SubReservationRef, TCacheProducer, + TCacheRead, +}; +use crate::MAX_GOSSIP_FRAME_SIZE; + +mod acquired; +pub use acquired::{AcquiredGossipFrame, AcquiredGossipSegment}; + +pub const MAX_GOSSIP_SEGMENTS: usize = 2 * 128 + 8; +const HEADER_BYTES: usize = 16; +const SEGMENT_BYTES: usize = 40; +const MAGIC: [u8; 8] = *b"SGFRAME1"; + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum GossipFrameError { + InvalidDescriptor, + TooLarge, + CacheFull, + Expired, + Stale, +} + +#[derive(Clone, Copy, Debug)] +pub enum GossipSegment { + Framing { + offset: usize, + length: usize, + }, + Gossip { + read: TCacheRead, + offset: usize, + length: usize, + }, + DataColumns { + read: TCacheRead, + offset: usize, + length: usize, + }, + Shared { + reservation: SubReservationRef, + part: usize, + second: bool, + offset: usize, + length: usize, + }, +} + +// Only the builder constructs this handle. Encoded source identities originate +// from typed cache descriptors, never from network input. +// Pointer identities and trusted layout metadata make this an in-process +// format. +#[derive(Clone, Copy, Debug)] +pub struct GossipFrameRef { + descriptor: TCacheRead, + expires: Instant, +} + +impl GossipFrameRef { + pub fn write( + producer: &mut Producer, + expires: Instant, + framing: &[u8], + segments: impl ExactSizeIterator, + ) -> Result { + let count = segments.len(); + if count == 0 || count > MAX_GOSSIP_SEGMENTS || framing.len() > MAX_GOSSIP_FRAME_SIZE { + return Err(GossipFrameError::TooLarge); + } + let framing_start = HEADER_BYTES + count * SEGMENT_BYTES; + let descriptor_len = framing_start + framing.len(); + let cache = producer.cache_ref(); + let mut reservation = + producer.reserve(descriptor_len, false).ok_or(GossipFrameError::CacheFull)?; + let buffer = reservation.buffer().map_err(|_| GossipFrameError::Stale)?; + buffer[..8].copy_from_slice(&MAGIC); + buffer[8..12].copy_from_slice(&(count as u32).to_le_bytes()); + buffer[framing_start..].copy_from_slice(framing); + let mut total = 0usize; + let mut written = 0; + for (index, segment) in segments.enumerate() { + if index >= count { + return Err(GossipFrameError::InvalidDescriptor); + } + let (kind, read, metadata, offset, length) = match segment { + GossipSegment::Framing { offset, length } => { + if offset.checked_add(length).is_none_or(|end| end > framing.len()) { + return Err(GossipFrameError::InvalidDescriptor); + } + (0u64, None, 0, offset, length) + } + GossipSegment::Gossip { read, offset, length } => { + if read.tcache.cache != cache.cache { + return Err(GossipFrameError::InvalidDescriptor); + } + (1, Some(read), 0, offset, length) + } + GossipSegment::DataColumns { read, offset, length } => { + (2, Some(read), 0, offset, length) + } + GossipSegment::Shared { reservation, part, second, offset, length } => { + if part >= 128 { + return Err(GossipFrameError::InvalidDescriptor); + } + let metadata = ((reservation.header_bytes as u64) << 32) | + ((part as u64) << 1) | + u64::from(second); + (3, Some(reservation.read()), metadata, offset, length) + } + }; + if length == 0 || offset > u32::MAX as usize || length > u32::MAX as usize { + return Err(GossipFrameError::InvalidDescriptor); + } + total = total.checked_add(length).ok_or(GossipFrameError::TooLarge)?; + if total > MAX_GOSSIP_FRAME_SIZE { + return Err(GossipFrameError::TooLarge); + } + let start = HEADER_BYTES + index * SEGMENT_BYTES; + let entry = &mut buffer[start..start + SEGMENT_BYTES]; + entry[..8].copy_from_slice(&kind.to_le_bytes()); + entry[8..16] + .copy_from_slice(&read.map_or(0, |r| r.tcache.cache as usize as u64).to_le_bytes()); + entry[16..24].copy_from_slice(&read.map_or(0, |r| r.seq).to_le_bytes()); + entry[24..32].copy_from_slice(&metadata.to_le_bytes()); + entry[32..36].copy_from_slice(&(offset as u32).to_le_bytes()); + entry[36..40].copy_from_slice(&(length as u32).to_le_bytes()); + written += 1; + } + if written != count { + return Err(GossipFrameError::InvalidDescriptor); + } + buffer[12..16].copy_from_slice(&(total as u32).to_le_bytes()); + reservation.flush().map_err(|_| GossipFrameError::Stale)?; + Ok(Self { descriptor: reservation.read(), expires }) + } + + pub fn read(self) -> TCacheRead { + self.descriptor + } + + pub fn acquire( + self, + consumer: &mut RandomAccessConsumer, + now: Instant, + ) -> Result { + if now >= self.expires { + return Err(GossipFrameError::Expired); + } + if !consumer.is_strict() || consumer.cache.cache != self.descriptor.tcache.cache { + return Err(GossipFrameError::InvalidDescriptor); + } + let read = consumer.acquire_strict(self.descriptor).ok_or(GossipFrameError::Stale)?; + let buffer = read.buffer().map_err(|_| GossipFrameError::Stale)?.0; + if buffer.len() < HEADER_BYTES || buffer[..8] != MAGIC { + return Err(GossipFrameError::InvalidDescriptor); + } + let count = u32::from_le_bytes(buffer[8..12].try_into().unwrap()) as usize; + let wire_len = u32::from_le_bytes(buffer[12..16].try_into().unwrap()) as usize; + if count == 0 || count > MAX_GOSSIP_SEGMENTS { + return Err(GossipFrameError::InvalidDescriptor); + } + let framing_start = HEADER_BYTES + count * SEGMENT_BYTES; + if framing_start > buffer.len() || wire_len == 0 || wire_len > MAX_GOSSIP_FRAME_SIZE { + return Err(GossipFrameError::InvalidDescriptor); + } + let descriptor_len = buffer.len(); + let view = GossipFrameView { read, count, wire_len, framing_start, descriptor_len }; + let mut total = 0usize; + for segment in view.segments() { + if segment.length == 0 || + segment.kind > 3 || + (segment.kind == 0 && + segment + .offset + .checked_add(segment.length) + .is_none_or(|end| end > view.descriptor_len() - framing_start)) + { + return Err(GossipFrameError::InvalidDescriptor); + } + total = total.checked_add(segment.length).ok_or(GossipFrameError::TooLarge)?; + } + if total != wire_len { + return Err(GossipFrameError::InvalidDescriptor); + } + Ok(view) + } +} + +#[derive(Debug)] +pub struct GossipFrameView { + read: AcquiredRead, + count: usize, + wire_len: usize, + framing_start: usize, + descriptor_len: usize, +} + +impl GossipFrameView { + pub fn wire_len(&self) -> usize { + self.wire_len + } + + pub fn segment_count(&self) -> usize { + self.count + } + + pub fn descriptor_len(&self) -> usize { + self.descriptor_len + } + + pub fn descriptor_range(&self) -> AcquiredRange { + self.read.with_range(0, self.descriptor_len()).expect("acquired descriptor") + } + + pub fn acquire_segments( + self, + gossip: &mut RandomAccessConsumer, + columns: Option<&mut RandomAccessConsumer>, + ) -> Option { + AcquiredGossipFrame::new(self, gossip, columns) + } + + pub fn segments(&self) -> impl ExactSizeIterator + '_ { + self.read.buffer().expect("acquired descriptor").0[HEADER_BYTES..self.framing_start] + .chunks_exact(SEGMENT_BYTES) + .map(|entry| GossipFrameSegment::decode(entry, self.framing_start)) + } + + fn segment(&self, index: usize) -> GossipFrameSegment { + assert!(index < self.count); + let start = HEADER_BYTES + index * SEGMENT_BYTES; + let buffer = self.read.buffer().expect("acquired descriptor").0; + GossipFrameSegment::decode(&buffer[start..start + SEGMENT_BYTES], self.framing_start) + } +} + +pub struct GossipFrameSegment { + kind: u64, + cache: u64, + seq: u64, + metadata: u64, + offset: usize, + length: usize, + framing_start: usize, +} + +impl GossipFrameSegment { + fn decode(entry: &[u8], framing_start: usize) -> Self { + Self { + kind: u64::from_le_bytes(entry[..8].try_into().unwrap()), + cache: u64::from_le_bytes(entry[8..16].try_into().unwrap()), + seq: u64::from_le_bytes(entry[16..24].try_into().unwrap()), + metadata: u64::from_le_bytes(entry[24..32].try_into().unwrap()), + offset: u32::from_le_bytes(entry[32..36].try_into().unwrap()) as usize, + length: u32::from_le_bytes(entry[36..40].try_into().unwrap()) as usize, + framing_start, + } + } + + pub fn framing_range(&self) -> Option> { + (self.kind == 0).then(|| { + self.framing_start + self.offset..self.framing_start + self.offset + self.length + }) + } + + pub fn acquire( + &self, + gossip: &mut RandomAccessConsumer, + columns: Option<&mut RandomAccessConsumer>, + ) -> Option { + let consumer = match self.kind { + 1 => gossip, + 2 | 3 => columns?, + _ => return None, + }; + if !consumer.is_strict() || + consumer.cache.cache as usize as u64 != self.cache || + !self.seq.is_multiple_of(super::ALIGN as u64) + { + return None; + } + let read = TCacheRead { tcache: consumer.cache, seq: self.seq }; + if self.kind == 3 { + let reference = + SubReservationRef { read, header_bytes: (self.metadata >> 32) as usize }; + let acquired = reference.acquire(consumer).ok()?; + let [first, second] = acquired.ranges(((self.metadata as u32) >> 1) as usize)?; + let range = if self.metadata & 1 == 0 { first } else { second }; + range.slice(self.offset, self.length) + } else { + consumer.acquire_strict(read)?.with_range(self.offset, self.length) + } + } +} + +#[cfg(test)] +mod tests; diff --git a/crates/common/src/spine/tcache/gossip_frame/acquired.rs b/crates/common/src/spine/tcache/gossip_frame/acquired.rs new file mode 100644 index 00000000..2aa2d1e7 --- /dev/null +++ b/crates/common/src/spine/tcache/gossip_frame/acquired.rs @@ -0,0 +1,141 @@ +use std::{ + mem, + ops::{Deref, Range}, + ptr::NonNull, +}; + +use super::{ + AcquiredRange, AcquiredRead, GossipFrameSegment, GossipFrameView, RandomAccessConsumer, + SubReservationRef, TCacheRead, +}; + +pub enum AcquiredGossipSegment { + Framing(Range), + Data(AcquiredRange), +} + +#[derive(Debug)] +pub struct AcquiredGossipFrame { + view: GossipFrameView, + gossip: NonNull, + columns: Option>, + // Every non-framing descriptor in [next, acquired_end) owns one bucket + // count. The descriptor stays pinned until those counts are released. + next: usize, + acquired_end: usize, +} + +// As with AcquiredRead, consumers stay at stable addresses and outlive their +// reads. Acquisition, handoff, and drops remain on the consumer's tile. +unsafe impl Send for AcquiredGossipFrame {} + +impl AcquiredGossipFrame { + pub(super) fn new( + view: GossipFrameView, + gossip: &mut RandomAccessConsumer, + mut columns: Option<&mut RandomAccessConsumer>, + ) -> Option { + let mut frame = Self { + view, + gossip: NonNull::from(&mut *gossip), + columns: columns.as_deref_mut().map(NonNull::from), + next: 0, + acquired_end: 0, + }; + for segment in frame.view.segments() { + if segment.kind != 0 { + let range = segment.acquire(gossip, columns.as_deref_mut())?; + // No fallible work separates forgetting this owner and recording + // its count in the frame's acquired prefix. + mem::forget(range); + } + frame.acquired_end += 1; + } + Some(frame) + } + + pub fn take_next(&mut self) -> Option { + if self.next == self.acquired_end { + return None; + } + let segment = self.view.segment(self.next); + if let Some(range) = segment.framing_range() { + self.next += 1; + return Some(AcquiredGossipSegment::Framing(range)); + } + let mut range = self.take_range(&segment); + while self.next < self.acquired_end { + let next = self.view.segment(self.next); + if next.kind == 0 || + range.read.consumer != self.consumer(&next).as_ptr() || + range.read.seq() != next.seq || + range.offset + range.length != Self::offset(&next, range.read.read) + { + break; + } + let next = self.take_range(&next); + assert!(range.extend_contiguous(&next)); + } + Some(AcquiredGossipSegment::Data(range)) + } + + fn consumer(&self, segment: &GossipFrameSegment) -> NonNull { + if segment.kind == 1 { self.gossip } else { self.columns.expect("acquired column segment") } + } + + fn take_read(&mut self, segment: &GossipFrameSegment) -> AcquiredRead { + let consumer = self.consumer(segment); + // Each call transfers exactly one existing count. Nothing increments + // here, and frame cleanup excludes the transferred descriptor. + let read = AcquiredRead { + consumer: consumer.as_ptr(), + read: TCacheRead { tcache: unsafe { consumer.as_ref() }.cache, seq: segment.seq }, + acquired: self.view.read.acquired, + }; + self.next += 1; + read + } + + fn take_range(&mut self, segment: &GossipFrameSegment) -> AcquiredRange { + let read = self.take_read(segment); + let offset = Self::offset(segment, read.read); + AcquiredRange { read, offset, length: segment.length } + } + + fn offset(segment: &GossipFrameSegment, read: TCacheRead) -> usize { + if segment.kind != 3 { + return segment.offset; + } + let reference = SubReservationRef { read, header_bytes: (segment.metadata >> 32) as usize }; + // Admission validated this part and retains its count. Its immutable + // layout remains valid even after the reservation is closed. + let base = unsafe { + reference.acquired_offset( + ((segment.metadata as u32) >> 1) as usize, + segment.metadata & 1 != 0, + ) + }; + base + segment.offset + } +} + +impl Deref for AcquiredGossipFrame { + type Target = GossipFrameView; + + fn deref(&self) -> &Self::Target { + &self.view + } +} + +impl Drop for AcquiredGossipFrame { + fn drop(&mut self) { + while self.next < self.acquired_end { + let segment = self.view.segment(self.next); + if segment.kind == 0 { + self.next += 1; + } else { + drop(self.take_read(&segment)); + } + } + } +} diff --git a/crates/common/src/spine/tcache/gossip_frame/tests.rs b/crates/common/src/spine/tcache/gossip_frame/tests.rs new file mode 100644 index 00000000..9ed1cad5 --- /dev/null +++ b/crates/common/src/spine/tcache/gossip_frame/tests.rs @@ -0,0 +1,445 @@ +use std::{ + panic::{AssertUnwindSafe, catch_unwind}, + time::Duration, +}; + +use super::*; +use crate::{P2pSend, SubLayout, TCache}; + +fn write(producer: &mut Producer, bytes: &[u8]) -> TCacheRead { + let mut reservation = producer.reserve(bytes.len(), false).unwrap(); + reservation.write_all(bytes).unwrap(); + reservation.flush().unwrap(); + reservation.read() +} + +#[test] +fn copy_handle_round_trips_framing_and_source_ranges() { + fn is_copy() {} + is_copy::(); + is_copy::(); + let mut producer = TCache::producer("", 1 << 18); + let mut consumer = Box::new(producer.cache_ref().strict_random_access("", true).unwrap()); + let source = write(&mut producer, b"0123456789"); + let now = Instant::now(); + let frame = GossipFrameRef::write( + &mut producer, + now + Duration::from_secs(1), + b"ab--cd", + [ + GossipSegment::Framing { offset: 0, length: 2 }, + GossipSegment::Gossip { read: source, offset: 3, length: 4 }, + GossipSegment::Framing { offset: 4, length: 2 }, + ] + .into_iter(), + ) + .unwrap(); + let view = frame.acquire(&mut consumer, now).unwrap(); + assert_eq!(view.wire_len(), 8); + assert_eq!(view.segment_count(), 3); + let descriptor = view.descriptor_range(); + let mut wire = Vec::new(); + for segment in view.segments() { + if let Some(range) = segment.framing_range() { + wire.extend_from_slice(&descriptor.as_ref()[range]); + } else { + wire.extend_from_slice(segment.acquire(&mut consumer, None).unwrap().as_ref()); + } + } + assert_eq!(wire, b"ab3456cd"); + assert!(matches!( + frame.acquire(&mut consumer, now + Duration::from_secs(1)), + Err(GossipFrameError::Expired) + )); +} + +#[test] +fn shared_segments_expose_only_verified_subranges() { + let mut producer = TCache::producer("", 1 << 18); + let mut consumer = Box::new(producer.cache_ref().strict_random_access("", true).unwrap()); + let mut columns = TCache::producer("", 1 << 18); + let mut reader = Box::new(columns.cache_ref().retained_random_access("").unwrap()); + let reference = columns + .sub_reservation(SubLayout { parts: 2, first_len: 4, second_len: 2 }, b"", b"") + .unwrap(); + let pending = columns + .view_sub_reservation(reference) + .unwrap() + .claim(0) + .unwrap() + .write(b"cell", b"pf") + .unwrap(); + let now = Instant::now(); + let frame = GossipFrameRef::write( + &mut producer, + now + Duration::from_secs(1), + b"", + [ + GossipSegment::Shared { + reservation: reference, + part: 0, + second: false, + offset: 1, + length: 2, + }, + GossipSegment::Shared { + reservation: reference, + part: 0, + second: true, + offset: 0, + length: 2, + }, + ] + .into_iter(), + ) + .unwrap(); + let view = frame.acquire(&mut consumer, now).unwrap(); + assert!(view.segments().next().unwrap().acquire(&mut consumer, Some(&mut reader)).is_none()); + pending.acquire(&mut reader).unwrap().accept().unwrap(); + let ranges: Vec<_> = view + .segments() + .map(|segment| segment.acquire(&mut consumer, Some(&mut reader)).unwrap()) + .collect(); + assert_eq!(ranges[0].as_ref(), b"el"); + assert_eq!(ranges[1].as_ref(), b"pf"); + columns.view_sub_reservation(reference).unwrap().close(); + reader.advance_retention(columns.next_seq()); + assert!(view.segments().next().unwrap().acquire(&mut consumer, Some(&mut reader)).is_none()); + assert_eq!(ranges[0].as_ref(), b"el"); +} + +#[test] +fn descriptor_bounds_and_sources_are_checked() { + let mut producer = TCache::producer("", 1 << 18); + let mut consumer = Box::new(producer.cache_ref().strict_random_access("", true).unwrap()); + let mut other = TCache::producer("", 1 << 18); + let mut other_reader = Box::new(other.cache_ref().retained_random_access("").unwrap()); + let other_read = write(&mut other, b"data"); + let expires = Instant::now() + Duration::from_secs(1); + for segment in [ + GossipSegment::Framing { offset: usize::MAX, length: 1 }, + GossipSegment::Framing { offset: 0, length: 0 }, + GossipSegment::Framing { offset: 1, length: 4 }, + GossipSegment::Gossip { read: other_read, offset: 0, length: 4 }, + ] { + assert!( + GossipFrameRef::write(&mut producer, expires, b"data", [segment].into_iter()).is_err() + ); + } + assert!(GossipFrameRef::write(&mut producer, expires, b"", [].into_iter()).is_err()); + assert!( + GossipFrameRef::write( + &mut producer, + expires, + b"x", + std::iter::repeat_n( + GossipSegment::Framing { offset: 0, length: 1 }, + MAX_GOSSIP_SEGMENTS + 1, + ) + ) + .is_err() + ); + let frame = GossipFrameRef::write( + &mut producer, + expires, + b"", + [GossipSegment::DataColumns { read: other_read, offset: 2, length: 4 }].into_iter(), + ) + .unwrap(); + let view = frame.acquire(&mut consumer, Instant::now()).unwrap(); + let segment = view.segments().next().unwrap(); + assert!(segment.acquire(&mut consumer, None).is_none()); + assert!(segment.acquire(&mut consumer, Some(&mut other_reader)).is_none()); + + for bytes in [&b"not a descriptor"[..], &b"SGFRAME1\xff\xff\xff\xff\x01\x00\x00\x00"[..]] { + let malformed = GossipFrameRef { descriptor: write(&mut producer, bytes), expires }; + assert!(matches!( + malformed.acquire(&mut consumer, Instant::now()), + Err(GossipFrameError::InvalidDescriptor) + )); + } +} + +#[test] +fn owned_range_slices_check_bounds() { + let mut producer = TCache::producer("", 1 << 16); + let mut consumer = Box::new(producer.cache_ref().strict_random_access("", true).unwrap()); + let read = write(&mut producer, b"0123456789"); + let acquired = consumer.acquire_strict(read).unwrap(); + let range = acquired.with_range(2, 6).unwrap(); + assert_eq!(range.clone().slice(2, 3).unwrap().as_ref(), b"456"); + assert!(range.clone().slice(6, 0).unwrap().is_empty()); + assert!(range.clone().slice(6, 1).is_none()); + assert!(range.clone().slice(usize::MAX, 1).is_none()); + assert!(range.slice(1, usize::MAX).is_none()); + let mut first = acquired.with_range(0, 3).unwrap(); + let next = acquired.with_range(3, 4).unwrap(); + let gap = acquired.with_range(8, 1).unwrap(); + assert!(!first.extend_contiguous(&gap)); + assert!(first.extend_contiguous(&next)); + drop(next); + assert_eq!(first.as_ref(), b"0123456"); +} + +#[test] +fn failed_acquisition_releases_every_successful_prefix_without_touching_other_reads() { + let mut producer = TCache::producer("", 1 << 18); + let mut consumer = Box::new(producer.cache_ref().strict_random_access("", true).unwrap()); + let mut columns = TCache::producer("", 1 << 18); + let mut reader = Box::new(columns.cache_ref().retained_random_access("").unwrap()); + let gossip = write(&mut producer, b"gossip"); + let column = write(&mut columns, b"column"); + let gossip_guard = consumer.acquire_strict(gossip).unwrap(); + let column_guard = reader.acquire_strict(column).unwrap(); + let shared = columns + .sub_reservation(SubLayout { parts: 2, first_len: 4, second_len: 2 }, b"", b"") + .unwrap(); + columns + .view_sub_reservation(shared) + .unwrap() + .claim(0) + .unwrap() + .write(b"cell", b"pf") + .unwrap() + .acquire(&mut reader) + .unwrap() + .accept() + .unwrap(); + let now = Instant::now(); + let segments = [ + GossipSegment::Framing { offset: 0, length: 1 }, + GossipSegment::Gossip { read: gossip, offset: 1, length: 3 }, + GossipSegment::DataColumns { read: column, offset: 0, length: 4 }, + GossipSegment::Shared { reservation: shared, part: 0, second: false, offset: 0, length: 4 }, + GossipSegment::Shared { reservation: shared, part: 0, second: true, offset: 0, length: 2 }, + GossipSegment::Gossip { read: gossip, offset: 1, length: 3 }, + GossipSegment::Framing { offset: 0, length: 1 }, + ]; + for failed in 0..segments.len() { + let mut descriptors = segments; + descriptors[failed] = GossipSegment::DataColumns { read: column, offset: 100, length: 1 }; + let frame = GossipFrameRef::write( + &mut producer, + now + Duration::from_secs(1), + b"f", + descriptors.into_iter(), + ) + .unwrap(); + assert!( + frame + .acquire(&mut consumer, now) + .unwrap() + .acquire_segments(&mut consumer, Some(&mut reader)) + .is_none() + ); + assert_eq!(consumer.active_count(), 1, "failed descriptor {failed}"); + assert_eq!(reader.active_count(), 1, "failed descriptor {failed}"); + assert_eq!(gossip_guard.buffer().unwrap().0, b"gossip"); + assert_eq!(column_guard.buffer().unwrap().0, b"column"); + } + drop(gossip_guard); + drop(column_guard); + assert_eq!(consumer.active_count(), 0); + assert_eq!(reader.active_count(), 0); +} + +#[test] +fn handoff_transfers_counts_and_frame_drop_releases_only_the_remainder() { + let mut producer = TCache::producer("", 1 << 18); + let mut consumer = Box::new(producer.cache_ref().strict_random_access("", true).unwrap()); + let mut columns = TCache::producer("", 1 << 18); + let mut reader = Box::new(columns.cache_ref().retained_random_access("").unwrap()); + let gossip = write(&mut producer, b"gossip"); + let column = write(&mut columns, b"column"); + let now = Instant::now(); + let reference = GossipFrameRef::write( + &mut producer, + now + Duration::from_secs(1), + b"f", + [ + GossipSegment::Framing { offset: 0, length: 1 }, + GossipSegment::Gossip { read: gossip, offset: 0, length: 6 }, + GossipSegment::DataColumns { read: column, offset: 0, length: 6 }, + GossipSegment::DataColumns { read: column, offset: 0, length: 6 }, + GossipSegment::Framing { offset: 0, length: 1 }, + ] + .into_iter(), + ) + .unwrap(); + let mut frame = reference + .acquire(&mut consumer, now) + .unwrap() + .acquire_segments(&mut consumer, Some(&mut reader)) + .unwrap(); + assert_eq!(consumer.active_count(), 2); + assert_eq!(reader.active_count(), 2); + assert!(matches!(frame.take_next(), Some(AcquiredGossipSegment::Framing(_)))); + let Some(AcquiredGossipSegment::Data(gossip_range)) = frame.take_next() else { panic!() }; + let Some(AcquiredGossipSegment::Data(column_range)) = frame.take_next() else { panic!() }; + assert_eq!(consumer.active_count(), 2); + assert_eq!(reader.active_count(), 2); + reader.advance_retention(columns.next_seq()); + drop(frame); + assert_eq!(consumer.active_count(), 1); + assert_eq!(reader.active_count(), 1); + assert_eq!(gossip_range.as_ref(), b"gossip"); + assert_eq!(column_range.as_ref(), b"column"); + drop(gossip_range); + drop(column_range); + assert_eq!(consumer.active_count(), 0); + assert_eq!(reader.active_count(), 0); +} + +#[test] +fn shared_handoff_survives_closure_without_exposing_unverified_gaps() { + let mut producer = TCache::producer("", 1 << 18); + let mut consumer = Box::new(producer.cache_ref().strict_random_access("", true).unwrap()); + let mut columns = TCache::producer("", 1 << 18); + let mut reader = Box::new(columns.cache_ref().retained_random_access("").unwrap()); + let shared = columns + .sub_reservation(SubLayout { parts: 3, first_len: 4, second_len: 2 }, b"hdr", b"mid") + .unwrap(); + for part in [0, 2] { + columns + .view_sub_reservation(shared) + .unwrap() + .claim(part) + .unwrap() + .write(&[part as u8; 4], &[part as u8 + 10; 2]) + .unwrap() + .acquire(&mut reader) + .unwrap() + .accept() + .unwrap(); + } + let now = Instant::now(); + let reference = GossipFrameRef::write( + &mut producer, + now + Duration::from_secs(1), + b"", + [ + GossipSegment::Shared { + reservation: shared, + part: 0, + second: false, + offset: 1, + length: 3, + }, + GossipSegment::Shared { + reservation: shared, + part: 2, + second: false, + offset: 0, + length: 4, + }, + GossipSegment::Shared { + reservation: shared, + part: 0, + second: true, + offset: 0, + length: 2, + }, + GossipSegment::Shared { + reservation: shared, + part: 2, + second: true, + offset: 1, + length: 1, + }, + ] + .into_iter(), + ) + .unwrap(); + let mut frame = reference + .acquire(&mut consumer, now) + .unwrap() + .acquire_segments(&mut consumer, Some(&mut reader)) + .unwrap(); + assert_eq!(reader.active_count(), 4); + columns.view_sub_reservation(shared).unwrap().close(); + reader.advance_retention(columns.next_seq()); + assert!( + reference + .acquire(&mut consumer, now) + .unwrap() + .acquire_segments(&mut consumer, Some(&mut reader)) + .is_none() + ); + assert_eq!(reader.active_count(), 4); + for (index, expected) in [&[0; 3][..], &[2; 4], &[10; 2], &[12; 1]].into_iter().enumerate() { + let Some(AcquiredGossipSegment::Data(range)) = frame.take_next() else { panic!() }; + assert_eq!(reader.active_count(), 4 - index); + assert_eq!(range.as_ref(), expected); + drop(range); + assert_eq!(reader.active_count(), 3 - index); + } + assert!(frame.take_next().is_none()); + drop(frame); + assert_eq!(consumer.active_count(), 0); + assert_eq!(reader.active_count(), 0); +} + +#[test] +fn coalescing_transfers_one_pin_and_releases_redundant_pins() { + let mut producer = TCache::producer("", 1 << 18); + let mut consumer = Box::new(producer.cache_ref().strict_random_access("", true).unwrap()); + let source = write(&mut producer, b"0123456789"); + let now = Instant::now(); + let reference = GossipFrameRef::write( + &mut producer, + now + Duration::from_secs(1), + b"", + [ + GossipSegment::Gossip { read: source, offset: 1, length: 3 }, + GossipSegment::Gossip { read: source, offset: 4, length: 3 }, + GossipSegment::Gossip { read: source, offset: 7, length: 2 }, + ] + .into_iter(), + ) + .unwrap(); + let mut frame = reference + .acquire(&mut consumer, now) + .unwrap() + .acquire_segments(&mut consumer, None) + .unwrap(); + assert_eq!(consumer.active_count(), 4); + let Some(AcquiredGossipSegment::Data(range)) = frame.take_next() else { panic!() }; + assert_eq!(consumer.active_count(), 2); + assert!(frame.take_next().is_none()); + drop(frame); + assert_eq!(consumer.active_count(), 1); + assert_eq!(range.as_ref(), b"12345678"); + drop(range); + assert_eq!(consumer.active_count(), 0); +} + +#[test] +fn unwinding_releases_untransferred_pins_but_not_the_handed_off_read() { + let mut producer = TCache::producer("", 1 << 18); + let mut consumer = Box::new(producer.cache_ref().strict_random_access("", true).unwrap()); + let source = write(&mut producer, b"data"); + let now = Instant::now(); + let reference = GossipFrameRef::write( + &mut producer, + now + Duration::from_secs(1), + b"", + [GossipSegment::Gossip { read: source, offset: 0, length: 4 }; 3].into_iter(), + ) + .unwrap(); + let mut handed_off = None; + let result = catch_unwind(AssertUnwindSafe(|| { + let mut frame = reference + .acquire(&mut consumer, now) + .unwrap() + .acquire_segments(&mut consumer, None) + .unwrap(); + let Some(AcquiredGossipSegment::Data(range)) = frame.take_next() else { panic!() }; + handed_off = Some(range); + panic!("abort a partially handed-off frame"); + })); + assert!(result.is_err()); + assert_eq!(consumer.active_count(), 1); + assert_eq!(handed_off.as_ref().unwrap().as_ref(), b"data"); + drop(handed_off); + assert_eq!(consumer.active_count(), 0); +} From 76c2d9a0844eb4bc655821822833918231b58e96 Mon Sep 17 00:00:00 2001 From: vladimir-ea Date: Fri, 11 Sep 2026 14:49:28 +0100 Subject: [PATCH 2/5] partial payloads: cell store and control ingress CellStore assembles data column sidecars cell-by-cell in a tcache via per-row sub-reservations, alongside verified full sidecars. Capacity is derived from the blob schedule and delivery retention. Slot rollover closes assemblies and emits a retention boundary for downstream retained consumers. Control gains CellIngress driving context admission, validation outcomes, and availability events. Co-Authored-By: Claude Fable 5 --- Cargo.lock | 2 + Cargo.toml | 2 +- crates/columns/Cargo.toml | 1 + crates/columns/src/cell_store.rs | 588 +++++++++++++ crates/columns/src/cell_store/config.rs | 124 +++ crates/columns/src/cell_store/context.rs | 161 ++++ crates/columns/src/cell_store/tests.rs | 1015 ++++++++++++++++++++++ crates/columns/src/counters.rs | 12 + crates/columns/src/lib.rs | 1 + crates/columns/tests/cell_store_alloc.rs | 130 +++ crates/common/src/cells.rs | 150 ++++ crates/common/src/spine.rs | 6 + crates/common/src/ticker.rs | 20 + crates/control/Cargo.toml | 1 + crates/control/src/cell_ingress.rs | 103 +++ crates/control/src/lib.rs | 1 + crates/control/tests/cell_ingress.rs | 399 +++++++++ 17 files changed, 2715 insertions(+), 1 deletion(-) create mode 100644 crates/columns/src/cell_store.rs create mode 100644 crates/columns/src/cell_store/config.rs create mode 100644 crates/columns/src/cell_store/context.rs create mode 100644 crates/columns/src/cell_store/tests.rs create mode 100644 crates/columns/tests/cell_store_alloc.rs create mode 100644 crates/common/src/cells.rs create mode 100644 crates/control/src/cell_ingress.rs create mode 100644 crates/control/tests/cell_ingress.rs diff --git a/Cargo.lock b/Cargo.lock index 60294d69..e13e9b44 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4861,6 +4861,7 @@ dependencies = [ "rand 0.8.6", "silver_beacon_state_data", "silver_common", + "slab", "snap 1.1.1", "tracing", ] @@ -4933,6 +4934,7 @@ dependencies = [ "fxhash", "raft", "silver_chain_spec", + "silver_columns", "silver_common", "silver_gossip", "silver_peer", diff --git a/Cargo.toml b/Cargo.toml index d874d328..dc3dbc41 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -100,7 +100,7 @@ aes = "0.8" aes-gcm = "0.10" alloy-rlp = { version = "0.3.12", default-features = false, features = ["derive", "std"] } base64 = "0.22" -blst = {version = "0.3", features = ["no-threads"]} +blst = {version = "0.3.16", features = ["no-threads"]} buffa = "0.2.0" build-info = { version = "0.0.46", default-features = false } build-info-build = "0.0.46" diff --git a/crates/columns/Cargo.toml b/crates/columns/Cargo.toml index 04140a62..c1cae1d1 100644 --- a/crates/columns/Cargo.toml +++ b/crates/columns/Cargo.toml @@ -14,6 +14,7 @@ fxhash.workspace = true hex.workspace = true silver_beacon_state_data.workspace = true silver_common.workspace = true +slab.workspace = true rand.workspace = true tracing.workspace = true diff --git a/crates/columns/src/cell_store.rs b/crates/columns/src/cell_store.rs new file mode 100644 index 00000000..8471910c --- /dev/null +++ b/crates/columns/src/cell_store.rs @@ -0,0 +1,588 @@ +use std::{ + io::Write, + ptr, + time::{Duration, Instant}, +}; + +pub use config::CellStoreConfig; +pub use context::{CommitmentContext, ContextData}; +use fxhash::FxHashMap; +use silver_beacon_state_data::{ForkName, SLOTS_PER_EPOCH}; +pub use silver_common::cells::{AcquiredCell, CellKey, CellRef, ColumnRef, PendingCell}; +use silver_common::{ + SubReservationError, SubReservationRef, TCacheProducer, TCacheRead, TProducer, TReservation, + cells::{CellSource, RetentionEvent}, + ssz_view::{BYTES_PER_CELL, BYTES_PER_KZG_PROOF}, +}; +use slab::Slab; + +use crate::{BlockRoot, DataColumnCounters}; + +mod config; +mod context; + +#[cfg(test)] +mod tests; + +pub const CELL_RECORD_BYTES: usize = BYTES_PER_CELL + BYTES_PER_KZG_PROOF; + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum StoreError { + InvalidConfig, + UnsupportedBlobCount(u64), + CapacityOverflow, + CacheTooSmall, + WrongCache, + InvalidContext, + ConflictingContext, + ContextExpired, + OutsideServingSlot, + BelowSlotFloor, + UnknownCell, + Full, + CacheFull, +} + +impl From for StoreError { + fn from(error: SubReservationError) -> Self { + match error { + SubReservationError::CacheFull => { + DataColumnCounters::CellStoreCacheFull.inc(); + Self::CacheFull + } + SubReservationError::Closed | SubReservationError::Stale => Self::ContextExpired, + SubReservationError::WrongConsumer | SubReservationError::WrongProducer => { + Self::WrongCache + } + _ => Self::InvalidContext, + } + } +} + +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +pub struct CellMask(u128); + +impl CellMask { + #[inline] + pub fn bits(self) -> u128 { + self.0 + } + + #[inline] + pub fn contains(self, row: usize) -> bool { + row < u128::BITS as usize && self.0 & (1u128 << row) != 0 + } + + #[inline] + fn all(rows: usize) -> Self { + Self(u128::MAX.checked_shr(u128::BITS - rows as u32).unwrap_or(0)) + } +} + +#[derive(Clone, Copy, Debug)] +pub struct ColumnStatus { + pub admitted: CellMask, + pub available: CellMask, + pub complete: bool, +} + +#[derive(Clone, Copy, Debug)] +pub struct ColumnUpdate { + pub new_cells: CellMask, + pub column_completed: bool, + pub complete_read: Option, +} + +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +pub struct StoreCounts { + pub cells: usize, + // Reserved assembly payload bytes include missing cells. + pub bytes: usize, + pub full_bytes: usize, + pub contexts: usize, + pub active_slots: usize, + pub blocks: usize, +} + +struct Block { + context: CommitmentContext, + ssz: Option, +} + +struct FullColumn { + read: TCacheRead, + cell_offset: usize, + proof_offset: usize, +} + +#[derive(Default)] +struct Column { + assembly: Option, + full: Option, + admitted: CellMask, + complete: bool, +} + +impl Column { + fn available(&self, producer: &TProducer, rows: usize) -> CellMask { + if self.full.is_some() { + CellMask::all(rows) + } else { + CellMask(self.assembly.map_or(0, |reference| { + producer.view_sub_reservation(reference).expect("retained assembly").ready() + })) + } + } +} + +/// Downstream retention boundaries protect stored descriptors between calls. +/// Local views borrow the sole producer, preventing allocation during access. +pub struct CellStore { + blocks: Slab, + columns: Box<[Column]>, + retention_boundary: Option, + producer: TProducer, + config: CellStoreConfig, + roots: FxHashMap, + counts: StoreCounts, + now: Instant, + slot: u64, + slot_end: Instant, + min_slot: u64, + dirty: bool, +} + +impl CellStore { + pub fn new( + config: CellStoreConfig, + producer: TProducer, + slot: u64, + slot_start: Instant, + ) -> Result { + if producer.cache_ref().capacity() < config.cache_capacity() { + return Err(StoreError::CacheTooSmall); + } + let column_entries = config + .block_capacity + .checked_mul(config.column_indices.len()) + .ok_or(StoreError::CapacityOverflow)?; + let store = Self { + blocks: Slab::with_capacity(config.block_capacity), + columns: std::iter::repeat_with(Column::default).take(column_entries).collect(), + retention_boundary: None, + producer, + roots: FxHashMap::with_capacity_and_hasher( + config.block_capacity * 2, + Default::default(), + ), + counts: StoreCounts::default(), + now: slot_start, + slot, + slot_end: slot_start + config.slot_duration, + min_slot: 0, + dirty: false, + config, + }; + store.publish_gauges(); + Ok(store) + } + + pub fn reserve_full(&mut self, len: usize) -> Result { + self.producer.reserve(len, false).ok_or(StoreError::CacheFull) + } + + pub fn admit_context( + &mut self, + context: CommitmentContext, + data: ContextData<'_>, + ) -> Result { + if context.slot < self.min_slot { + return Err(StoreError::BelowSlotFloor); + } + if context.slot != self.slot { + return Err(StoreError::OutsideServingSlot); + } + if !matches!(context.format, ForkName::Fulu | ForkName::Gloas) || + self.config.spec.fork_at_slot(context.slot) != context.format || + context.blob_count > self.config.max_blobs || + context.blob_count > + self.config.spec.blob_params_at(context.slot / SLOTS_PER_EPOCH).max_blobs_per_block + as usize || + !data.valid_for(context) + { + return Err(StoreError::InvalidContext); + } + if let Some(&index) = self.roots.get(&context.block_root) { + let block = &self.blocks[index]; + if block.context != context { + return Err(StoreError::ConflictingContext); + } + let read = block.ssz.ok_or(StoreError::ContextExpired)?; + if !data.matches(self.producer.read_buffer(read).expect("retained context")) { + return Err(StoreError::ConflictingContext); + } + return Ok(false); + } + if self.blocks.len() >= self.config.block_capacity || + self.counts.contexts >= self.config.live_blocks + { + DataColumnCounters::CellStoreFull.inc(); + return Err(StoreError::Full); + } + let ssz = { + let Some(mut reservation) = self.producer.reserve(data.encoded_len(), false) else { + DataColumnCounters::CellStoreCacheFull.inc(); + return Err(StoreError::CacheFull); + }; + data.write(reservation.buffer().expect("new context reservation")); + reservation.flush().expect("new context reservation"); + reservation.read() + }; + let index = self.blocks.vacant_key(); + let start = index * self.config.column_indices.len(); + let mut assembly_bytes = 0; + for (position, &column) in self.config.column_indices.iter().enumerate() { + match data.reserve_column(context, column, &mut self.producer) { + Ok(reference) => { + let view = self.producer.view_sub_reservation(reference).expect("new assembly"); + assembly_bytes += view.len(); + let complete = context.blob_count == 0; + if complete { + view.finish().expect("empty column"); + } + self.columns[start + position] = + Column { assembly: Some(reference), complete, ..Column::default() }; + } + Err(error) => { + for column in &mut self.columns[start..start + self.config.column_indices.len()] + { + if let Some(reference) = column.assembly { + self.producer + .view_sub_reservation(reference) + .expect("new assembly") + .close(); + } + *column = Column::default(); + } + return Err(error.into()); + } + } + } + assert_eq!(self.blocks.insert(Block { context, ssz: Some(ssz) }), index); + self.roots.insert(context.block_root, index); + self.counts.bytes += assembly_bytes; + self.counts.contexts += 1; + self.counts.active_slots = 1; + self.counts.blocks += 1; + self.dirty = true; + Ok(true) + } + + pub fn stage_cell( + &mut self, + key: CellKey, + cell: &[u8; BYTES_PER_CELL], + proof: &[u8; BYTES_PER_KZG_PROOF], + ) -> Result, StoreError> { + let (block, index) = self.index(key).ok_or(StoreError::UnknownCell)?; + let column = &self.columns[index]; + if (column.admitted.0 | + column.available(&self.producer, self.blocks[block].context.blob_count).0) & + (1u128 << key.row) != + 0 + { + DataColumnCounters::CellStoreDuplicates.inc(); + return Ok(None); + } + if self.blocks[block].context.slot < self.min_slot { + return Err(StoreError::BelowSlotFloor); + } + let reference = self.column_ref(block, key.column).ok_or(StoreError::ContextExpired)?; + let view = self.producer.view_sub_reservation(reference.reservation)?; + let claim = match view.claim(key.row) { + Ok(claim) => claim, + Err(SubReservationError::Claimed | SubReservationError::Published) => return Ok(None), + Err(error) => return Err(error.into()), + }; + Ok(Some(PendingCell { key, data: claim.write(cell, proof)? })) + } + + pub fn cancel_pending(&self, pending: PendingCell) -> Result { + let (block, _) = self.index(pending.key).ok_or(StoreError::UnknownCell)?; + let reference = + self.column_ref(block, pending.key.column).ok_or(StoreError::ContextExpired)?; + let read = pending.data.reservation().read(); + if pending.key.row != pending.data.part() || + read.seq() != reference.reservation.read().seq() || + !ptr::eq(&*read.cache_ref(), &*reference.reservation.read().cache_ref()) + { + return Err(StoreError::UnknownCell); + } + self.producer + .view_sub_reservation(reference.reservation)? + .cancel(pending.data) + .map_err(Into::into) + } + + pub fn refresh_column( + &mut self, + root: &BlockRoot, + column: usize, + ) -> Result { + let block = *self.roots.get(root).ok_or(StoreError::UnknownCell)?; + let position = self.config.column_position(column).ok_or(StoreError::UnknownCell)?; + if self.columns[block * self.config.column_indices.len() + position].assembly.is_none() { + return Err(StoreError::ContextExpired); + } + Ok(self.refresh_column_at(block, position)) + } + + #[inline] + fn refresh_column_at(&mut self, block: usize, position: usize) -> ColumnUpdate { + let entry = &mut self.columns[block * self.config.column_indices.len() + position]; + debug_assert!(entry.assembly.is_some()); + let available = entry.available(&self.producer, self.blocks[block].context.blob_count); + let new_cells = CellMask(available.0 & !entry.admitted.0); + let complete_read = match &entry.full { + Some(full) => Some(full.read), + None => entry.assembly.and_then(|reference| { + self.producer + .view_sub_reservation(reference) + .expect("retained assembly") + .finish() + .ok() + }), + }; + let column_completed = !entry.complete && complete_read.is_some(); + entry.admitted = available; + entry.complete |= column_completed; + self.dirty |= new_cells.0 != 0 || column_completed; + let added = new_cells.0.count_ones(); + self.counts.cells += added as usize; + DataColumnCounters::CellStoreAdmissions.add(added as u64); + ColumnUpdate { new_cells, column_completed, complete_read } + } + + /// The sidecar must already be verified; this checks its layout and + /// context. + pub fn retain_full( + &mut self, + root: &BlockRoot, + column: usize, + read: TCacheRead, + ) -> Result { + if !ptr::eq(&*read.cache_ref(), &*self.producer.cache_ref()) { + return Err(StoreError::WrongCache); + } + let block = *self.roots.get(root).ok_or(StoreError::UnknownCell)?; + let position = self.config.column_position(column).ok_or(StoreError::UnknownCell)?; + let context = &self.blocks[block]; + let context_bytes = self + .producer + .read_buffer(context.ssz.ok_or(StoreError::ContextExpired)?) + .map_err(|_| StoreError::ContextExpired)?; + let bytes = self.producer.read_buffer(read).map_err(|_| StoreError::ContextExpired)?; + let (cell_offset, proof_offset) = context + .context + .full_offsets(bytes, context_bytes, column) + .ok_or(StoreError::InvalidContext)?; + let entry = &mut self.columns[block * self.config.column_indices.len() + position]; + if entry.assembly.is_none() { + return Err(StoreError::ContextExpired); + } + if entry.full.is_none() { + entry.full = Some(FullColumn { read, cell_offset, proof_offset }); + self.counts.full_bytes += bytes.len(); + self.dirty = true; + } + Ok(self.refresh_column_at(block, position)) + } + + #[inline] + pub fn cell(&self, key: CellKey) -> Option { + let (block, index) = self.index(key)?; + let column = &self.columns[index]; + if !column + .available(&self.producer, self.blocks[block].context.blob_count) + .contains(key.row) + { + return None; + } + let source = match &column.full { + Some(full) => CellSource::Full { + read: full.read, + cell: full.cell_offset + key.row * BYTES_PER_CELL, + proof: full.proof_offset + key.row * BYTES_PER_KZG_PROOF, + }, + None => CellSource::Assembly { reservation: column.assembly?, row: key.row }, + }; + Some(CellRef { source, slot: self.blocks[block].context.slot, expires: self.slot_end }) + } + + fn column_ref(&self, block: usize, column: usize) -> Option { + let position = self.config.column_position(column)?; + Some(ColumnRef { + block_root: self.blocks[block].context.block_root, + column, + reservation: self.columns[block * self.config.column_indices.len() + position] + .assembly?, + slot: self.blocks[block].context.slot, + expires: self.slot_end, + }) + } + + pub fn reservations(&self, root: &BlockRoot) -> impl Iterator + '_ { + self.roots.get(root).copied().into_iter().flat_map(move |block| { + self.config + .column_indices + .iter() + .filter_map(move |&column| self.column_ref(block, column)) + }) + } + + pub fn context(&self, root: &BlockRoot) -> Option<(&CommitmentContext, TCacheRead)> { + let block = &self.blocks[*self.roots.get(root)?]; + Some((&block.context, block.ssz?)) + } + + pub fn column(&self, root: &BlockRoot, column: usize) -> Option { + let block = *self.roots.get(root)?; + let position = self.config.column_position(column)?; + let entry = &self.columns[block * self.config.column_indices.len() + position]; + let available = entry.available(&self.producer, self.blocks[block].context.blob_count); + Some(ColumnStatus { + admitted: CellMask(entry.admitted.0 | available.0), + available, + complete: entry.complete || + (entry.assembly.is_some() && + available == CellMask::all(self.blocks[block].context.blob_count)), + }) + } + + pub fn slot_end(&self) -> Instant { + self.slot_end + } + + pub fn take_retention_event(&mut self) -> Option { + self.retention_boundary.take() + } + + pub fn advance(&mut self, now: Instant, min_slot: u64, mut on_expired: impl FnMut(CellKey)) { + assert!(now >= self.now, "cell store clock moved backwards"); + self.now = now; + let floor_changed = min_slot > self.min_slot; + self.min_slot = self.min_slot.max(min_slot); + let slot_changed = now >= self.slot_end; + if slot_changed { + let retain_from = self.producer.next_seq(); + if retain_from != 0 { + self.retention_boundary = + Some(RetentionEvent { expired_slot: self.slot, retain_from }); + } + let elapsed = now.duration_since(self.slot_end).as_nanos(); + let slot_nanos = self.config.slot_duration.as_nanos(); + let slots = u64::try_from(elapsed / slot_nanos + 1).expect("cell store slot overflow"); + self.slot = self.slot.checked_add(slots).expect("cell store slot overflow"); + let remainder = elapsed % slot_nanos; + let remainder = Duration::new( + (remainder / 1_000_000_000) as u64, + (remainder % 1_000_000_000) as u32, + ); + self.slot_end = now + (self.config.slot_duration - remainder); + } + if floor_changed || slot_changed { + self.blocks.retain(|index, block| { + let start = index * self.config.column_indices.len(); + let columns = &mut self.columns[start..start + self.config.column_indices.len()]; + if block.context.slot < self.slot && block.ssz.take().is_some() { + for (position, column) in columns.iter_mut().enumerate() { + if let Some(reference) = column.assembly { + self.producer + .view_sub_reservation(reference) + .expect("retained assembly") + .close(); + } + let available = column.available(&self.producer, block.context.blob_count); + column.admitted.0 |= available.0; + column.complete |= available == CellMask::all(block.context.blob_count); + let mut rows = available.0; + while rows != 0 { + let row = rows.trailing_zeros() as usize; + rows &= rows - 1; + on_expired(CellKey { + block_root: block.context.block_root, + column: self.config.column_indices[position], + row, + }); + } + DataColumnCounters::CellStoreExpired.add(available.0.count_ones() as u64); + column.assembly = None; + column.full = None; + } + self.counts.contexts -= 1; + self.dirty = true; + } + if block.context.slot < self.min_slot && block.ssz.is_none() { + self.roots.remove(&block.context.block_root); + for column in columns { + *column = Column::default(); + } + self.dirty = true; + return false; + } + true + }); + self.counts.blocks = self.blocks.len(); + } + if slot_changed { + debug_assert_eq!(self.counts.contexts, 0); + self.counts.cells = 0; + self.counts.bytes = 0; + self.counts.full_bytes = 0; + self.counts.active_slots = 0; + } + if self.dirty { + self.publish_gauges(); + self.dirty = false; + } + } + + #[inline] + pub fn counts(&self) -> StoreCounts { + self.counts + } + + #[inline] + fn index(&self, key: CellKey) -> Option<(usize, usize)> { + let block = *self.roots.get(&key.block_root)?; + if key.row >= self.blocks[block].context.blob_count { + return None; + } + let column = + block * self.config.column_indices.len() + self.config.column_position(key.column)?; + Some((block, column)) + } + + fn publish_gauges(&self) { + let counts = self.counts(); + DataColumnCounters::CellStoreCapacity.set(self.producer.cache_ref().capacity() as u64); + DataColumnCounters::CellStoreLiveCells.set(counts.cells as u64); + DataColumnCounters::CellStoreLiveBytes.set(counts.bytes as u64); + DataColumnCounters::CellStoreFullBytes.set(counts.full_bytes as u64); + DataColumnCounters::CellStoreContexts.set(counts.contexts as u64); + DataColumnCounters::CellStoreActiveSlots.set(counts.active_slots as u64); + DataColumnCounters::CellStoreBlocks.set(counts.blocks as u64); + } +} + +impl Drop for CellStore { + fn drop(&mut self) { + for column in &self.columns { + if let Some(reference) = column.assembly { + if let Ok(view) = self.producer.view_sub_reservation(reference) { + view.close(); + } + } + } + } +} diff --git a/crates/columns/src/cell_store/config.rs b/crates/columns/src/cell_store/config.rs new file mode 100644 index 00000000..e819840a --- /dev/null +++ b/crates/columns/src/cell_store/config.rs @@ -0,0 +1,124 @@ +use std::{sync::Arc, time::Duration}; + +use silver_beacon_state_data::{FAR_FUTURE_EPOCH, SLOTS_PER_EPOCH, SpecConfig}; +use silver_common::{ + SubLayout, + ssz_view::{ + BYTES_PER_CELL, BYTES_PER_KZG_COMMITMENT, BYTES_PER_KZG_PROOF, DATA_COLUMN_SIDECAR_MIN, + }, +}; + +use super::StoreError; + +pub struct CellStoreConfig { + pub(super) spec: Arc, + pub(super) columns: u128, + pub(super) column_indices: Box<[usize]>, + pub(super) max_blobs: usize, + pub(super) slot_duration: Duration, + pub(super) block_capacity: usize, + pub(super) live_blocks: usize, + pub(super) cell_capacity: usize, + cache_bytes: usize, +} + +impl CellStoreConfig { + pub fn new( + spec: Arc, + columns: u128, + delivery_retention: Duration, + ) -> Result { + let max_blobs = spec + .blob_schedule + .iter() + .filter(|entry| entry.epoch != FAR_FUTURE_EPOCH) + .map(|entry| entry.max_blobs_per_block) + .fold(spec.max_blobs_per_block_electra, u64::max); + if max_blobs > u128::BITS as u64 { + return Err(StoreError::UnsupportedBlobCount(max_blobs)); + } + if columns == 0 || spec.slot_duration_ms() == 0 { + return Err(StoreError::InvalidConfig); + } + + let slot_duration = Duration::from_millis(spec.slot_duration_ms()); + let retention = + slot_duration.checked_add(delivery_retention).ok_or(StoreError::CapacityOverflow)?; + // One additional block covers a burst at the ends of the retention window. + let live_blocks = usize::try_from(retention.as_nanos().div_ceil(slot_duration.as_nanos())) + .ok() + .and_then(|blocks| blocks.checked_add(1)) + .ok_or(StoreError::CapacityOverflow)?; + // Keep duplicate/completion metadata across two epochs, allowing fork siblings. + let block_capacity = live_blocks + .max(2 * SLOTS_PER_EPOCH as usize) + .checked_mul(2) + .ok_or(StoreError::CapacityOverflow)?; + let max_blobs = max_blobs as usize; + let column_count = columns.count_ones() as usize; + let cell_capacity = live_blocks + .checked_mul(column_count) + .and_then(|n| n.checked_mul(max_blobs)) + .ok_or(StoreError::CapacityOverflow)?; + let context_bytes = DATA_COLUMN_SIDECAR_MIN + max_blobs * BYTES_PER_KZG_COMMITMENT; + let column_bytes = SubLayout { + parts: max_blobs, + first_len: BYTES_PER_CELL, + second_len: BYTES_PER_KZG_PROOF, + } + .reservation_bytes(DATA_COLUMN_SIDECAR_MIN, max_blobs * BYTES_PER_KZG_COMMITMENT) + .ok_or(StoreError::CapacityOverflow)?; + let payload_bytes = live_blocks + .checked_mul(column_count) + .and_then(|n| n.checked_mul(column_bytes)) + .and_then(|n| n.checked_add(live_blocks.checked_mul(context_bytes)?)) + .ok_or(StoreError::CapacityOverflow)?; + let full_column_bytes = DATA_COLUMN_SIDECAR_MIN + + max_blobs * (BYTES_PER_CELL + BYTES_PER_KZG_COMMITMENT + BYTES_PER_KZG_PROOF); + let full_payload_bytes = live_blocks + .checked_mul(column_count) + .and_then(|n| n.checked_mul(full_column_bytes)) + .ok_or(StoreError::CapacityOverflow)?; + // Full sidecars and assemblies share one ring. Headroom covers ingress + // duplicates, reservation headers, wrap padding, and bucket rounding. + let payload_bytes = + payload_bytes.checked_add(full_payload_bytes).ok_or(StoreError::CapacityOverflow)?; + let cache_bytes = payload_bytes + .checked_add(payload_bytes / 2) + .and_then(|n| n.checked_add(64 * 1024)) + .and_then(usize::checked_next_power_of_two) + .filter(|n| u32::try_from(*n).is_ok()) + .ok_or(StoreError::CapacityOverflow)?; + Ok(Self { + spec, + columns, + column_indices: (0..u128::BITS as usize) + .filter(|column| columns & (1u128 << column) != 0) + .collect(), + max_blobs, + slot_duration, + block_capacity, + live_blocks, + cell_capacity, + cache_bytes, + }) + } + + pub fn cache_capacity(&self) -> usize { + self.cache_bytes + } + + pub fn cell_capacity(&self) -> usize { + self.cell_capacity + } + + pub fn max_blobs(&self) -> usize { + self.max_blobs + } + + #[inline] + pub(super) fn column_position(&self, column: usize) -> Option { + let bit = 1u128.checked_shl(u32::try_from(column).ok()?)?; + (self.columns & bit != 0).then(|| (self.columns & (bit - 1)).count_ones() as usize) + } +} diff --git a/crates/columns/src/cell_store/context.rs b/crates/columns/src/cell_store/context.rs new file mode 100644 index 00000000..1e4f1c7b --- /dev/null +++ b/crates/columns/src/cell_store/context.rs @@ -0,0 +1,161 @@ +use silver_beacon_state_data::ForkName; +use silver_common::{ + SubLayout, SubReservationError, SubReservationRef, TProducer, + ssz_view::{ + BYTES_PER_CELL, BYTES_PER_KZG_COMMITMENT, BYTES_PER_KZG_PROOF, + DATA_COLUMN_SIDECAR_GLOAS_MIN, DATA_COLUMN_SIDECAR_MIN, DataColumnSidecarFuluView, + DataColumnSidecarGloasView, + }, +}; + +use crate::BlockRoot; + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct CommitmentContext { + pub block_root: BlockRoot, + pub slot: u64, + pub format: ForkName, + pub blob_count: usize, +} + +impl CommitmentContext { + pub(super) fn full_offsets( + self, + bytes: &[u8], + context: &[u8], + column: usize, + ) -> Option<(usize, usize)> { + let rows = self.blob_count; + match self.format { + ForkName::Fulu => { + if !DataColumnSidecarFuluView::check_size(bytes) || + DataColumnSidecarFuluView::index(bytes) != column as u64 || + DataColumnSidecarFuluView::column(bytes).len() != rows * BYTES_PER_CELL || + DataColumnSidecarFuluView::kzg_commitments(bytes).len() != + rows * BYTES_PER_KZG_COMMITMENT || + DataColumnSidecarFuluView::kzg_proofs(bytes).len() != + rows * BYTES_PER_KZG_PROOF || + bytes[20..356] != context[4..340] || + DataColumnSidecarFuluView::kzg_commitments(bytes) != &context[340..] + { + return None; + } + Some(( + DATA_COLUMN_SIDECAR_MIN, + DATA_COLUMN_SIDECAR_MIN + rows * (BYTES_PER_CELL + BYTES_PER_KZG_COMMITMENT), + )) + } + ForkName::Gloas => { + if !DataColumnSidecarGloasView::check_size(bytes) || + DataColumnSidecarGloasView::index(bytes) != column as u64 || + DataColumnSidecarGloasView::slot(bytes) != self.slot || + DataColumnSidecarGloasView::beacon_block_root(bytes) != &self.block_root || + DataColumnSidecarGloasView::column(bytes).len() != rows * BYTES_PER_CELL || + DataColumnSidecarGloasView::kzg_proofs(bytes).len() != + rows * BYTES_PER_KZG_PROOF + { + return None; + } + Some(( + DATA_COLUMN_SIDECAR_GLOAS_MIN, + DATA_COLUMN_SIDECAR_GLOAS_MIN + rows * BYTES_PER_CELL, + )) + } + _ => None, + } + } +} + +#[derive(Clone, Copy)] +pub enum ContextData<'a> { + Fulu { signed_header: &'a [u8; 208], inclusion_proof: &'a [u8; 128], commitments: &'a [u8] }, + Gloas { commitments: &'a [u8] }, +} + +impl<'a> ContextData<'a> { + pub(super) fn valid_for(self, context: CommitmentContext) -> bool { + let format_matches = match self { + Self::Fulu { signed_header, .. } => { + context.format == ForkName::Fulu && signed_header[..8] == context.slot.to_le_bytes() + } + Self::Gloas { .. } => context.format == ForkName::Gloas, + }; + format_matches && self.commitments().len() == context.blob_count * BYTES_PER_KZG_COMMITMENT + } + + fn commitments(self) -> &'a [u8] { + match self { + Self::Fulu { commitments, .. } | Self::Gloas { commitments } => commitments, + } + } + + pub(super) fn encoded_len(self) -> usize { + self.commitments().len() + if matches!(self, Self::Fulu { .. }) { 340 } else { 0 } + } + + pub(super) fn write(self, out: &mut [u8]) { + match self { + Self::Fulu { signed_header, inclusion_proof, commitments } => { + out[..4].copy_from_slice(&340u32.to_le_bytes()); + out[4..212].copy_from_slice(signed_header); + out[212..340].copy_from_slice(inclusion_proof); + out[340..].copy_from_slice(commitments); + } + Self::Gloas { commitments } => out.copy_from_slice(commitments), + } + } + + pub(super) fn matches(self, bytes: &[u8]) -> bool { + if bytes.len() != self.encoded_len() { + return false; + } + match self { + Self::Fulu { signed_header, inclusion_proof, commitments } => { + bytes[..4] == 340u32.to_le_bytes() && + bytes[4..212] == *signed_header && + bytes[212..340] == *inclusion_proof && + bytes[340..] == *commitments + } + Self::Gloas { commitments } => bytes == commitments, + } + } + + pub(super) fn reserve_column( + self, + context: CommitmentContext, + column: usize, + producer: &mut TProducer, + ) -> Result { + let layout = SubLayout { + parts: context.blob_count, + first_len: BYTES_PER_CELL, + second_len: BYTES_PER_KZG_PROOF, + }; + let mut prefix = [0; DATA_COLUMN_SIDECAR_MIN]; + prefix[..8].copy_from_slice(&(column as u64).to_le_bytes()); + let (length, middle) = match self { + Self::Fulu { signed_header, inclusion_proof, commitments } => { + let cells_end = DATA_COLUMN_SIDECAR_MIN + context.blob_count * BYTES_PER_CELL; + prefix[8..12].copy_from_slice(&(DATA_COLUMN_SIDECAR_MIN as u32).to_le_bytes()); + prefix[12..16].copy_from_slice(&(cells_end as u32).to_le_bytes()); + prefix[16..20] + .copy_from_slice(&((cells_end + commitments.len()) as u32).to_le_bytes()); + prefix[20..228].copy_from_slice(signed_header); + prefix[228..356].copy_from_slice(inclusion_proof); + (DATA_COLUMN_SIDECAR_MIN, commitments) + } + Self::Gloas { .. } => { + prefix[8..12] + .copy_from_slice(&(DATA_COLUMN_SIDECAR_GLOAS_MIN as u32).to_le_bytes()); + prefix[12..16].copy_from_slice( + &((DATA_COLUMN_SIDECAR_GLOAS_MIN + context.blob_count * BYTES_PER_CELL) as u32) + .to_le_bytes(), + ); + prefix[16..24].copy_from_slice(&context.slot.to_le_bytes()); + prefix[24..56].copy_from_slice(&context.block_root); + (DATA_COLUMN_SIDECAR_GLOAS_MIN, &[][..]) + } + }; + producer.sub_reservation(layout, &prefix[..length], middle) + } +} diff --git a/crates/columns/src/cell_store/tests.rs b/crates/columns/src/cell_store/tests.rs new file mode 100644 index 00000000..99832663 --- /dev/null +++ b/crates/columns/src/cell_store/tests.rs @@ -0,0 +1,1015 @@ +use std::{io::Write, sync::Arc, time::Duration}; + +use silver_beacon_state_data::{BlobParameters, FAR_FUTURE_EPOCH, SpecConfig}; +use silver_common::{ + TCache, TCacheRef, TRandomAccess, column_util::push_data_column_sidecar_prefix, +}; + +use super::*; + +const ROOT: BlockRoot = [1; 32]; +const CELL: [u8; BYTES_PER_CELL] = [0x11; BYTES_PER_CELL]; +const PROOF: [u8; BYTES_PER_KZG_PROOF] = [0x22; BYTES_PER_KZG_PROOF]; + +struct Harness { + store: CellStore, + consumer: Box, + cache: TCacheRef, + start: Instant, +} + +#[derive(Debug)] +enum CellAdmission { + Inserted { cell: CellRef, column_completed: bool }, + Duplicate, +} + +impl Harness { + fn spec(rows: u64) -> SpecConfig { + SpecConfig { + fulu_fork_epoch: 0, + gloas_fork_epoch: 2, + max_blobs_per_block_electra: rows, + blob_schedule: Vec::new(), + ..SpecConfig::mainnet() + } + } + + fn new(rows: u64, columns: u128) -> Self { + let config = + CellStoreConfig::new(Arc::new(Self::spec(rows)), columns, Duration::from_secs(11)) + .unwrap(); + Self::configured(config) + } + + fn configured(config: CellStoreConfig) -> Self { + Self::at_slot(config, 0) + } + + fn at_slot(config: CellStoreConfig, slot: u64) -> Self { + let producer = TCache::producer("", config.cache_capacity()); + let cache = producer.cache_ref(); + let consumer = Box::new(cache.retained_random_access("").unwrap()); + let start = Instant::now(); + Self { + store: CellStore::new(config, producer, slot, start).unwrap(), + consumer, + cache, + start, + } + } + + fn context(&mut self, root: BlockRoot, slot: u64, rows: usize) -> CommitmentContext { + let context = CommitmentContext { + block_root: root, + slot, + format: self.store.config.spec.fork_at_slot(slot), + blob_count: rows, + }; + assert!(self.admit(context, 0x33).unwrap()); + context + } + + fn admit(&mut self, context: CommitmentContext, value: u8) -> Result { + let mut header = [value; 208]; + header[..8].copy_from_slice(&context.slot.to_le_bytes()); + let proof = [value; 128]; + let commitments = [value; 128 * 48]; + let commitments = &commitments[..context.blob_count.min(128) * 48]; + let data = match context.format { + ForkName::Gloas => ContextData::Gloas { commitments }, + _ => ContextData::Fulu { signed_header: &header, inclusion_proof: &proof, commitments }, + }; + self.store.admit_context(context, data) + } + + fn insert(&mut self, key: CellKey) -> (CellRef, bool) { + let CellAdmission::Inserted { cell, column_completed } = + self.admit_cell(key, &CELL, &PROOF).unwrap() + else { + panic!("expected a new cell") + }; + (cell, column_completed) + } + + fn assert_counts(&self) { + let mut expected = + StoreCounts { blocks: self.store.blocks.len(), ..StoreCounts::default() }; + for (index, block) in &self.store.blocks { + if block.ssz.is_none() { + continue; + } + expected.contexts += 1; + expected.active_slots = 1; + let start = index * self.store.config.column_indices.len(); + for column in &self.store.columns[start..start + self.store.config.column_indices.len()] + { + expected.cells += column.admitted.0.count_ones() as usize; + if let Some(reference) = column.assembly { + expected.bytes += + self.store.producer.view_sub_reservation(reference).unwrap().len(); + } + if let Some(full) = &column.full { + expected.full_bytes += + self.store.producer.read_buffer(full.read).unwrap().len(); + } + } + } + assert_eq!(self.store.counts(), expected); + } + + fn advance_ms(&mut self, elapsed: u64) { + self.advance(self.start + Duration::from_millis(elapsed), 0, |_| {}); + } + + fn advance(&mut self, now: Instant, min_slot: u64, on_expired: impl FnMut(CellKey)) { + self.store.advance(now, min_slot, on_expired); + if let Some(event) = self.store.take_retention_event() { + self.consumer.advance_retention(event.retain_from); + } + } + + fn admit_cell( + &mut self, + key: CellKey, + cell: &[u8; BYTES_PER_CELL], + proof: &[u8; BYTES_PER_KZG_PROOF], + ) -> Result { + let Some(pending) = self.store.stage_cell(key, cell, proof)? else { + return Ok(CellAdmission::Duplicate); + }; + pending.data.acquire(&mut self.consumer)?.accept()?; + let update = self.store.refresh_column(&key.block_root, key.column)?; + Ok(CellAdmission::Inserted { + cell: self.store.cell(key).unwrap(), + column_completed: update.column_completed, + }) + } + + fn acquire_cell(&mut self, key: CellKey) -> Option { + self.store.cell(key)?.acquire(&mut self.consumer) + } + + fn full_bytes(&self, root: &BlockRoot, column: usize) -> Vec { + let block = &self.store.blocks[self.store.roots[root]]; + let context = block.context; + let bytes = self.store.producer.read_buffer(block.ssz.unwrap()).unwrap(); + let mut full = Vec::new(); + match context.format { + ForkName::Fulu => push_data_column_sidecar_prefix( + &mut full, + column as u64, + context.blob_count, + bytes[4..212].try_into().unwrap(), + bytes[212..340].try_into().unwrap(), + ), + ForkName::Gloas => { + full.extend_from_slice(&(column as u64).to_le_bytes()); + full.extend_from_slice(&56u32.to_le_bytes()); + full.extend_from_slice( + &((56 + context.blob_count * BYTES_PER_CELL) as u32).to_le_bytes(), + ); + full.extend_from_slice(&context.slot.to_le_bytes()); + full.extend_from_slice(root); + } + _ => unreachable!(), + } + for row in 0..context.blob_count { + full.extend_from_slice(&[0x11 + row as u8; BYTES_PER_CELL]); + } + if context.format == ForkName::Fulu { + full.extend_from_slice(&bytes[340..]); + } + for row in 0..context.blob_count { + full.extend_from_slice(&[0x22 + row as u8; BYTES_PER_KZG_PROOF]); + } + full + } +} + +fn key(column: usize, row: usize) -> CellKey { + CellKey { block_root: ROOT, column, row } +} + +#[test] +fn capacity_uses_the_entire_blob_schedule_and_selected_columns() { + let mut spec = Harness::spec(3); + spec.blob_schedule = vec![BlobParameters { epoch: 1, max_blobs_per_block: 17 }]; + let one = CellStoreConfig::new(Arc::new(spec.clone()), 1, Duration::from_secs(11)).unwrap(); + let two = CellStoreConfig::new(Arc::new(spec), 3, Duration::from_secs(11)).unwrap(); + assert_eq!(one.max_blobs(), 17); + assert_eq!(one.cell_capacity(), 3 * 17); + assert_eq!(two.cell_capacity(), one.cell_capacity() * 2); + assert!(two.cache_capacity() >= one.cache_capacity()); + assert!(two.cache_capacity().is_power_of_two()); + + let fallback = CellStoreConfig::new(Arc::new(Harness::spec(3)), 1, Duration::ZERO).unwrap(); + assert_eq!(fallback.max_blobs(), 3); + assert_eq!(fallback.cell_capacity(), 2 * 3); +} + +#[test] +fn mainnet_and_hoodi_capacity_use_their_configured_schedule() { + for spec in [SpecConfig::mainnet(), SpecConfig::hoodi()] { + let expected = spec + .blob_schedule + .iter() + .map(|entry| entry.max_blobs_per_block) + .fold(spec.max_blobs_per_block_electra, u64::max); + let config = + CellStoreConfig::new(Arc::new(spec), u128::MAX, Duration::from_secs(11)).unwrap(); + assert_eq!(config.max_blobs(), expected as usize); + assert_eq!(config.cell_capacity(), 3 * 128 * expected as usize); + } +} + +#[test] +fn larger_delivery_window_provisions_more_cells() { + let spec = Arc::new(Harness::spec(21)); + let short = CellStoreConfig::new(spec.clone(), 3, Duration::ZERO).unwrap(); + let long = CellStoreConfig::new(spec, 3, Duration::from_secs(60)).unwrap(); + assert!(long.cell_capacity() > short.cell_capacity()); + assert!(long.cache_capacity() >= short.cache_capacity()); +} + +#[test] +fn unsupported_future_blob_counts_are_rejected_at_construction() { + let mut spec = Harness::spec(3); + spec.blob_schedule = vec![BlobParameters { epoch: 100, max_blobs_per_block: 129 }]; + assert!(matches!( + CellStoreConfig::new(Arc::new(spec), 1, Duration::ZERO), + Err(StoreError::UnsupportedBlobCount(129)) + )); + assert!(matches!( + CellStoreConfig::new(Arc::new(Harness::spec(129)), 1, Duration::ZERO), + Err(StoreError::UnsupportedBlobCount(129)) + )); +} + +#[test] +fn unscheduled_blob_entries_do_not_inflate_capacity() { + let mut spec = Harness::spec(3); + spec.blob_schedule = + vec![BlobParameters { epoch: FAR_FUTURE_EPOCH, max_blobs_per_block: u64::MAX }]; + let config = CellStoreConfig::new(Arc::new(spec), 1, Duration::ZERO).unwrap(); + assert_eq!(config.max_blobs(), 3); +} + +#[test] +fn invalid_and_overflowing_configurations_are_rejected() { + assert!(matches!( + CellStoreConfig::new(Arc::new(Harness::spec(3)), 0, Duration::ZERO), + Err(StoreError::InvalidConfig) + )); + let mut spec = Harness::spec(3); + spec.slot_duration_ms = Some(0); + assert!(matches!( + CellStoreConfig::new(Arc::new(spec), 1, Duration::ZERO), + Err(StoreError::InvalidConfig) + )); + assert!(matches!( + CellStoreConfig::new(Arc::new(Harness::spec(3)), 1, Duration::MAX), + Err(StoreError::CapacityOverflow) + )); +} + +#[test] +fn an_undersized_cache_is_rejected() { + let config = + CellStoreConfig::new(Arc::new(Harness::spec(21)), u128::MAX, Duration::ZERO).unwrap(); + let producer = TCache::producer("", 1 << 16); + assert_eq!(producer.cache_ref().capacity(), 1 << 16); + assert!(matches!( + CellStore::new(config, producer, 0, Instant::now()), + Err(StoreError::CacheTooSmall) + )); +} + +#[test] +fn cell_and_proof_share_one_record() { + let mut h = Harness::new(2, 1); + h.context(ROOT, 0, 2); + let (reference, complete) = h.insert(key(0, 0)); + assert!(!complete); + assert_eq!(reference.read().len().unwrap_err().to_string(), "reservation is incomplete"); + let acquired = h.acquire_cell(key(0, 0)).unwrap(); + let cell = acquired.cell; + let proof = acquired.proof; + assert_eq!(cell.as_ref(), &CELL); + assert_eq!(proof.as_ref(), &PROOF); + assert_eq!(h.store.counts().cells, 1); + assert_eq!(h.store.counts().bytes, 356 + 2 * (CELL_RECORD_BYTES + 48)); + assert_eq!(h.store.counts().contexts, 1); +} + +#[test] +fn incremental_counts_track_refresh_duplicates_full_columns_and_expiry() { + for slot in [0, 64] { + let config = CellStoreConfig::new(Arc::new(Harness::spec(2)), 3, Duration::ZERO).unwrap(); + let mut h = Harness::at_slot(config, slot); + h.assert_counts(); + let context = h.context(ROOT, slot, 2); + h.assert_counts(); + let admitted = h.store.counts(); + assert!(!h.admit(context, 0x33).unwrap()); + assert_eq!(h.store.counts(), admitted); + + let pending = h.store.stage_cell(key(0, 0), &CELL, &PROOF).unwrap().unwrap(); + pending.data.acquire(&mut h.consumer).unwrap().accept().unwrap(); + assert_eq!(h.store.column(&ROOT, 0).unwrap().available.bits(), 1); + assert_eq!(h.store.counts().cells, 0); + h.assert_counts(); + assert_eq!(h.store.refresh_column(&ROOT, 0).unwrap().new_cells.bits(), 1); + assert_eq!(h.store.counts().cells, 1); + h.assert_counts(); + assert_eq!(h.store.refresh_column(&ROOT, 0).unwrap().new_cells.bits(), 0); + h.assert_counts(); + + let full = h.full_bytes(&ROOT, 0); + let mut write = h.store.reserve_full(full.len()).unwrap(); + write.write_all(&full).unwrap(); + write.flush().unwrap(); + let read = write.read(); + assert_eq!(h.store.retain_full(&ROOT, 0, read).unwrap().new_cells.bits(), 2); + assert_eq!(h.store.counts().cells, 2); + assert_eq!(h.store.counts().full_bytes, full.len()); + h.assert_counts(); + assert_eq!(h.store.retain_full(&ROOT, 0, read).unwrap().new_cells.bits(), 0); + h.assert_counts(); + + let pending = h.store.stage_cell(key(1, 0), &CELL, &PROOF).unwrap().unwrap(); + pending.data.acquire(&mut h.consumer).unwrap().accept().unwrap(); + h.context([2; 32], slot, 1); + h.assert_counts(); + let mut expired = 0; + h.advance(h.start + Duration::from_secs(12), 0, |_| expired += 1); + assert_eq!(expired, 3); + assert_eq!(h.store.counts(), StoreCounts { blocks: 2, ..StoreCounts::default() }); + h.assert_counts(); + assert!(matches!(h.store.refresh_column(&ROOT, 1), Err(StoreError::ContextExpired))); + h.advance(h.start + Duration::from_secs(12), slot + 1, |_| panic!("already expired")); + assert_eq!(h.store.counts(), StoreCounts::default()); + h.assert_counts(); + } +} + +#[test] +fn failed_context_reservation_does_not_change_incremental_counts() { + let mut h = Harness::new(1, 3); + let mut pressure = h.store.reserve_full(h.cache.capacity() - 4096).unwrap(); + pressure.buffer().unwrap().fill(0); + pressure.flush().unwrap(); + let context = + CommitmentContext { block_root: ROOT, slot: 0, format: ForkName::Fulu, blob_count: 1 }; + assert_eq!(h.admit(context, 0x33), Err(StoreError::CacheFull)); + assert_eq!(h.store.counts(), StoreCounts::default()); + h.assert_counts(); + assert!(h.store.columns.iter().all(|column| column.assembly.is_none())); + h.advance_ms(12_000); + h.context(ROOT, 1, 1); + h.assert_counts(); + h.insert(key(0, 0)); + h.assert_counts(); +} + +#[test] +fn duplicates_do_not_copy_or_refresh_deadlines() { + let mut h = Harness::new(2, 1); + let context = h.context(ROOT, 0, 2); + let (first, _) = h.insert(key(0, 0)); + h.advance_ms(11_000); + assert!(!h.admit(context, 0x33).unwrap()); + assert!(matches!(h.admit_cell(key(0, 0), &CELL, &PROOF), Ok(CellAdmission::Duplicate))); + assert_eq!(h.store.cell(key(0, 0)).unwrap().expires, first.expires); + assert_eq!(h.store.cell(key(0, 0)).unwrap().read().seq(), first.read().seq()); + h.advance_ms(12_000); + assert!(h.store.cell(key(0, 0)).is_none()); + assert!(matches!(h.admit_cell(key(0, 0), &CELL, &PROOF), Ok(CellAdmission::Duplicate))); + assert_eq!(h.store.counts().cells, 0); +} + +#[test] +fn slot_deadline_advances_without_contexts() { + let mut h = Harness::new(1, 1); + assert_eq!(h.store.slot_end(), h.start + Duration::from_secs(12)); + assert_eq!(h.store.counts().active_slots, 0); + h.advance_ms(12_000); + assert_eq!(h.store.slot_end(), h.start + Duration::from_secs(24)); + assert_eq!(h.store.counts().active_slots, 0); +} + +#[test] +fn late_slot_admission_expires_at_the_slot_boundary() { + let mut h = Harness::new(1, 1); + h.advance_ms(11_999); + h.context(ROOT, 0, 1); + let (cell, complete) = h.insert(key(0, 0)); + assert!(complete); + assert_eq!(cell.expires, h.start + Duration::from_secs(12)); + assert_eq!(h.store.slot_end(), cell.expires); + h.advance_ms(12_000); + assert!(h.store.cell(key(0, 0)).is_none()); + assert!(h.store.context(&ROOT).is_none()); + assert_eq!(h.store.counts().active_slots, 0); +} + +#[test] +fn slot_duration_uses_milliseconds_at_fixed_boundaries() { + let mut spec = Harness::spec(1); + spec.slot_duration_ms = Some(500); + let config = CellStoreConfig::new(Arc::new(spec), 1, Duration::ZERO).unwrap(); + let mut h = Harness::configured(config); + h.context(ROOT, 0, 1); + h.advance_ms(499); + let (cell, _) = h.insert(key(0, 0)); + assert_eq!(cell.expires, h.start + Duration::from_millis(500)); + assert_eq!(h.store.slot_end(), cell.expires); + h.advance_ms(500); + assert!(h.store.cell(key(0, 0)).is_none()); + assert_eq!(h.store.counts().active_slots, 0); +} + +#[test] +fn cells_and_fork_siblings_share_the_slot_deadline() { + let mut h = Harness::new(3, 1); + h.context(ROOT, 0, 3); + let (first, _) = h.insert(key(0, 0)); + h.advance_ms(500); + let (second, _) = h.insert(key(0, 1)); + assert_eq!(first.slot, second.slot); + assert_eq!(h.store.counts().active_slots, 1); + h.advance_ms(1000); + let (third, _) = h.insert(key(0, 2)); + h.advance_ms(11_000); + h.context([2; 32], 0, 3); + let other = CellKey { block_root: [2; 32], column: 0, row: 0 }; + let (sibling, _) = h.insert(other); + assert_eq!(second.slot, third.slot); + assert_eq!(third.slot, sibling.slot); + assert_eq!(first.expires, second.expires); + assert_eq!(second.expires, third.expires); + assert_eq!(third.expires, sibling.expires); + assert_eq!(h.store.counts().active_slots, 1); + assert_eq!(h.store.slot_end(), sibling.expires); + h.advance_ms(11_999); + assert_eq!(h.store.column(&ROOT, 0).unwrap().available.bits(), 7); + assert!(h.store.cell(other).is_some()); + h.advance_ms(12_000); + assert_eq!(h.store.column(&ROOT, 0).unwrap().available.bits(), 0); + assert_eq!(h.store.column(&other.block_root, 0).unwrap().available.bits(), 0); + assert_eq!(h.store.counts().cells, 0); + assert_eq!(h.store.counts().contexts, 0); + assert_eq!(h.store.counts().active_slots, 0); +} + +#[test] +fn completion_survives_serving_expiry_without_reopening_admission() { + let mut h = Harness::new(2, 1); + h.context(ROOT, 0, 2); + assert!(!h.insert(key(0, 0)).1); + assert!(h.insert(key(0, 1)).1); + h.advance_ms(13_000); + let column = h.store.column(&ROOT, 0).unwrap(); + assert!(column.complete); + assert_eq!(column.available.bits(), 0); + assert_eq!(column.admitted.bits(), 3); + assert!(matches!(h.admit_cell(key(0, 1), &CELL, &PROOF), Ok(CellAdmission::Duplicate))); + assert_eq!(h.store.counts().cells, 0); +} + +#[test] +fn unfinished_columns_cannot_continue_in_the_next_slot() { + let mut h = Harness::new(3, 1); + h.context(ROOT, 0, 3); + h.insert(key(0, 0)); + h.advance_ms(11_000); + h.insert(key(0, 1)); + h.advance_ms(13_000); + assert!(matches!(h.admit_cell(key(0, 2), &CELL, &PROOF), Err(StoreError::ContextExpired))); + let status = h.store.column(&ROOT, 0).unwrap(); + assert_eq!(status.admitted.bits(), 3); + assert_eq!(status.available.bits(), 0); + assert!(!status.complete); +} + +#[test] +fn context_survives_until_the_slot_boundary() { + let mut h = Harness::new(2, 1); + h.context(ROOT, 0, 2); + h.insert(key(0, 0)); + h.advance_ms(11_000); + h.insert(key(0, 1)); + h.advance_ms(11_999); + let read = h.store.context(&ROOT).unwrap().1; + assert_eq!(&h.store.producer.read_buffer(read).unwrap()[12..20], &[0x33; 8]); + h.advance_ms(12_000); + assert!(h.store.context(&ROOT).is_none()); + assert_eq!(h.store.counts().contexts, 0); +} + +#[test] +fn expired_incomplete_context_requires_full_sidecar_recovery() { + let mut h = Harness::new(2, 1); + let context = h.context(ROOT, 0, 2); + h.insert(key(0, 0)); + h.advance_ms(13_000); + assert!(matches!(h.admit_cell(key(0, 1), &CELL, &PROOF), Err(StoreError::ContextExpired))); + assert_eq!(h.admit(context, 0x33), Err(StoreError::OutsideServingSlot)); + assert!(!h.store.column(&ROOT, 0).unwrap().complete); +} + +#[test] +fn schedule_and_fork_boundaries_use_the_blocks_slot() { + let mut spec = Harness::spec(2); + spec.blob_schedule = vec![BlobParameters { epoch: 1, max_blobs_per_block: 4 }]; + let config = CellStoreConfig::new(Arc::new(spec), 1, Duration::ZERO).unwrap(); + let mut h = Harness::at_slot(config, 31); + let mut context = + CommitmentContext { block_root: ROOT, slot: 31, format: ForkName::Fulu, blob_count: 4 }; + assert_eq!(h.admit(context, 0x33), Err(StoreError::InvalidContext)); + h.advance_ms(12_000); + context.slot = 32; + assert!(h.admit(context, 0x33).unwrap()); + let old = h.insert(key(0, 0)).0; + let sent = h.acquire_cell(key(0, 0)).unwrap(); + h.advance_ms(33 * 12_000); + context.block_root = [2; 32]; + context.slot = 64; + assert_eq!(h.admit(context, 0x33), Err(StoreError::InvalidContext)); + context.format = ForkName::Gloas; + assert!(h.admit(context, 0x33).unwrap()); + h.insert(CellKey { block_root: [2; 32], column: 0, row: 3 }); + assert_eq!(h.store.blocks[h.store.roots[&ROOT]].context.format, ForkName::Fulu); + assert!(h.store.cell(key(0, 0)).is_none()); + assert_eq!(old.expires, h.start + Duration::from_secs(24)); + assert_eq!(sent.cell.as_ref(), CELL); +} + +#[test] +fn non_current_slots_are_not_admitted() { + let config = CellStoreConfig::new(Arc::new(Harness::spec(2)), 1, Duration::ZERO).unwrap(); + let mut h = Harness::at_slot(config, 40); + for slot in [39, 41] { + let context = + CommitmentContext { block_root: ROOT, slot, format: ForkName::Fulu, blob_count: 2 }; + assert_eq!(h.admit(context, 0x33), Err(StoreError::OutsideServingSlot)); + } + assert_eq!(h.store.counts().blocks, 0); + h.advance_ms(4000); + h.context(ROOT, 40, 2); + assert_eq!(h.insert(key(0, 0)).0.expires, h.start + Duration::from_secs(12)); +} + +#[test] +fn conflicting_and_oversized_contexts_do_not_replace_descriptors() { + let mut h = Harness::new(2, 1); + let context = h.context(ROOT, 0, 2); + let seq = h.store.context(&ROOT).unwrap().1.seq(); + assert_eq!(h.admit(context, 0x44), Err(StoreError::ConflictingContext)); + assert_eq!( + h.admit(CommitmentContext { blob_count: 1, ..context }, 0x33), + Err(StoreError::ConflictingContext) + ); + let mut header = [0x33; 208]; + header[..8].copy_from_slice(&context.slot.to_le_bytes()); + let data = ContextData::Fulu { + signed_header: &header, + inclusion_proof: &[0x33; 128], + commitments: &[0; 3 * 48], + }; + assert_eq!(h.store.admit_context(context, data), Err(StoreError::InvalidContext)); + assert_eq!(h.store.context(&ROOT).unwrap().1.seq(), seq); +} + +#[test] +fn zero_and_128_blob_masks_are_supported() { + assert_eq!(CellMask::all(0).bits(), 0); + assert_eq!(CellMask::all(128).bits(), u128::MAX); + assert!(!CellMask::all(128).contains(128)); + let mut h = Harness::new(128, 1); + h.context(ROOT, 0, 128); + for row in 0..128 { + assert_eq!(h.insert(key(0, row)).1, row == 127); + } + assert_eq!(h.store.column(&ROOT, 0).unwrap().available.bits(), u128::MAX); + + let mut h = Harness::new(0, 1); + h.context(ROOT, 0, 0); + assert!(h.store.column(&ROOT, 0).unwrap().complete); + assert!(matches!(h.admit_cell(key(0, 0), &CELL, &PROOF), Err(StoreError::UnknownCell))); +} + +#[test] +fn sparse_columns_and_invalid_indices_are_isolated() { + let mut h = Harness::new(2, (1 << 3) | (1 << 127)); + h.context(ROOT, 0, 2); + let first = h.insert(key(3, 0)).0; + let second = h.insert(key(127, 1)).0; + assert_ne!(first.read().seq(), second.read().seq()); + assert_eq!(h.store.column(&ROOT, 3).unwrap().available.bits(), 1); + assert_eq!(h.store.column(&ROOT, 127).unwrap().available.bits(), 2); + for invalid in [key(0, 0), key(128, 0), key(usize::MAX, 0), key(3, 2), key(3, usize::MAX)] { + assert!(h.store.cell(invalid).is_none()); + assert!(matches!(h.admit_cell(invalid, &CELL, &PROOF), Err(StoreError::UnknownCell))); + } + assert_eq!(h.store.counts().cells, 2); +} + +#[test] +fn admission_pressure_does_not_publish_failed_cells() { + let mut config = CellStoreConfig::new(Arc::new(Harness::spec(2)), 1, Duration::ZERO).unwrap(); + config.live_blocks = 1; + let mut h = Harness::configured(config); + h.context(ROOT, 0, 2); + h.insert(key(0, 0)); + let context = + CommitmentContext { block_root: [2; 32], slot: 0, format: ForkName::Fulu, blob_count: 2 }; + assert!(matches!(h.admit(context, 0x33), Err(StoreError::Full))); + assert_eq!(h.store.column(&ROOT, 0).unwrap().admitted.bits(), 1); + assert_eq!(h.store.counts().cells, 1); +} + +#[test] +fn slot_jumps_expire_the_previous_slot() { + let mut h = Harness::new(2, 1); + h.context(ROOT, 0, 2); + h.insert(key(0, 0)); + h.advance_ms(100_001); + assert_eq!(h.store.counts().active_slots, 0); + assert_eq!(h.store.counts().cells, 0); + assert_eq!(h.store.counts().contexts, 0); + assert_eq!(h.store.column(&ROOT, 0).unwrap().admitted.bits(), 1); + h.context([2; 32], 8, 2); + let cell = h.insert(CellKey { block_root: [2; 32], column: 0, row: 0 }).0; + assert_eq!(cell.expires, h.start + Duration::from_secs(108)); + assert_eq!(cell.slot, 8); +} + +#[test] +fn context_table_pressure_preserves_existing_blocks() { + let mut config = CellStoreConfig::new(Arc::new(Harness::spec(1)), 1, Duration::ZERO).unwrap(); + config.block_capacity = 1; + let mut h = Harness::configured(config); + h.store.blocks.reserve(4); + let capacity = h.store.blocks.capacity(); + assert!(capacity > h.store.config.block_capacity); + let context = h.context(ROOT, 0, 1); + assert_eq!( + h.admit(CommitmentContext { block_root: [2; 32], ..context }, 0x33), + Err(StoreError::Full) + ); + assert_eq!(h.store.context(&ROOT).unwrap().0, &context); + assert!(!h.admit(context, 0x33).unwrap()); + assert_eq!(h.store.blocks.len(), 1); + assert_eq!(h.store.blocks.capacity(), capacity); +} + +#[test] +fn expiry_reports_rows_and_slot_floor_prevents_readmission() { + let mut h = Harness::new(2, 1); + let context = h.context(ROOT, 0, 2); + h.insert(key(0, 0)); + h.advance(h.start + Duration::from_secs(1), 1, |_| {}); + assert!(h.store.cell(key(0, 0)).is_some(), "slot floor must not shorten serving"); + let mut expired = Vec::new(); + h.advance(h.start + Duration::from_secs(13), 1, |key| expired.push(key)); + assert_eq!(expired, [key(0, 0)]); + assert!(h.store.column(&ROOT, 0).is_none()); + h.advance(h.start + Duration::from_secs(14), 0, |_| {}); + assert_eq!(h.admit(context, 0x33), Err(StoreError::BelowSlotFloor)); +} + +#[test] +fn block_slot_reuse_does_not_alias_expired_cell_indices() { + let mut config = CellStoreConfig::new(Arc::new(Harness::spec(2)), 1, Duration::ZERO).unwrap(); + config.block_capacity = 1; + let mut h = Harness::configured(config); + h.context(ROOT, 0, 2); + h.insert(key(0, 0)); + h.advance(h.start + Duration::from_secs(13), 1, |_| {}); + h.context([2; 32], 1, 2); + let missing = CellKey { block_root: [2; 32], column: 0, row: 0 }; + assert!(h.store.cell(missing).is_none()); + let new = CellKey { row: 1, ..missing }; + h.insert(new); + assert!(h.store.cell(missing).is_none()); + assert!(h.store.cell(key(0, 0)).is_none()); + assert!(h.store.cell(new).is_some()); +} + +#[test] +fn slab_reuses_holes_without_moving_live_blocks() { + let mut config = + CellStoreConfig::new(Arc::new(Harness::spec(2)), 3, Duration::from_secs(11)).unwrap(); + config.block_capacity = 3; + let mut h = Harness::configured(config); + h.context(ROOT, 0, 2); + h.advance_ms(12_000); + h.context([2; 32], 1, 2); + let removed_index = h.store.roots[&[2; 32]]; + h.insert(CellKey { block_root: [2; 32], column: 0, row: 0 }); + h.advance(h.start + Duration::from_secs(24), 1, |_| {}); + h.context([3; 32], 2, 2); + h.context([4; 32], 2, 2); + let live_indices = [h.store.roots[&[3; 32]], h.store.roots[&[4; 32]]]; + let first_key = CellKey { block_root: [3; 32], column: 0, row: 0 }; + let first = h.insert(first_key).0; + let other = CellKey { block_root: [4; 32], column: 1, row: 1 }; + let second = h.insert(other).0; + + h.advance(h.start + Duration::from_secs(25), 2, |_| {}); + assert!(!h.store.roots.contains_key(&[2; 32])); + assert_eq!([h.store.roots[&[3; 32]], h.store.roots[&[4; 32]]], live_indices); + h.context([5; 32], 2, 2); + assert_eq!(h.store.roots[&[5; 32]], removed_index); + let new = CellKey { block_root: [5; 32], column: 0, row: 0 }; + assert!(h.store.cell(new).is_none()); + assert_eq!(h.store.column(&new.block_root, 0).unwrap().admitted.bits(), 0); + h.insert(new); + + assert_eq!(h.store.counts().blocks, 3); + assert_eq!(h.store.cell(first_key).unwrap().read().seq(), first.read().seq()); + assert_eq!(h.store.cell(other).unwrap().read().seq(), second.read().seq()); + assert_eq!(h.acquire_cell(other).unwrap().cell.as_ref(), CELL); +} + +#[test] +fn acquired_send_outlives_expiry_and_blocks_overwrite() { + let mut h = Harness::new(1, 1); + let mut outbound = Box::new(h.cache.strict_random_access("", true).unwrap()); + h.context(ROOT, 0, 1); + let reference = h.insert(key(0, 0)).0; + let sent = reference.acquire(&mut outbound).unwrap(); + let mut blocked = false; + let mut next_context = None; + for slot in 1..128 { + h.advance(h.start + Duration::from_secs(slot * 12), slot, |_| {}); + let context = CommitmentContext { + block_root: [slot as u8 + 1; 32], + slot, + format: h.store.config.spec.fork_at_slot(slot), + blob_count: 1, + }; + match h.admit(context, 0x33) { + Err(StoreError::CacheFull) => { + blocked = true; + next_context = Some(context); + break; + } + Ok(true) => {} + result => panic!("unexpected context admission: {result:?}"), + } + let key = CellKey { block_root: context.block_root, column: 0, row: 0 }; + match h.admit_cell(key, &CELL, &PROOF) { + Ok(CellAdmission::Inserted { cell, .. }) => drop(cell.acquire(&mut outbound).unwrap()), + Err(StoreError::CacheFull) => { + blocked = true; + next_context = Some(context); + break; + } + result => panic!("unexpected cell admission: {result:?}"), + } + } + assert!(blocked, "an ACK-held record must eventually block the producer"); + assert_eq!(sent.cell.as_ref(), CELL); + assert!(h.store.cell(key(0, 0)).is_none()); + drop(sent); + outbound.free(); + let context = next_context.unwrap(); + h.admit(context, 0x33).unwrap(); + h.admit_cell(CellKey { block_root: context.block_root, column: 0, row: 0 }, &CELL, &PROOF) + .unwrap(); +} + +#[test] +fn ingress_is_copied_before_validation_and_can_be_reused_immediately() { + let mut ingress = TCache::producer("", 1 << 17); + let mut incoming = Box::new(ingress.cache_ref().strict_random_access("", true).unwrap()); + let mut h = Harness::new(1, 1); + h.context(ROOT, 0, 1); + let mut reservation = ingress.reserve(CELL_RECORD_BYTES, true).unwrap(); + let bytes = reservation.buffer().unwrap(); + bytes[..BYTES_PER_CELL].copy_from_slice(&CELL); + bytes[BYTES_PER_CELL..].copy_from_slice(&PROOF); + reservation.increment_offset(CELL_RECORD_BYTES); + let old = reservation.read(); + let read = incoming.acquire_strict(old).unwrap(); + let bytes = read.buffer().unwrap().0; + let pending = h + .store + .stage_cell( + key(0, 0), + bytes[..BYTES_PER_CELL].try_into().unwrap(), + bytes[BYTES_PER_CELL..].try_into().unwrap(), + ) + .unwrap() + .unwrap(); + drop(read); + drop(reservation); + assert!(h.store.cell(key(0, 0)).is_none()); + assert_eq!(h.store.counts().cells, 0); + + for _ in 0..64 { + let mut reservation = ingress.reserve(4096, true).unwrap(); + reservation.buffer().unwrap().fill(0xcc); + reservation.increment_offset(4096); + drop(incoming.acquire_strict(reservation.read()).unwrap()); + } + assert!(old.len().is_err(), "the ingress buffer must actually have been reused"); + let validation = pending.data.acquire(&mut h.consumer).unwrap(); + assert_eq!(validation.buffers(), [&CELL[..], &PROOF[..]]); + validation.accept().unwrap(); + let update = h.store.refresh_column(&ROOT, 0).unwrap(); + assert_eq!(update.new_cells.bits(), 1); + assert!(update.column_completed); + assert_eq!(h.acquire_cell(key(0, 0)).unwrap().cell.as_ref(), CELL); +} + +#[test] +fn independent_ingress_writers_ignore_duplicates_and_retry_failed_validation() { + let mut h = Harness::new(2, 3); + let mut el = Box::new(h.cache.retained_random_access("").unwrap()); + h.context(ROOT, 0, 2); + let columns: Vec<_> = h.store.reservations(&ROOT).collect(); + assert_eq!(columns.len(), 2); + assert_eq!(columns[0].expires, h.start + Duration::from_secs(12)); + let old = h.store.stage_cell(key(0, 0), &[0; BYTES_PER_CELL], &PROOF).unwrap().unwrap(); + assert!(columns[0].stage(&mut el, 0, &CELL, &PROOF).unwrap().is_none()); + let validation = old.data.acquire(&mut h.consumer).unwrap(); + assert!(h.store.cell(key(0, 0)).is_none()); + assert!(!old.data.cancel(&mut el).unwrap()); + drop(validation); + let retry = columns[0].stage(&mut el, 0, &CELL, &PROOF).unwrap().unwrap(); + assert!(!h.store.cancel_pending(old).unwrap()); + assert!(matches!(old.data.acquire(&mut h.consumer), Err(SubReservationError::Stale))); + let validation = retry.data.acquire(&mut h.consumer).unwrap(); + assert_eq!(validation.buffers(), [&CELL[..], &PROOF[..]]); + validation.accept().unwrap(); + assert!(h.store.stage_cell(key(0, 0), &CELL, &PROOF).unwrap().is_none()); + assert_eq!(h.store.refresh_column(&ROOT, 0).unwrap().new_cells.bits(), 1); + assert_eq!(h.store.refresh_column(&ROOT, 0).unwrap().new_cells.bits(), 0); + assert_eq!(h.store.column(&ROOT, 1).unwrap().available.bits(), 0); +} + +#[test] +fn pending_validation_and_writes_survive_slot_expiry_without_publishing() { + let mut h = Harness::new(3, 1); + let mut writer = Box::new(h.cache.strict_random_access("", true).unwrap()); + h.context(ROOT, 0, 3); + let column = h.store.reservations(&ROOT).next().unwrap(); + let pending = column.stage(&mut writer, 0, &CELL, &PROOF).unwrap().unwrap(); + let queued = column.stage(&mut writer, 1, &CELL, &PROOF).unwrap().unwrap(); + let validation = pending.data.acquire(&mut h.consumer).unwrap(); + let acquired = column.reservation.acquire(&mut writer).unwrap(); + let writing = acquired.claim(2).unwrap(); + h.advance_ms(12_000); + assert_eq!(h.store.counts().cells, 0); + assert!(h.store.cell(key(0, 0)).is_none()); + assert_eq!(validation.buffers(), [&CELL[..], &PROOF[..]]); + assert_eq!(validation.accept(), Err(SubReservationError::Closed)); + assert!(matches!(queued.data.acquire(&mut writer), Err(SubReservationError::Closed))); + assert!(matches!(writing.write(&CELL, &PROOF), Err(SubReservationError::Closed))); + assert_eq!(acquired.ready(), 0); +} + +#[test] +fn full_sidecars_are_retained_without_copying_and_match_completed_assemblies() { + for slot in [0, 64] { + let config = + CellStoreConfig::new(Arc::new(Harness::spec(2)), 1 << 3, Duration::ZERO).unwrap(); + let mut h = Harness::at_slot(config, slot); + let mut network = Box::new(h.cache.retained_random_access("").unwrap()); + h.context(ROOT, slot, 2); + let (old_cell, _) = h.insert(key(3, 0)); + let assembly_send = old_cell.acquire(&mut network).unwrap(); + let pending = h + .store + .stage_cell(key(3, 1), &[0x12; BYTES_PER_CELL], &[0x23; BYTES_PER_KZG_PROOF]) + .unwrap() + .unwrap(); + let validation = pending.data.acquire(&mut h.consumer).unwrap(); + let full = h.full_bytes(&ROOT, 3); + let mut write = h.store.reserve_full(full.len()).unwrap(); + write.write_all(&full).unwrap(); + write.flush().unwrap(); + let read = write.read(); + drop(write); + let update = h.store.retain_full(&ROOT, 3, read).unwrap(); + assert!(update.column_completed); + assert_eq!(update.new_cells.bits(), 2); + assert_eq!(update.complete_read.unwrap().seq(), read.seq()); + let column = h.store.reservations(&ROOT).next().unwrap(); + assert_eq!(column.reservation.acquire(&mut network).unwrap().ready(), 1); + let cell = h.store.cell(key(3, 1)).unwrap(); + assert!(ptr::eq(&*cell.read().cache_ref(), &*h.cache)); + assert_eq!(cell.read().seq(), read.seq()); + let full_send = cell.acquire(&mut network).unwrap(); + assert_eq!(full_send.cell.as_ref(), &[0x12; BYTES_PER_CELL]); + assert_eq!(full_send.proof.as_ref(), &[0x23; BYTES_PER_KZG_PROOF]); + assert_eq!(h.store.counts().full_bytes, full.len()); + assert!(h.store.stage_cell(key(3, 1), &CELL, &PROOF).unwrap().is_none()); + + validation.accept().unwrap(); + assert!(!h.store.refresh_column(&ROOT, 3).unwrap().column_completed); + let entry = &h.store.columns[h.store.roots[&ROOT]]; + let complete = h + .store + .producer + .view_sub_reservation(entry.assembly.unwrap()) + .unwrap() + .finish() + .unwrap(); + let assembled = h.consumer.acquire_strict(complete).unwrap(); + assert_eq!(assembled.buffer().unwrap().0, full); + drop(assembled); + h.advance_ms(12_000); + assert!(h.store.cell(key(3, 1)).is_none()); + assert_eq!(h.store.counts().full_bytes, 0); + assert_eq!(assembly_send.cell.as_ref(), CELL); + assert_eq!(full_send.cell.as_ref(), &[0x12; BYTES_PER_CELL]); + } +} + +#[test] +fn mismatched_full_sidecars_cannot_replace_the_context_or_cell_source() { + for slot in [0, 64] { + let config = CellStoreConfig::new(Arc::new(Harness::spec(2)), 1, Duration::ZERO).unwrap(); + let mut h = Harness::at_slot(config, slot); + h.context(ROOT, slot, 2); + let original = h.insert(key(0, 0)).0; + let full = h.full_bytes(&ROOT, 0); + for corrupt_offset in [0, 8, 12, 24] { + let mut write = h.store.reserve_full(full.len()).unwrap(); + let bytes = write.buffer().unwrap(); + bytes.copy_from_slice(&full); + bytes[corrupt_offset] ^= 1; + write.flush().unwrap(); + let read = write.read(); + drop(write); + assert!(matches!(h.store.retain_full(&ROOT, 0, read), Err(StoreError::InvalidContext))); + assert_eq!(h.store.cell(key(0, 0)).unwrap().read().seq(), original.read().seq()); + assert_eq!(h.store.counts().full_bytes, 0); + } + } +} + +#[test] +fn expired_context_rejects_full_sidecars_without_changing_state() { + for slot in [0, 64] { + let config = CellStoreConfig::new(Arc::new(Harness::spec(2)), 1, Duration::ZERO).unwrap(); + let mut h = Harness::at_slot(config, slot); + h.context(ROOT, slot, 2); + h.insert(key(0, 0)); + let full = h.full_bytes(&ROOT, 0); + let mut write = h.store.reserve_full(full.len()).unwrap(); + write.write_all(&full).unwrap(); + write.flush().unwrap(); + let read = write.read(); + drop(write); + let pin = h.consumer.acquire_strict(read).unwrap(); + + h.advance_ms(12_000); + assert_eq!(pin.buffer().unwrap().0, full); + let before = h.store.counts(); + assert!(matches!(h.store.retain_full(&ROOT, 0, read), Err(StoreError::ContextExpired))); + assert_eq!(h.store.counts(), before); + assert!(!h.store.dirty); + assert!(h.store.columns[h.store.roots[&ROOT]].full.is_none()); + assert!(h.store.context(&ROOT).is_none()); + assert!(h.store.cell(key(0, 0)).is_none()); + let status = h.store.column(&ROOT, 0).unwrap(); + assert_eq!(status.admitted.bits(), 1); + assert_eq!(status.available.bits(), 0); + assert!(!status.complete); + } +} + +#[test] +fn rpc_sidecars_do_not_become_sendable_cells() { + let mut h = Harness::new(2, 1); + h.context(ROOT, 0, 2); + let full = h.full_bytes(&ROOT, 0); + let mut rpc = TCache::producer("", 1 << 16); + let mut reservation = rpc.reserve(full.len(), true).unwrap(); + reservation.write_all(&full).unwrap(); + assert!(matches!( + h.store.retain_full(&ROOT, 0, reservation.read()), + Err(StoreError::WrongCache) + )); + assert_eq!(h.store.counts().full_bytes, 0); + assert!(h.store.cell(key(0, 0)).is_none()); +} + +#[test] +fn dropping_the_store_closes_descriptors_without_invalidating_active_validation() { + let mut h = Harness::new(1, 1); + h.context(ROOT, 0, 1); + let column = h.store.reservations(&ROOT).next().unwrap(); + let pending = h.store.stage_cell(key(0, 0), &CELL, &PROOF).unwrap().unwrap(); + let Harness { store, mut consumer, .. } = h; + let validation = pending.data.acquire(&mut consumer).unwrap(); + drop(store); + assert!(matches!(column.reservation.acquire(&mut consumer), Err(SubReservationError::Closed))); + assert_eq!(validation.buffers(), [&CELL[..], &PROOF[..]]); + assert_eq!(validation.accept(), Err(SubReservationError::Closed)); +} diff --git a/crates/columns/src/counters.rs b/crates/columns/src/counters.rs index f55bb964..1c23c73e 100644 --- a/crates/columns/src/counters.rs +++ b/crates/columns/src/counters.rs @@ -18,5 +18,17 @@ silver_common::declare_counters! { KzgBatchesVerified, KzgBatchColumns, KzgBatchRejects, + CellStoreCapacity, + CellStoreLiveCells, + CellStoreLiveBytes, + CellStoreFullBytes, + CellStoreContexts, + CellStoreActiveSlots, + CellStoreBlocks, + CellStoreAdmissions, + CellStoreDuplicates, + CellStoreExpired, + CellStoreFull, + CellStoreCacheFull, } } diff --git a/crates/columns/src/lib.rs b/crates/columns/src/lib.rs index 4e04f805..727c8404 100644 --- a/crates/columns/src/lib.rs +++ b/crates/columns/src/lib.rs @@ -1,5 +1,6 @@ mod availability; mod batch; +pub mod cell_store; pub mod counters; mod el_blobs; mod sync; diff --git a/crates/columns/tests/cell_store_alloc.rs b/crates/columns/tests/cell_store_alloc.rs new file mode 100644 index 00000000..15f6521c --- /dev/null +++ b/crates/columns/tests/cell_store_alloc.rs @@ -0,0 +1,130 @@ +use std::{ + alloc::{GlobalAlloc, Layout, System}, + cell::Cell, + hint::black_box, + io::Write, + sync::Arc, + time::{Duration, Instant}, +}; + +use silver_beacon_state_data::{ForkName, SpecConfig}; +use silver_columns::cell_store::{ + CellKey, CellStore, CellStoreConfig, CommitmentContext, ContextData, +}; +use silver_common::{ + TCache, TCacheProducer, + ssz_view::{BYTES_PER_CELL, BYTES_PER_KZG_PROOF}, +}; + +thread_local! { + static ALLOCATION_EVENTS: Cell = const { Cell::new(0) }; +} + +struct CountingAllocator; + +unsafe impl GlobalAlloc for CountingAllocator { + unsafe fn alloc(&self, layout: Layout) -> *mut u8 { + ALLOCATION_EVENTS.with(|count| count.set(count.get() + 1)); + unsafe { System.alloc(layout) } + } + + unsafe fn dealloc(&self, ptr: *mut u8, layout: Layout) { + unsafe { System.dealloc(ptr, layout) } + } + + unsafe fn alloc_zeroed(&self, layout: Layout) -> *mut u8 { + ALLOCATION_EVENTS.with(|count| count.set(count.get() + 1)); + unsafe { System.alloc_zeroed(layout) } + } + + unsafe fn realloc(&self, ptr: *mut u8, layout: Layout, new_size: usize) -> *mut u8 { + ALLOCATION_EVENTS.with(|count| count.set(count.get() + 1)); + unsafe { System.realloc(ptr, layout, new_size) } + } +} + +#[global_allocator] +static ALLOCATOR: CountingAllocator = CountingAllocator; + +#[test] +fn cell_admission_expiry_and_block_churn_allocate_nothing() { + let spec = Arc::new(SpecConfig { + fulu_fork_epoch: 0, + max_blobs_per_block_electra: 2, + blob_schedule: Vec::new(), + slot_duration_ms: Some(1000), + ..SpecConfig::mainnet() + }); + let config = CellStoreConfig::new(spec, 3, Duration::ZERO).unwrap(); + let producer = TCache::producer("", config.cache_capacity()); + let mut writer = Box::new(producer.cache_ref().retained_random_access("").unwrap()); + let mut network = Box::new(producer.cache_ref().retained_random_access("").unwrap()); + let now = Instant::now(); + let mut store = CellStore::new(config, producer, 0, now).unwrap(); + let cell = [0x11; BYTES_PER_CELL]; + let proof = [0x22; BYTES_PER_KZG_PROOF]; + let before = ALLOCATION_EVENTS.with(Cell::get); + assert!(before > 0); + + for slot in 0u64..512 { + store.advance(now + Duration::from_secs(slot), slot.saturating_sub(63), |_| {}); + if let Some(event) = store.take_retention_event() { + writer.advance_retention(event.retain_from); + network.advance_retention(event.retain_from); + } + let mut block_root = [0; 32]; + block_root[..8].copy_from_slice(&slot.to_le_bytes()); + let context = CommitmentContext { block_root, slot, format: ForkName::Fulu, blob_count: 2 }; + let mut header = [0x33; 208]; + header[..8].copy_from_slice(&slot.to_le_bytes()); + let data = ContextData::Fulu { + signed_header: &header, + inclusion_proof: &[0x33; 128], + commitments: &[0x33; 2 * 48], + }; + assert!(store.admit_context(context, data).unwrap()); + for column in 0..2 { + let reservation = store.reservations(&block_root).nth(column).unwrap(); + for row in 0..2 { + let key = CellKey { block_root, column, row }; + let pending = store.stage_cell(key, &cell, &proof).unwrap().unwrap(); + drop(pending.data.acquire(&mut writer).unwrap()); + let retry = reservation.stage(&mut writer, row, &cell, &proof).unwrap().unwrap(); + assert!(!store.cancel_pending(pending).unwrap()); + let validation = retry.data.acquire(&mut writer).unwrap(); + black_box(validation.buffers()); + validation.accept().unwrap(); + assert_eq!( + store.refresh_column(&block_root, column).unwrap().column_completed, + row == 1 + ); + assert!(reservation.stage(&mut writer, row, &cell, &proof).unwrap().is_none()); + let acquired = store.cell(key).unwrap().acquire(&mut network).unwrap(); + black_box(acquired.cell.as_ref()); + black_box(acquired.proof.as_ref()); + } + let completed = + store.refresh_column(&block_root, column).unwrap().complete_read.unwrap(); + let assembly = writer.acquire_strict(completed).unwrap(); + let bytes = assembly.buffer().unwrap().0; + let mut full = store.reserve_full(bytes.len()).unwrap(); + full.write_all(bytes).unwrap(); + full.flush().unwrap(); + let read = full.read(); + drop(full); + assert!(!store.retain_full(&block_root, column, read).unwrap().column_completed); + let cell = store.cell(CellKey { block_root, column, row: 0 }).unwrap(); + let send = cell.acquire(&mut network).unwrap(); + black_box(send.cell.as_ref()); + black_box(send.proof.as_ref()); + } + black_box(store.column(&block_root, 0).unwrap()); + } + store.advance(now + Duration::from_secs(512), 512, |_| {}); + let event = store.take_retention_event().unwrap(); + writer.advance_retention(event.retain_from); + network.advance_retention(event.retain_from); + assert_eq!(store.counts().cells, 0); + assert_eq!(store.counts().blocks, 0); + assert_eq!(ALLOCATION_EVENTS.with(Cell::get) - before, 0); +} diff --git a/crates/common/src/cells.rs b/crates/common/src/cells.rs new file mode 100644 index 00000000..7bd2aa5a --- /dev/null +++ b/crates/common/src/cells.rs @@ -0,0 +1,150 @@ +use std::{ + ptr, + time::{Duration, Instant}, +}; + +use silver_beacon_state_data::ForkName; + +use crate::{ + AcquiredRange, GossipTopic, MessageId, Nanos, P2pStreamId, PendingSubReservation, + SubReservationError, SubReservationRef, TCacheRead, TRandomAccess, + ssz_view::{BYTES_PER_CELL, BYTES_PER_KZG_PROOF}, +}; + +pub const GOSSIP_DELIVERY_RETENTION: Duration = Duration::from_secs(11); + +#[derive(Clone, Copy, Debug)] +pub struct RetentionEvent { + pub expired_slot: u64, + // Captured before the sole producer allocates any next-slot data. + pub retain_from: u64, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct CellKey { + pub block_root: [u8; 32], + pub column: usize, + pub row: usize, +} + +#[derive(Clone, Copy, Debug)] +pub enum CellSource { + Full { read: TCacheRead, cell: usize, proof: usize }, + Assembly { reservation: SubReservationRef, row: usize }, +} + +#[derive(Clone, Copy, Debug)] +pub struct CellRef { + pub source: CellSource, + pub slot: u64, + pub expires: Instant, +} + +impl CellRef { + pub fn read(self) -> TCacheRead { + match self.source { + CellSource::Full { read, .. } => read, + CellSource::Assembly { reservation, .. } => reservation.read(), + } + } + + pub fn acquire(self, consumer: &mut TRandomAccess) -> Option { + let [cell, proof] = match self.source { + CellSource::Assembly { reservation, row } => { + reservation.acquire(consumer).ok()?.ranges(row)? + } + CellSource::Full { read, cell, proof } => { + if !consumer.is_strict() || !ptr::eq(&*consumer.cache_ref(), &*read.cache_ref()) { + return None; + } + let pin = consumer.acquire_strict(read)?; + [pin.with_range(cell, BYTES_PER_CELL)?, pin.with_range(proof, BYTES_PER_KZG_PROOF)?] + } + }; + Some(AcquiredCell { cell, proof }) + } +} + +pub struct AcquiredCell { + pub cell: AcquiredRange, + pub proof: AcquiredRange, +} + +#[derive(Clone, Copy, Debug)] +pub struct ColumnRef { + pub block_root: [u8; 32], + pub column: usize, + pub reservation: SubReservationRef, + pub slot: u64, + pub expires: Instant, +} + +impl ColumnRef { + pub fn stage( + self, + consumer: &mut TRandomAccess, + row: usize, + cell: &[u8; BYTES_PER_CELL], + proof: &[u8; BYTES_PER_KZG_PROOF], + ) -> Result, SubReservationError> { + let acquired = self.reservation.acquire(consumer)?; + let claim = match acquired.claim(row) { + Ok(claim) => claim, + Err(SubReservationError::Claimed | SubReservationError::Published) => return Ok(None), + Err(error) => return Err(error), + }; + Ok(Some(PendingCell { + key: CellKey { block_root: self.block_root, column: self.column, row }, + data: claim.write(cell, proof)?, + })) + } +} + +#[derive(Clone, Copy, Debug)] +pub struct PendingCell { + pub key: CellKey, + pub data: PendingSubReservation, +} + +#[derive(Clone, Copy, Debug)] +pub enum CellOrigin { + Gossip { stream_id: P2pStreamId, topic: GossipTopic, message_id: MessageId, received: Nanos }, + El { request_id: u64 }, +} + +#[derive(Clone, Copy, Debug)] +pub struct CellValidationRequest { + pub pending: PendingCell, + pub origin: CellOrigin, + pub deadline: Instant, +} + +#[derive(Clone, Copy, Debug)] +pub enum CellValidationOutcome { + Accepted, + Rejected, + Ignored, +} + +#[derive(Clone, Copy, Debug)] +pub enum CellStoreEvent { + Context { + block_root: [u8; 32], + slot: u64, + format: ForkName, + blob_count: usize, + ssz: TCacheRead, + expires: Instant, + }, + Reservation(ColumnRef), + Available { + key: CellKey, + cell: CellRef, + }, + Validate(CellValidationRequest), + Cancel(PendingCell), + Validation { + request: CellValidationRequest, + outcome: CellValidationOutcome, + }, +} diff --git a/crates/common/src/spine.rs b/crates/common/src/spine.rs index 96637914..ec215703 100644 --- a/crates/common/src/spine.rs +++ b/crates/common/src/spine.rs @@ -28,6 +28,8 @@ pub use tcache::{ TCacheRead, TCacheRef, }; +use crate::cells::{CellStoreEvent, RetentionEvent}; + mod messages; mod stream_id; mod stream_protocol; @@ -71,6 +73,10 @@ pub struct SilverSpine { #[queue(size(2usize.pow(13)))] pub data_columns: SpineQueue, #[queue(size(2usize.pow(10)))] + pub retention: SpineQueue, + #[queue(size(2usize.pow(13)))] + pub cells: SpineQueue, + #[queue(size(2usize.pow(10)))] pub sync_target: SpineQueue, #[queue(size(2usize.pow(14)))] pub sync_needs: SpineQueue, diff --git a/crates/common/src/ticker.rs b/crates/common/src/ticker.rs index f8b08e99..6bafa970 100644 --- a/crates/common/src/ticker.rs +++ b/crates/common/src/ticker.rs @@ -117,6 +117,15 @@ impl SlotTicker { self.millis_since_genesis() / self.slot_ms } + pub fn current_slot_start(&self) -> (u64, Instant) { + let now = Instant::now(); + let since_genesis = + Duration::from_millis(self.anchor_genesis_ms) + now.duration_since(self.anchor); + let slot = (since_genesis.as_millis() / self.slot_ms as u128) as u64; + let into_slot = since_genesis - Duration::from_millis(slot * self.slot_ms); + (slot, now - into_slot) + } + /// Whether `slot` is current after extending both ends of its wall-clock /// interval by `disparity`. Consensus gossip validation uses this for its /// permitted clock skew around slot boundaries. @@ -237,6 +246,17 @@ mod tests { assert!(t.tick() == TickEvent::None); } + #[test] + fn slot_start_uses_the_same_anchor_as_the_wall_slot() { + let slot_duration = Duration::from_millis(1250); + let mut ticker = + SlotTicker::new(genesis_secs_ago(20), slot_duration, Duration::from_millis(500)); + ticker.set_since_genesis_ms(10 * 1250 + 750); + let (slot, start) = ticker.current_slot_start(); + assert_eq!(slot, 10); + assert_eq!(start, ticker.anchor - Duration::from_millis(750)); + } + #[test] fn skips_stale_phases_on_catchup() { let slot_dur = Duration::from_secs(12); diff --git a/crates/control/Cargo.toml b/crates/control/Cargo.toml index 287dd13b..4565d508 100644 --- a/crates/control/Cargo.toml +++ b/crates/control/Cargo.toml @@ -11,6 +11,7 @@ bytes.workspace = true flux.workspace = true raft.workspace = true silver_chain_spec.workspace = true +silver_columns.workspace = true silver_common.workspace = true silver_gossip.workspace = true silver_peer.workspace = true diff --git a/crates/control/src/cell_ingress.rs b/crates/control/src/cell_ingress.rs new file mode 100644 index 00000000..afeedd8d --- /dev/null +++ b/crates/control/src/cell_ingress.rs @@ -0,0 +1,103 @@ +use std::time::Instant; + +use flux::spine::SpineProducers; +use silver_columns::cell_store::{ + CellStore, CellStoreConfig, CommitmentContext, ContextData, StoreError, +}; +use silver_common::{ + SilverSpineProducers, TProducer, + cells::{CellKey, CellStoreEvent, CellValidationOutcome, PendingCell}, +}; + +pub struct CellIngress { + store: CellStore, + min_slot: u64, +} + +impl CellIngress { + pub fn new( + config: CellStoreConfig, + producer: TProducer, + slot: u64, + slot_start: Instant, + ) -> Result { + Ok(Self { store: CellStore::new(config, producer, slot, slot_start)?, min_slot: 0 }) + } + + pub fn set_min_slot(&mut self, min_slot: u64) { + self.min_slot = self.min_slot.max(min_slot); + } + + pub fn spin(&mut self, now: Instant, producers: &SilverSpineProducers) { + self.store.advance(now, self.min_slot, |_| {}); + if let Some(event) = self.store.take_retention_event() { + producers.produce(event); + } + } + + pub fn admit_context( + &mut self, + context: CommitmentContext, + data: ContextData<'_>, + now: Instant, + producers: &SilverSpineProducers, + ) -> Result { + self.spin(now, producers); + if !self.store.admit_context(context, data)? { + return Ok(false); + } + let expires = self.store.slot_end(); + let (_, ssz) = self.store.context(&context.block_root).unwrap(); + producers.produce(CellStoreEvent::Context { + block_root: context.block_root, + slot: context.slot, + format: context.format, + blob_count: context.blob_count, + ssz, + expires, + }); + for column in self.store.reservations(&context.block_root) { + producers.produce(CellStoreEvent::Reservation(column)); + } + Ok(true) + } + + pub fn handle( + &mut self, + event: CellStoreEvent, + now: Instant, + producers: &SilverSpineProducers, + ) { + match event { + CellStoreEvent::Cancel(pending) => { + self.cancel(pending, now); + } + CellStoreEvent::Validation { request, outcome: CellValidationOutcome::Accepted } => { + self.spin(now, producers); + let key = request.pending.key; + let Ok(update) = self.store.refresh_column(&key.block_root, key.column) else { + return; + }; + let mut rows = update.new_cells.bits(); + while rows != 0 { + let row = rows.trailing_zeros() as usize; + rows &= rows - 1; + let key = CellKey { row, ..key }; + if let Some(cell) = self.store.cell(key) { + producers.produce(CellStoreEvent::Available { key, cell }); + } + } + } + _ => {} + } + } + + pub fn cancel(&mut self, pending: PendingCell, now: Instant) -> bool { + self.store.advance(now, self.min_slot, |_| {}); + self.store.cancel_pending(pending).unwrap_or(false) + } + + pub fn store_mut(&mut self) -> &mut CellStore { + &mut self.store + } +} diff --git a/crates/control/src/lib.rs b/crates/control/src/lib.rs index b2dbe010..f6642f73 100644 --- a/crates/control/src/lib.rs +++ b/crates/control/src/lib.rs @@ -1,4 +1,5 @@ pub mod cluster; +pub mod cell_ingress; mod counters; pub mod sync_engine; mod tile; diff --git a/crates/control/tests/cell_ingress.rs b/crates/control/tests/cell_ingress.rs new file mode 100644 index 00000000..8b657640 --- /dev/null +++ b/crates/control/tests/cell_ingress.rs @@ -0,0 +1,399 @@ +use std::{ + array, + io::Write, + ptr, + sync::Arc, + time::{Duration, Instant}, +}; + +use flux::{ + spine::{SpineAdapter, SpineProducers}, + tile::Tile, +}; +use silver_chain_spec::{ForkName, SpecConfig}; +use silver_columns::cell_store::{CellStoreConfig, CommitmentContext, ContextData}; +use silver_common::{ + GossipTopic, MessageId, Nanos, P2pStreamId, SilverSpine, StreamProtocol, SubReservationError, + TCache, TCacheProducer, TCacheRead, TCacheRef, TRandomAccess, + cells::{ + CellKey, CellOrigin, CellSource, CellStoreEvent, CellValidationOutcome, + CellValidationRequest, ColumnRef, RetentionEvent, + }, + column_util::push_data_column_sidecar_prefix, + ssz_view::{BYTES_PER_CELL, BYTES_PER_KZG_PROOF}, +}; +use silver_control::cell_ingress::CellIngress; +use tempfile::TempDir; + +const ROOT: [u8; 32] = [1; 32]; +const PROOF: [u8; BYTES_PER_KZG_PROOF] = [0x22; BYTES_PER_KZG_PROOF]; +const SLOT: Duration = Duration::from_secs(12); + +struct Endpoint; + +impl Tile for Endpoint { + fn loop_body(&mut self, _: &mut SpineAdapter) {} +} + +struct Rig { + control: CellIngress, + columns: Box, + network: Box, + adapters: [SpineAdapter; 3], + cache: TCacheRef, + now: Instant, + start: Instant, + context: CommitmentContext, + _spine: Box, + _directory: TempDir, +} + +impl Rig { + fn new(format: ForkName) -> Self { + let spec = Arc::new(SpecConfig { + fulu_fork_epoch: 0, + gloas_fork_epoch: if format == ForkName::Gloas { 0 } else { u64::MAX }, + max_blobs_per_block_electra: 2, + blob_schedule: Vec::new(), + ..SpecConfig::mainnet() + }); + let config = CellStoreConfig::new(spec, 7, Duration::from_secs(11)).unwrap(); + let producer = TCache::producer("", config.cache_capacity()); + let cache = producer.cache_ref(); + let columns = Box::new(cache.retained_random_access("").unwrap()); + let network = Box::new(cache.retained_random_access("").unwrap()); + let now = Instant::now(); + let directory = tempfile::tempdir().unwrap(); + let mut spine = Box::new(SilverSpine::new_with_base_dir(directory.path(), None)); + let adapters = array::from_fn(|_| { + let mut adapter = SpineAdapter::connect_tile(&Endpoint, &mut *spine); + adapter.consume(|_: RetentionEvent, _| {}); + adapter.consume(|_: CellStoreEvent, _| {}); + adapter + }); + Self { + control: CellIngress::new(config, producer, 0, now).unwrap(), + columns, + network, + adapters, + cache, + now, + start: now, + context: CommitmentContext { block_root: ROOT, slot: 0, format, blob_count: 2 }, + _spine: spine, + _directory: directory, + } + } + + fn admit(&mut self) { + let mut header = [0x33; 208]; + header[..8].copy_from_slice(&self.context.slot.to_le_bytes()); + let data = if self.context.format == ForkName::Fulu { + ContextData::Fulu { + signed_header: &header, + inclusion_proof: &[0x33; 128], + commitments: &[0x33; 96], + } + } else { + ContextData::Gloas { commitments: &[0x33; 96] } + }; + assert!( + self.control + .admit_context(self.context, data, self.now, &self.adapters[0].producers) + .unwrap() + ); + } + + fn reservations(&mut self) -> Vec { + let mut columns = Vec::new(); + let mut contexts = 0; + self.adapters[1].consume(|event: CellStoreEvent, _| match event { + CellStoreEvent::Context { block_root, format, blob_count, ssz, .. } => { + assert_eq!(block_root, ROOT); + assert_eq!(format, self.context.format); + assert_eq!(blob_count, 2); + assert!(ptr::eq(&*ssz.cache_ref(), &*self.cache)); + contexts += 1; + } + CellStoreEvent::Reservation(column) => { + assert!(ptr::eq(&*column.reservation.read().cache_ref(), &*self.cache)); + columns.push(column); + } + _ => {} + }); + assert_eq!(contexts, 1); + columns + } + + fn write(&mut self, len: usize, byte: u8) -> TCacheRead { + let mut write = self.control.store_mut().reserve_full(len).unwrap(); + write.buffer().unwrap().fill(byte); + write.flush().unwrap(); + write.read() + } + + fn fill(&mut self) { + let mut count = 0; + while let Ok(mut write) = self.control.store_mut().reserve_full(8192) { + write.buffer().unwrap().fill(0xcc); + write.flush().unwrap(); + count += 1; + assert!(count <= self.cache.capacity() / 8192 + 2); + } + } + + fn expire(&mut self) -> RetentionEvent { + self.control.spin(self.now, &self.adapters[0].producers); + let mut boundary = None; + self.adapters[1].consume(|event: RetentionEvent, _| { + assert!(boundary.is_none()); + self.columns.advance_retention(event.retain_from); + boundary = Some(event); + }); + boundary.unwrap() + } + + fn network_boundaries(&mut self) { + self.adapters[2].consume(|event: RetentionEvent, _| { + self.network.advance_retention(event.retain_from); + }); + } + + fn full_bytes(&self, column: usize) -> Vec { + let mut bytes = Vec::new(); + if self.context.format == ForkName::Fulu { + let mut header = [0x33; 208]; + header[..8].copy_from_slice(&self.context.slot.to_le_bytes()); + push_data_column_sidecar_prefix(&mut bytes, column as u64, 2, &header, &[0x33; 128]); + } else { + bytes.extend_from_slice(&(column as u64).to_le_bytes()); + bytes.extend_from_slice(&56u32.to_le_bytes()); + bytes.extend_from_slice(&((56 + 2 * BYTES_PER_CELL) as u32).to_le_bytes()); + bytes.extend_from_slice(&self.context.slot.to_le_bytes()); + bytes.extend_from_slice(&ROOT); + } + for row in 0..2 { + bytes.extend_from_slice(&[0x11 + row as u8; BYTES_PER_CELL]); + } + if self.context.format == ForkName::Fulu { + bytes.extend_from_slice(&[0x33; 96]); + } + bytes.extend_from_slice(&[0x22; 2 * BYTES_PER_KZG_PROOF]); + bytes + } +} + +#[test] +fn gossip_full_sidecars_and_cells_share_one_cache_through_validation_and_expiry() { + for format in [ForkName::Fulu, ForkName::Gloas] { + let mut rig = Rig::new(format); + let expires = rig.start + SLOT; + let bytes = rig.full_bytes(0); + let mut full = rig.control.store_mut().reserve_full(bytes.len()).unwrap(); + full.write_all(&bytes).unwrap(); + full.flush().unwrap(); + let full_read = full.read(); + drop(full); + + // The first full sidecar precedes the block's assembly reservations. + rig.admit(); + let columns = rig.reservations(); + assert_eq!(columns.len(), 3); + assert!(columns[0].reservation.read().seq() > full_read.seq()); + let validation = rig.columns.acquire_strict(full_read).unwrap(); + assert_eq!(validation.buffer().unwrap().0, bytes); + drop(validation); + // Cryptographic validation is outside this ownership fixture. + rig.control.store_mut().retain_full(&ROOT, 0, full_read).unwrap(); + + for row in 0..2 { + let pending = if row == 0 { + rig.control + .store_mut() + .stage_cell( + CellKey { block_root: ROOT, column: 1, row }, + &[0x11; BYTES_PER_CELL], + &PROOF, + ) + .unwrap() + .unwrap() + } else { + columns[1] + .stage(&mut rig.columns, row, &[0x12; BYTES_PER_CELL], &PROOF) + .unwrap() + .unwrap() + }; + let origin = if row == 0 { + CellOrigin::Gossip { + stream_id: P2pStreamId::new(42, 7, StreamProtocol::GossipSub, true), + topic: GossipTopic::DataColumnSidecar(1), + message_id: MessageId::default(), + received: Nanos::now(), + } + } else { + CellOrigin::El { request_id: 99 } + }; + rig.adapters[row].produce(CellStoreEvent::Validate(CellValidationRequest { + pending, + origin, + deadline: expires, + })); + } + + let mut validated = 0; + rig.adapters[1].consume(|event: CellStoreEvent, producers| { + if let CellStoreEvent::Validate(request) = event { + let validation = request.pending.data.acquire(&mut rig.columns).unwrap(); + assert_eq!( + validation.buffers()[0], + &[0x11 + request.pending.key.row as u8; BYTES_PER_CELL] + ); + validation.accept().unwrap(); + producers.produce(CellStoreEvent::Validation { + request, + outcome: CellValidationOutcome::Accepted, + }); + validated += 1; + } + }); + assert_eq!(validated, 2); + rig.adapters[0].consume(|event: CellStoreEvent, producers| { + rig.control.handle(event, rig.now, producers); + }); + let mut available = 0; + rig.adapters[2].consume(|event: CellStoreEvent, _| { + if let CellStoreEvent::Available { cell, .. } = event { + assert!(ptr::eq(&*cell.read().cache_ref(), &*rig.cache)); + let acquired = cell.acquire(&mut rig.network).unwrap(); + assert_eq!(acquired.proof.as_ref(), PROOF); + available += 1; + } + }); + assert_eq!(available, 2); + + let full = + rig.control.store_mut().cell(CellKey { block_root: ROOT, column: 0, row: 0 }).unwrap(); + let assembly = + rig.control.store_mut().cell(CellKey { block_root: ROOT, column: 1, row: 0 }).unwrap(); + assert!(matches!(full.source, CellSource::Full { .. })); + assert!(matches!(assembly.source, CellSource::Assembly { .. })); + assert_eq!(full.read().seq(), full_read.seq()); + assert!(ptr::eq(&*full.read().cache_ref(), &*assembly.read().cache_ref())); + + for _ in 0..rig.cache.capacity() / 8192 / 2 { + let newer = rig.write(8192, 0xcc); + drop(rig.columns.acquire_strict(newer).unwrap()); + drop(rig.network.acquire_strict(newer).unwrap()); + } + rig.now = expires - Duration::from_nanos(1); + rig.control.spin(rig.now, &rig.adapters[0].producers); + rig.adapters[2].consume(|_: RetentionEvent, _| panic!("expired before the slot boundary")); + let full_send = full.acquire(&mut rig.network).unwrap(); + let assembly_send = assembly.acquire(&mut rig.network).unwrap(); + assert_eq!(full_send.cell.as_ref(), &[0x11; BYTES_PER_CELL]); + assert_eq!(assembly_send.cell.as_ref(), &[0x11; BYTES_PER_CELL]); + + rig.now = expires; + let event = rig.expire(); + assert_eq!(event.expired_slot, 0); + rig.network_boundaries(); + assert_eq!(rig.control.store_mut().counts().cells, 0); + assert_eq!(full_send.cell.as_ref(), &[0x11; BYTES_PER_CELL]); + assert_eq!(assembly_send.cell.as_ref(), &[0x11; BYTES_PER_CELL]); + assert!(columns[1].reservation.acquire(&mut rig.columns).is_err()); + } +} + +#[test] +fn cancellation_cannot_reset_a_retry_or_an_active_validator() { + let mut rig = Rig::new(ForkName::Fulu); + rig.admit(); + let column = rig.reservations()[0]; + let key = CellKey { block_root: ROOT, column: 0, row: 0 }; + let old = + rig.control.store_mut().stage_cell(key, &[0x99; BYTES_PER_CELL], &PROOF).unwrap().unwrap(); + let validation = old.data.acquire(&mut rig.columns).unwrap(); + drop(validation); + let retry = + rig.control.store_mut().stage_cell(key, &[0x11; BYTES_PER_CELL], &PROOF).unwrap().unwrap(); + assert!(!rig.control.cancel(old, rig.now)); + rig.adapters[1].produce(CellStoreEvent::Cancel(retry)); + rig.adapters[0].consume(|event: CellStoreEvent, producers| { + rig.control.handle(event, rig.now, producers); + }); + let pending = + column.stage(&mut rig.columns, 0, &[0x11; BYTES_PER_CELL], &PROOF).unwrap().unwrap(); + let validation = pending.data.acquire(&mut rig.columns).unwrap(); + assert!(!rig.control.cancel(pending, rig.now)); + assert_eq!(validation.buffers()[0], &[0x11; BYTES_PER_CELL]); + validation.accept().unwrap(); +} + +#[test] +fn delayed_expiry_preserves_next_slot_data_and_newer_events_recover_missed_ones() { + let mut rig = Rig::new(ForkName::Fulu); + for _ in 0..8 { + rig.write(8192, 0x11); + } + rig.now += SLOT; + let old = rig.expire(); + let next = rig.write(32, 0x22); + for _ in 0..8 { + let newer = rig.write(8192, 0x33); + drop(rig.network.acquire_strict(newer).unwrap()); + } + rig.network_boundaries(); + assert_eq!(rig.network.acquire_strict(next).unwrap().buffer().unwrap().0, &[0x22; 32]); + rig.now += SLOT; + let missed = rig.expire(); + for _ in 0..8 { + rig.write(8192, 0x33); + } + rig.now += SLOT; + let latest = rig.expire(); + // The latest boundary is sufficient even if earlier notifications are missed. + rig.network.advance_retention(latest.retain_from); + rig.network.advance_retention(missed.retain_from); + rig.network.advance_retention(old.retain_from); + for _ in 0..20 { + rig.write(8192, 0x44); + } +} + +#[test] +fn active_writers_validators_and_sends_survive_expiry_without_store_pins() { + let mut rig = Rig::new(ForkName::Fulu); + rig.admit(); + let columns = rig.reservations(); + let key = CellKey { block_root: ROOT, column: 0, row: 0 }; + let pending = + rig.control.store_mut().stage_cell(key, &[0x11; BYTES_PER_CELL], &PROOF).unwrap().unwrap(); + pending.data.acquire(&mut rig.columns).unwrap().accept().unwrap(); + rig.control.store_mut().refresh_column(&ROOT, 0).unwrap(); + let send = rig.control.store_mut().cell(key).unwrap().acquire(&mut rig.network).unwrap(); + let pending = + columns[0].stage(&mut rig.columns, 1, &[0x22; BYTES_PER_CELL], &PROOF).unwrap().unwrap(); + let validation = pending.data.acquire(&mut rig.columns).unwrap(); + let writer = columns[1].reservation.acquire(&mut rig.columns).unwrap(); + let writing = writer.claim(0).unwrap(); + rig.now += SLOT; + rig.expire(); + rig.network_boundaries(); + rig.fill(); + assert_eq!(send.cell.as_ref(), &[0x11; BYTES_PER_CELL]); + assert_eq!(validation.buffers()[0], &[0x22; BYTES_PER_CELL]); + drop(send); + assert!(rig.control.store_mut().reserve_full(8192).is_err()); + drop(validation); + rig.fill(); + rig.now += SLOT; + rig.expire(); + rig.network_boundaries(); + assert!(rig.control.store_mut().reserve_full(8192).is_err()); + assert!(matches!( + writing.write(&[0x33; BYTES_PER_CELL], &PROOF), + Err(SubReservationError::Closed) + )); + drop(writer); + assert!(rig.control.store_mut().reserve_full(8192).is_ok()); +} From 84635793a4a13ce9f323ac770e5eeda07f8d6a3e Mon Sep 17 00:00:00 2001 From: vladimir-ea Date: Fri, 11 Sep 2026 16:11:14 +0100 Subject: [PATCH 3/5] fmt --- crates/common/src/lib.rs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/crates/common/src/lib.rs b/crates/common/src/lib.rs index c1f8d929..659dbd9b 100644 --- a/crates/common/src/lib.rs +++ b/crates/common/src/lib.rs @@ -46,6 +46,8 @@ pub use crate::{ wheel::Wheel, wither::{CountingWitherFilter, WitherFilter}, }; + +pub mod cells; pub mod column_util; mod enr; mod error; From 0b12c3214c34c9e77193597aa3a9e2f5f35042ec Mon Sep 17 00:00:00 2001 From: vladimir-ea Date: Fri, 11 Sep 2026 21:02:33 +0100 Subject: [PATCH 4/5] merge --- crates/control/src/lib.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/control/src/lib.rs b/crates/control/src/lib.rs index f6642f73..7f336888 100644 --- a/crates/control/src/lib.rs +++ b/crates/control/src/lib.rs @@ -1,5 +1,5 @@ -pub mod cluster; pub mod cell_ingress; +pub mod cluster; mod counters; pub mod sync_engine; mod tile; From cba825d039e24884953261ef505cd2f4f2c76b75 Mon Sep 17 00:00:00 2001 From: vladimir-ea Date: Tue, 15 Sep 2026 10:11:34 +0100 Subject: [PATCH 5/5] rebase snafu --- crates/common/src/spine/tcache.rs | 2 - .../common/src/spine/tcache/gossip_frame.rs | 299 ------------ .../src/spine/tcache/gossip_frame/acquired.rs | 141 ------ .../src/spine/tcache/gossip_frame/tests.rs | 445 ------------------ 4 files changed, 887 deletions(-) delete mode 100644 crates/common/src/spine/tcache/gossip_frame.rs delete mode 100644 crates/common/src/spine/tcache/gossip_frame/acquired.rs delete mode 100644 crates/common/src/spine/tcache/gossip_frame/tests.rs diff --git a/crates/common/src/spine/tcache.rs b/crates/common/src/spine/tcache.rs index 944c25e7..ff148ab3 100644 --- a/crates/common/src/spine/tcache.rs +++ b/crates/common/src/spine/tcache.rs @@ -16,7 +16,6 @@ pub use consumer::{ AcquiredRange, AcquiredRead, AcquiredWithOffset, Consumer, RandomAccessConsumer, TCacheRead, }; use flux::{Timer, timing::Nanos, tracing}; - pub use producer::{MultiProducer, Producer, Reservation, TCacheProducer}; pub use sub_reservation::{ AcquiredSubReservation, PendingSubReservation, SubLayout, SubReservation, SubReservationError, @@ -47,7 +46,6 @@ const fn lag_threshold(len: u32) -> u64 { mod cache_frame; mod consumer; -mod gossip_frame; mod metrics; mod producer; mod sub_reservation; diff --git a/crates/common/src/spine/tcache/gossip_frame.rs b/crates/common/src/spine/tcache/gossip_frame.rs deleted file mode 100644 index ff30daec..00000000 --- a/crates/common/src/spine/tcache/gossip_frame.rs +++ /dev/null @@ -1,299 +0,0 @@ -use std::{io::Write, ops::Range, time::Instant}; - -use super::{ - AcquiredRange, AcquiredRead, Producer, RandomAccessConsumer, SubReservationRef, TCacheProducer, - TCacheRead, -}; -use crate::MAX_GOSSIP_FRAME_SIZE; - -mod acquired; -pub use acquired::{AcquiredGossipFrame, AcquiredGossipSegment}; - -pub const MAX_GOSSIP_SEGMENTS: usize = 2 * 128 + 8; -const HEADER_BYTES: usize = 16; -const SEGMENT_BYTES: usize = 40; -const MAGIC: [u8; 8] = *b"SGFRAME1"; - -#[derive(Clone, Copy, Debug, PartialEq, Eq)] -pub enum GossipFrameError { - InvalidDescriptor, - TooLarge, - CacheFull, - Expired, - Stale, -} - -#[derive(Clone, Copy, Debug)] -pub enum GossipSegment { - Framing { - offset: usize, - length: usize, - }, - Gossip { - read: TCacheRead, - offset: usize, - length: usize, - }, - DataColumns { - read: TCacheRead, - offset: usize, - length: usize, - }, - Shared { - reservation: SubReservationRef, - part: usize, - second: bool, - offset: usize, - length: usize, - }, -} - -// Only the builder constructs this handle. Encoded source identities originate -// from typed cache descriptors, never from network input. -// Pointer identities and trusted layout metadata make this an in-process -// format. -#[derive(Clone, Copy, Debug)] -pub struct GossipFrameRef { - descriptor: TCacheRead, - expires: Instant, -} - -impl GossipFrameRef { - pub fn write( - producer: &mut Producer, - expires: Instant, - framing: &[u8], - segments: impl ExactSizeIterator, - ) -> Result { - let count = segments.len(); - if count == 0 || count > MAX_GOSSIP_SEGMENTS || framing.len() > MAX_GOSSIP_FRAME_SIZE { - return Err(GossipFrameError::TooLarge); - } - let framing_start = HEADER_BYTES + count * SEGMENT_BYTES; - let descriptor_len = framing_start + framing.len(); - let cache = producer.cache_ref(); - let mut reservation = - producer.reserve(descriptor_len, false).ok_or(GossipFrameError::CacheFull)?; - let buffer = reservation.buffer().map_err(|_| GossipFrameError::Stale)?; - buffer[..8].copy_from_slice(&MAGIC); - buffer[8..12].copy_from_slice(&(count as u32).to_le_bytes()); - buffer[framing_start..].copy_from_slice(framing); - let mut total = 0usize; - let mut written = 0; - for (index, segment) in segments.enumerate() { - if index >= count { - return Err(GossipFrameError::InvalidDescriptor); - } - let (kind, read, metadata, offset, length) = match segment { - GossipSegment::Framing { offset, length } => { - if offset.checked_add(length).is_none_or(|end| end > framing.len()) { - return Err(GossipFrameError::InvalidDescriptor); - } - (0u64, None, 0, offset, length) - } - GossipSegment::Gossip { read, offset, length } => { - if read.tcache.cache != cache.cache { - return Err(GossipFrameError::InvalidDescriptor); - } - (1, Some(read), 0, offset, length) - } - GossipSegment::DataColumns { read, offset, length } => { - (2, Some(read), 0, offset, length) - } - GossipSegment::Shared { reservation, part, second, offset, length } => { - if part >= 128 { - return Err(GossipFrameError::InvalidDescriptor); - } - let metadata = ((reservation.header_bytes as u64) << 32) | - ((part as u64) << 1) | - u64::from(second); - (3, Some(reservation.read()), metadata, offset, length) - } - }; - if length == 0 || offset > u32::MAX as usize || length > u32::MAX as usize { - return Err(GossipFrameError::InvalidDescriptor); - } - total = total.checked_add(length).ok_or(GossipFrameError::TooLarge)?; - if total > MAX_GOSSIP_FRAME_SIZE { - return Err(GossipFrameError::TooLarge); - } - let start = HEADER_BYTES + index * SEGMENT_BYTES; - let entry = &mut buffer[start..start + SEGMENT_BYTES]; - entry[..8].copy_from_slice(&kind.to_le_bytes()); - entry[8..16] - .copy_from_slice(&read.map_or(0, |r| r.tcache.cache as usize as u64).to_le_bytes()); - entry[16..24].copy_from_slice(&read.map_or(0, |r| r.seq).to_le_bytes()); - entry[24..32].copy_from_slice(&metadata.to_le_bytes()); - entry[32..36].copy_from_slice(&(offset as u32).to_le_bytes()); - entry[36..40].copy_from_slice(&(length as u32).to_le_bytes()); - written += 1; - } - if written != count { - return Err(GossipFrameError::InvalidDescriptor); - } - buffer[12..16].copy_from_slice(&(total as u32).to_le_bytes()); - reservation.flush().map_err(|_| GossipFrameError::Stale)?; - Ok(Self { descriptor: reservation.read(), expires }) - } - - pub fn read(self) -> TCacheRead { - self.descriptor - } - - pub fn acquire( - self, - consumer: &mut RandomAccessConsumer, - now: Instant, - ) -> Result { - if now >= self.expires { - return Err(GossipFrameError::Expired); - } - if !consumer.is_strict() || consumer.cache.cache != self.descriptor.tcache.cache { - return Err(GossipFrameError::InvalidDescriptor); - } - let read = consumer.acquire_strict(self.descriptor).ok_or(GossipFrameError::Stale)?; - let buffer = read.buffer().map_err(|_| GossipFrameError::Stale)?.0; - if buffer.len() < HEADER_BYTES || buffer[..8] != MAGIC { - return Err(GossipFrameError::InvalidDescriptor); - } - let count = u32::from_le_bytes(buffer[8..12].try_into().unwrap()) as usize; - let wire_len = u32::from_le_bytes(buffer[12..16].try_into().unwrap()) as usize; - if count == 0 || count > MAX_GOSSIP_SEGMENTS { - return Err(GossipFrameError::InvalidDescriptor); - } - let framing_start = HEADER_BYTES + count * SEGMENT_BYTES; - if framing_start > buffer.len() || wire_len == 0 || wire_len > MAX_GOSSIP_FRAME_SIZE { - return Err(GossipFrameError::InvalidDescriptor); - } - let descriptor_len = buffer.len(); - let view = GossipFrameView { read, count, wire_len, framing_start, descriptor_len }; - let mut total = 0usize; - for segment in view.segments() { - if segment.length == 0 || - segment.kind > 3 || - (segment.kind == 0 && - segment - .offset - .checked_add(segment.length) - .is_none_or(|end| end > view.descriptor_len() - framing_start)) - { - return Err(GossipFrameError::InvalidDescriptor); - } - total = total.checked_add(segment.length).ok_or(GossipFrameError::TooLarge)?; - } - if total != wire_len { - return Err(GossipFrameError::InvalidDescriptor); - } - Ok(view) - } -} - -#[derive(Debug)] -pub struct GossipFrameView { - read: AcquiredRead, - count: usize, - wire_len: usize, - framing_start: usize, - descriptor_len: usize, -} - -impl GossipFrameView { - pub fn wire_len(&self) -> usize { - self.wire_len - } - - pub fn segment_count(&self) -> usize { - self.count - } - - pub fn descriptor_len(&self) -> usize { - self.descriptor_len - } - - pub fn descriptor_range(&self) -> AcquiredRange { - self.read.with_range(0, self.descriptor_len()).expect("acquired descriptor") - } - - pub fn acquire_segments( - self, - gossip: &mut RandomAccessConsumer, - columns: Option<&mut RandomAccessConsumer>, - ) -> Option { - AcquiredGossipFrame::new(self, gossip, columns) - } - - pub fn segments(&self) -> impl ExactSizeIterator + '_ { - self.read.buffer().expect("acquired descriptor").0[HEADER_BYTES..self.framing_start] - .chunks_exact(SEGMENT_BYTES) - .map(|entry| GossipFrameSegment::decode(entry, self.framing_start)) - } - - fn segment(&self, index: usize) -> GossipFrameSegment { - assert!(index < self.count); - let start = HEADER_BYTES + index * SEGMENT_BYTES; - let buffer = self.read.buffer().expect("acquired descriptor").0; - GossipFrameSegment::decode(&buffer[start..start + SEGMENT_BYTES], self.framing_start) - } -} - -pub struct GossipFrameSegment { - kind: u64, - cache: u64, - seq: u64, - metadata: u64, - offset: usize, - length: usize, - framing_start: usize, -} - -impl GossipFrameSegment { - fn decode(entry: &[u8], framing_start: usize) -> Self { - Self { - kind: u64::from_le_bytes(entry[..8].try_into().unwrap()), - cache: u64::from_le_bytes(entry[8..16].try_into().unwrap()), - seq: u64::from_le_bytes(entry[16..24].try_into().unwrap()), - metadata: u64::from_le_bytes(entry[24..32].try_into().unwrap()), - offset: u32::from_le_bytes(entry[32..36].try_into().unwrap()) as usize, - length: u32::from_le_bytes(entry[36..40].try_into().unwrap()) as usize, - framing_start, - } - } - - pub fn framing_range(&self) -> Option> { - (self.kind == 0).then(|| { - self.framing_start + self.offset..self.framing_start + self.offset + self.length - }) - } - - pub fn acquire( - &self, - gossip: &mut RandomAccessConsumer, - columns: Option<&mut RandomAccessConsumer>, - ) -> Option { - let consumer = match self.kind { - 1 => gossip, - 2 | 3 => columns?, - _ => return None, - }; - if !consumer.is_strict() || - consumer.cache.cache as usize as u64 != self.cache || - !self.seq.is_multiple_of(super::ALIGN as u64) - { - return None; - } - let read = TCacheRead { tcache: consumer.cache, seq: self.seq }; - if self.kind == 3 { - let reference = - SubReservationRef { read, header_bytes: (self.metadata >> 32) as usize }; - let acquired = reference.acquire(consumer).ok()?; - let [first, second] = acquired.ranges(((self.metadata as u32) >> 1) as usize)?; - let range = if self.metadata & 1 == 0 { first } else { second }; - range.slice(self.offset, self.length) - } else { - consumer.acquire_strict(read)?.with_range(self.offset, self.length) - } - } -} - -#[cfg(test)] -mod tests; diff --git a/crates/common/src/spine/tcache/gossip_frame/acquired.rs b/crates/common/src/spine/tcache/gossip_frame/acquired.rs deleted file mode 100644 index 2aa2d1e7..00000000 --- a/crates/common/src/spine/tcache/gossip_frame/acquired.rs +++ /dev/null @@ -1,141 +0,0 @@ -use std::{ - mem, - ops::{Deref, Range}, - ptr::NonNull, -}; - -use super::{ - AcquiredRange, AcquiredRead, GossipFrameSegment, GossipFrameView, RandomAccessConsumer, - SubReservationRef, TCacheRead, -}; - -pub enum AcquiredGossipSegment { - Framing(Range), - Data(AcquiredRange), -} - -#[derive(Debug)] -pub struct AcquiredGossipFrame { - view: GossipFrameView, - gossip: NonNull, - columns: Option>, - // Every non-framing descriptor in [next, acquired_end) owns one bucket - // count. The descriptor stays pinned until those counts are released. - next: usize, - acquired_end: usize, -} - -// As with AcquiredRead, consumers stay at stable addresses and outlive their -// reads. Acquisition, handoff, and drops remain on the consumer's tile. -unsafe impl Send for AcquiredGossipFrame {} - -impl AcquiredGossipFrame { - pub(super) fn new( - view: GossipFrameView, - gossip: &mut RandomAccessConsumer, - mut columns: Option<&mut RandomAccessConsumer>, - ) -> Option { - let mut frame = Self { - view, - gossip: NonNull::from(&mut *gossip), - columns: columns.as_deref_mut().map(NonNull::from), - next: 0, - acquired_end: 0, - }; - for segment in frame.view.segments() { - if segment.kind != 0 { - let range = segment.acquire(gossip, columns.as_deref_mut())?; - // No fallible work separates forgetting this owner and recording - // its count in the frame's acquired prefix. - mem::forget(range); - } - frame.acquired_end += 1; - } - Some(frame) - } - - pub fn take_next(&mut self) -> Option { - if self.next == self.acquired_end { - return None; - } - let segment = self.view.segment(self.next); - if let Some(range) = segment.framing_range() { - self.next += 1; - return Some(AcquiredGossipSegment::Framing(range)); - } - let mut range = self.take_range(&segment); - while self.next < self.acquired_end { - let next = self.view.segment(self.next); - if next.kind == 0 || - range.read.consumer != self.consumer(&next).as_ptr() || - range.read.seq() != next.seq || - range.offset + range.length != Self::offset(&next, range.read.read) - { - break; - } - let next = self.take_range(&next); - assert!(range.extend_contiguous(&next)); - } - Some(AcquiredGossipSegment::Data(range)) - } - - fn consumer(&self, segment: &GossipFrameSegment) -> NonNull { - if segment.kind == 1 { self.gossip } else { self.columns.expect("acquired column segment") } - } - - fn take_read(&mut self, segment: &GossipFrameSegment) -> AcquiredRead { - let consumer = self.consumer(segment); - // Each call transfers exactly one existing count. Nothing increments - // here, and frame cleanup excludes the transferred descriptor. - let read = AcquiredRead { - consumer: consumer.as_ptr(), - read: TCacheRead { tcache: unsafe { consumer.as_ref() }.cache, seq: segment.seq }, - acquired: self.view.read.acquired, - }; - self.next += 1; - read - } - - fn take_range(&mut self, segment: &GossipFrameSegment) -> AcquiredRange { - let read = self.take_read(segment); - let offset = Self::offset(segment, read.read); - AcquiredRange { read, offset, length: segment.length } - } - - fn offset(segment: &GossipFrameSegment, read: TCacheRead) -> usize { - if segment.kind != 3 { - return segment.offset; - } - let reference = SubReservationRef { read, header_bytes: (segment.metadata >> 32) as usize }; - // Admission validated this part and retains its count. Its immutable - // layout remains valid even after the reservation is closed. - let base = unsafe { - reference.acquired_offset( - ((segment.metadata as u32) >> 1) as usize, - segment.metadata & 1 != 0, - ) - }; - base + segment.offset - } -} - -impl Deref for AcquiredGossipFrame { - type Target = GossipFrameView; - - fn deref(&self) -> &Self::Target { - &self.view - } -} - -impl Drop for AcquiredGossipFrame { - fn drop(&mut self) { - while self.next < self.acquired_end { - let segment = self.view.segment(self.next); - if segment.kind == 0 { - self.next += 1; - } else { - drop(self.take_read(&segment)); - } - } - } -} diff --git a/crates/common/src/spine/tcache/gossip_frame/tests.rs b/crates/common/src/spine/tcache/gossip_frame/tests.rs deleted file mode 100644 index 9ed1cad5..00000000 --- a/crates/common/src/spine/tcache/gossip_frame/tests.rs +++ /dev/null @@ -1,445 +0,0 @@ -use std::{ - panic::{AssertUnwindSafe, catch_unwind}, - time::Duration, -}; - -use super::*; -use crate::{P2pSend, SubLayout, TCache}; - -fn write(producer: &mut Producer, bytes: &[u8]) -> TCacheRead { - let mut reservation = producer.reserve(bytes.len(), false).unwrap(); - reservation.write_all(bytes).unwrap(); - reservation.flush().unwrap(); - reservation.read() -} - -#[test] -fn copy_handle_round_trips_framing_and_source_ranges() { - fn is_copy() {} - is_copy::(); - is_copy::(); - let mut producer = TCache::producer("", 1 << 18); - let mut consumer = Box::new(producer.cache_ref().strict_random_access("", true).unwrap()); - let source = write(&mut producer, b"0123456789"); - let now = Instant::now(); - let frame = GossipFrameRef::write( - &mut producer, - now + Duration::from_secs(1), - b"ab--cd", - [ - GossipSegment::Framing { offset: 0, length: 2 }, - GossipSegment::Gossip { read: source, offset: 3, length: 4 }, - GossipSegment::Framing { offset: 4, length: 2 }, - ] - .into_iter(), - ) - .unwrap(); - let view = frame.acquire(&mut consumer, now).unwrap(); - assert_eq!(view.wire_len(), 8); - assert_eq!(view.segment_count(), 3); - let descriptor = view.descriptor_range(); - let mut wire = Vec::new(); - for segment in view.segments() { - if let Some(range) = segment.framing_range() { - wire.extend_from_slice(&descriptor.as_ref()[range]); - } else { - wire.extend_from_slice(segment.acquire(&mut consumer, None).unwrap().as_ref()); - } - } - assert_eq!(wire, b"ab3456cd"); - assert!(matches!( - frame.acquire(&mut consumer, now + Duration::from_secs(1)), - Err(GossipFrameError::Expired) - )); -} - -#[test] -fn shared_segments_expose_only_verified_subranges() { - let mut producer = TCache::producer("", 1 << 18); - let mut consumer = Box::new(producer.cache_ref().strict_random_access("", true).unwrap()); - let mut columns = TCache::producer("", 1 << 18); - let mut reader = Box::new(columns.cache_ref().retained_random_access("").unwrap()); - let reference = columns - .sub_reservation(SubLayout { parts: 2, first_len: 4, second_len: 2 }, b"", b"") - .unwrap(); - let pending = columns - .view_sub_reservation(reference) - .unwrap() - .claim(0) - .unwrap() - .write(b"cell", b"pf") - .unwrap(); - let now = Instant::now(); - let frame = GossipFrameRef::write( - &mut producer, - now + Duration::from_secs(1), - b"", - [ - GossipSegment::Shared { - reservation: reference, - part: 0, - second: false, - offset: 1, - length: 2, - }, - GossipSegment::Shared { - reservation: reference, - part: 0, - second: true, - offset: 0, - length: 2, - }, - ] - .into_iter(), - ) - .unwrap(); - let view = frame.acquire(&mut consumer, now).unwrap(); - assert!(view.segments().next().unwrap().acquire(&mut consumer, Some(&mut reader)).is_none()); - pending.acquire(&mut reader).unwrap().accept().unwrap(); - let ranges: Vec<_> = view - .segments() - .map(|segment| segment.acquire(&mut consumer, Some(&mut reader)).unwrap()) - .collect(); - assert_eq!(ranges[0].as_ref(), b"el"); - assert_eq!(ranges[1].as_ref(), b"pf"); - columns.view_sub_reservation(reference).unwrap().close(); - reader.advance_retention(columns.next_seq()); - assert!(view.segments().next().unwrap().acquire(&mut consumer, Some(&mut reader)).is_none()); - assert_eq!(ranges[0].as_ref(), b"el"); -} - -#[test] -fn descriptor_bounds_and_sources_are_checked() { - let mut producer = TCache::producer("", 1 << 18); - let mut consumer = Box::new(producer.cache_ref().strict_random_access("", true).unwrap()); - let mut other = TCache::producer("", 1 << 18); - let mut other_reader = Box::new(other.cache_ref().retained_random_access("").unwrap()); - let other_read = write(&mut other, b"data"); - let expires = Instant::now() + Duration::from_secs(1); - for segment in [ - GossipSegment::Framing { offset: usize::MAX, length: 1 }, - GossipSegment::Framing { offset: 0, length: 0 }, - GossipSegment::Framing { offset: 1, length: 4 }, - GossipSegment::Gossip { read: other_read, offset: 0, length: 4 }, - ] { - assert!( - GossipFrameRef::write(&mut producer, expires, b"data", [segment].into_iter()).is_err() - ); - } - assert!(GossipFrameRef::write(&mut producer, expires, b"", [].into_iter()).is_err()); - assert!( - GossipFrameRef::write( - &mut producer, - expires, - b"x", - std::iter::repeat_n( - GossipSegment::Framing { offset: 0, length: 1 }, - MAX_GOSSIP_SEGMENTS + 1, - ) - ) - .is_err() - ); - let frame = GossipFrameRef::write( - &mut producer, - expires, - b"", - [GossipSegment::DataColumns { read: other_read, offset: 2, length: 4 }].into_iter(), - ) - .unwrap(); - let view = frame.acquire(&mut consumer, Instant::now()).unwrap(); - let segment = view.segments().next().unwrap(); - assert!(segment.acquire(&mut consumer, None).is_none()); - assert!(segment.acquire(&mut consumer, Some(&mut other_reader)).is_none()); - - for bytes in [&b"not a descriptor"[..], &b"SGFRAME1\xff\xff\xff\xff\x01\x00\x00\x00"[..]] { - let malformed = GossipFrameRef { descriptor: write(&mut producer, bytes), expires }; - assert!(matches!( - malformed.acquire(&mut consumer, Instant::now()), - Err(GossipFrameError::InvalidDescriptor) - )); - } -} - -#[test] -fn owned_range_slices_check_bounds() { - let mut producer = TCache::producer("", 1 << 16); - let mut consumer = Box::new(producer.cache_ref().strict_random_access("", true).unwrap()); - let read = write(&mut producer, b"0123456789"); - let acquired = consumer.acquire_strict(read).unwrap(); - let range = acquired.with_range(2, 6).unwrap(); - assert_eq!(range.clone().slice(2, 3).unwrap().as_ref(), b"456"); - assert!(range.clone().slice(6, 0).unwrap().is_empty()); - assert!(range.clone().slice(6, 1).is_none()); - assert!(range.clone().slice(usize::MAX, 1).is_none()); - assert!(range.slice(1, usize::MAX).is_none()); - let mut first = acquired.with_range(0, 3).unwrap(); - let next = acquired.with_range(3, 4).unwrap(); - let gap = acquired.with_range(8, 1).unwrap(); - assert!(!first.extend_contiguous(&gap)); - assert!(first.extend_contiguous(&next)); - drop(next); - assert_eq!(first.as_ref(), b"0123456"); -} - -#[test] -fn failed_acquisition_releases_every_successful_prefix_without_touching_other_reads() { - let mut producer = TCache::producer("", 1 << 18); - let mut consumer = Box::new(producer.cache_ref().strict_random_access("", true).unwrap()); - let mut columns = TCache::producer("", 1 << 18); - let mut reader = Box::new(columns.cache_ref().retained_random_access("").unwrap()); - let gossip = write(&mut producer, b"gossip"); - let column = write(&mut columns, b"column"); - let gossip_guard = consumer.acquire_strict(gossip).unwrap(); - let column_guard = reader.acquire_strict(column).unwrap(); - let shared = columns - .sub_reservation(SubLayout { parts: 2, first_len: 4, second_len: 2 }, b"", b"") - .unwrap(); - columns - .view_sub_reservation(shared) - .unwrap() - .claim(0) - .unwrap() - .write(b"cell", b"pf") - .unwrap() - .acquire(&mut reader) - .unwrap() - .accept() - .unwrap(); - let now = Instant::now(); - let segments = [ - GossipSegment::Framing { offset: 0, length: 1 }, - GossipSegment::Gossip { read: gossip, offset: 1, length: 3 }, - GossipSegment::DataColumns { read: column, offset: 0, length: 4 }, - GossipSegment::Shared { reservation: shared, part: 0, second: false, offset: 0, length: 4 }, - GossipSegment::Shared { reservation: shared, part: 0, second: true, offset: 0, length: 2 }, - GossipSegment::Gossip { read: gossip, offset: 1, length: 3 }, - GossipSegment::Framing { offset: 0, length: 1 }, - ]; - for failed in 0..segments.len() { - let mut descriptors = segments; - descriptors[failed] = GossipSegment::DataColumns { read: column, offset: 100, length: 1 }; - let frame = GossipFrameRef::write( - &mut producer, - now + Duration::from_secs(1), - b"f", - descriptors.into_iter(), - ) - .unwrap(); - assert!( - frame - .acquire(&mut consumer, now) - .unwrap() - .acquire_segments(&mut consumer, Some(&mut reader)) - .is_none() - ); - assert_eq!(consumer.active_count(), 1, "failed descriptor {failed}"); - assert_eq!(reader.active_count(), 1, "failed descriptor {failed}"); - assert_eq!(gossip_guard.buffer().unwrap().0, b"gossip"); - assert_eq!(column_guard.buffer().unwrap().0, b"column"); - } - drop(gossip_guard); - drop(column_guard); - assert_eq!(consumer.active_count(), 0); - assert_eq!(reader.active_count(), 0); -} - -#[test] -fn handoff_transfers_counts_and_frame_drop_releases_only_the_remainder() { - let mut producer = TCache::producer("", 1 << 18); - let mut consumer = Box::new(producer.cache_ref().strict_random_access("", true).unwrap()); - let mut columns = TCache::producer("", 1 << 18); - let mut reader = Box::new(columns.cache_ref().retained_random_access("").unwrap()); - let gossip = write(&mut producer, b"gossip"); - let column = write(&mut columns, b"column"); - let now = Instant::now(); - let reference = GossipFrameRef::write( - &mut producer, - now + Duration::from_secs(1), - b"f", - [ - GossipSegment::Framing { offset: 0, length: 1 }, - GossipSegment::Gossip { read: gossip, offset: 0, length: 6 }, - GossipSegment::DataColumns { read: column, offset: 0, length: 6 }, - GossipSegment::DataColumns { read: column, offset: 0, length: 6 }, - GossipSegment::Framing { offset: 0, length: 1 }, - ] - .into_iter(), - ) - .unwrap(); - let mut frame = reference - .acquire(&mut consumer, now) - .unwrap() - .acquire_segments(&mut consumer, Some(&mut reader)) - .unwrap(); - assert_eq!(consumer.active_count(), 2); - assert_eq!(reader.active_count(), 2); - assert!(matches!(frame.take_next(), Some(AcquiredGossipSegment::Framing(_)))); - let Some(AcquiredGossipSegment::Data(gossip_range)) = frame.take_next() else { panic!() }; - let Some(AcquiredGossipSegment::Data(column_range)) = frame.take_next() else { panic!() }; - assert_eq!(consumer.active_count(), 2); - assert_eq!(reader.active_count(), 2); - reader.advance_retention(columns.next_seq()); - drop(frame); - assert_eq!(consumer.active_count(), 1); - assert_eq!(reader.active_count(), 1); - assert_eq!(gossip_range.as_ref(), b"gossip"); - assert_eq!(column_range.as_ref(), b"column"); - drop(gossip_range); - drop(column_range); - assert_eq!(consumer.active_count(), 0); - assert_eq!(reader.active_count(), 0); -} - -#[test] -fn shared_handoff_survives_closure_without_exposing_unverified_gaps() { - let mut producer = TCache::producer("", 1 << 18); - let mut consumer = Box::new(producer.cache_ref().strict_random_access("", true).unwrap()); - let mut columns = TCache::producer("", 1 << 18); - let mut reader = Box::new(columns.cache_ref().retained_random_access("").unwrap()); - let shared = columns - .sub_reservation(SubLayout { parts: 3, first_len: 4, second_len: 2 }, b"hdr", b"mid") - .unwrap(); - for part in [0, 2] { - columns - .view_sub_reservation(shared) - .unwrap() - .claim(part) - .unwrap() - .write(&[part as u8; 4], &[part as u8 + 10; 2]) - .unwrap() - .acquire(&mut reader) - .unwrap() - .accept() - .unwrap(); - } - let now = Instant::now(); - let reference = GossipFrameRef::write( - &mut producer, - now + Duration::from_secs(1), - b"", - [ - GossipSegment::Shared { - reservation: shared, - part: 0, - second: false, - offset: 1, - length: 3, - }, - GossipSegment::Shared { - reservation: shared, - part: 2, - second: false, - offset: 0, - length: 4, - }, - GossipSegment::Shared { - reservation: shared, - part: 0, - second: true, - offset: 0, - length: 2, - }, - GossipSegment::Shared { - reservation: shared, - part: 2, - second: true, - offset: 1, - length: 1, - }, - ] - .into_iter(), - ) - .unwrap(); - let mut frame = reference - .acquire(&mut consumer, now) - .unwrap() - .acquire_segments(&mut consumer, Some(&mut reader)) - .unwrap(); - assert_eq!(reader.active_count(), 4); - columns.view_sub_reservation(shared).unwrap().close(); - reader.advance_retention(columns.next_seq()); - assert!( - reference - .acquire(&mut consumer, now) - .unwrap() - .acquire_segments(&mut consumer, Some(&mut reader)) - .is_none() - ); - assert_eq!(reader.active_count(), 4); - for (index, expected) in [&[0; 3][..], &[2; 4], &[10; 2], &[12; 1]].into_iter().enumerate() { - let Some(AcquiredGossipSegment::Data(range)) = frame.take_next() else { panic!() }; - assert_eq!(reader.active_count(), 4 - index); - assert_eq!(range.as_ref(), expected); - drop(range); - assert_eq!(reader.active_count(), 3 - index); - } - assert!(frame.take_next().is_none()); - drop(frame); - assert_eq!(consumer.active_count(), 0); - assert_eq!(reader.active_count(), 0); -} - -#[test] -fn coalescing_transfers_one_pin_and_releases_redundant_pins() { - let mut producer = TCache::producer("", 1 << 18); - let mut consumer = Box::new(producer.cache_ref().strict_random_access("", true).unwrap()); - let source = write(&mut producer, b"0123456789"); - let now = Instant::now(); - let reference = GossipFrameRef::write( - &mut producer, - now + Duration::from_secs(1), - b"", - [ - GossipSegment::Gossip { read: source, offset: 1, length: 3 }, - GossipSegment::Gossip { read: source, offset: 4, length: 3 }, - GossipSegment::Gossip { read: source, offset: 7, length: 2 }, - ] - .into_iter(), - ) - .unwrap(); - let mut frame = reference - .acquire(&mut consumer, now) - .unwrap() - .acquire_segments(&mut consumer, None) - .unwrap(); - assert_eq!(consumer.active_count(), 4); - let Some(AcquiredGossipSegment::Data(range)) = frame.take_next() else { panic!() }; - assert_eq!(consumer.active_count(), 2); - assert!(frame.take_next().is_none()); - drop(frame); - assert_eq!(consumer.active_count(), 1); - assert_eq!(range.as_ref(), b"12345678"); - drop(range); - assert_eq!(consumer.active_count(), 0); -} - -#[test] -fn unwinding_releases_untransferred_pins_but_not_the_handed_off_read() { - let mut producer = TCache::producer("", 1 << 18); - let mut consumer = Box::new(producer.cache_ref().strict_random_access("", true).unwrap()); - let source = write(&mut producer, b"data"); - let now = Instant::now(); - let reference = GossipFrameRef::write( - &mut producer, - now + Duration::from_secs(1), - b"", - [GossipSegment::Gossip { read: source, offset: 0, length: 4 }; 3].into_iter(), - ) - .unwrap(); - let mut handed_off = None; - let result = catch_unwind(AssertUnwindSafe(|| { - let mut frame = reference - .acquire(&mut consumer, now) - .unwrap() - .acquire_segments(&mut consumer, None) - .unwrap(); - let Some(AcquiredGossipSegment::Data(range)) = frame.take_next() else { panic!() }; - handed_off = Some(range); - panic!("abort a partially handed-off frame"); - })); - assert!(result.is_err()); - assert_eq!(consumer.active_count(), 1); - assert_eq!(handed_off.as_ref().unwrap().as_ref(), b"data"); - drop(handed_off); - assert_eq!(consumer.active_count(), 0); -}