From 09c178f4baea90f901fa9cc1da4f517df9166269 Mon Sep 17 00:00:00 2001 From: owen Date: Wed, 9 Sep 2026 18:34:26 +0100 Subject: [PATCH 1/5] Report registration and TCP tile stats per slot Both tiles logged every 500 ms, a cadence inherited from the worker metrics tick, so an idle relay printed a wall of zeros. They now report on the slot boundary, keyed by `bid_slot`, like the other eight tiles. The metrics tick keeps its 500 ms period. Co-Authored-By: Claude Opus 5 (1M context) --- crates/relay/src/main.rs | 9 ++- crates/relay/src/registration/tile.rs | 86 ++++++++++++++++++--------- crates/relay/src/tcp_bid_recv/mod.rs | 47 ++++++++------- 3 files changed, 90 insertions(+), 52 deletions(-) diff --git a/crates/relay/src/main.rs b/crates/relay/src/main.rs index 35da0318b..3df6e920f 100644 --- a/crates/relay/src/main.rs +++ b/crates/relay/src/main.rs @@ -272,8 +272,12 @@ async fn run( if config.is_registration_instance { for core in config.cores.reg_workers.clone() { - let tile = - RegistrationTile::new(core, chain_info.as_ref().clone(), reg_worker_rx.clone()); + let tile = RegistrationTile::new( + core, + chain_info.as_ref().clone(), + reg_worker_rx.clone(), + slot_events.clone(), + ); attach_tile(tile, spine, TileConfig::new(core, ThreadPriority::OSDefault)); } } @@ -321,6 +325,7 @@ async fn run( config.tcp_max_connections, spine.spine.dcache_ptr_for::(), http_submissions.clone(), + slot_events.clone(), ); attach_tile( block_submission_tcp_listener, diff --git a/crates/relay/src/registration/tile.rs b/crates/relay/src/registration/tile.rs index 0b779e015..c53c5445d 100644 --- a/crates/relay/src/registration/tile.rs +++ b/crates/relay/src/registration/tile.rs @@ -1,21 +1,24 @@ -use std::time::{Duration, Instant}; +use std::{ + sync::Arc, + time::{Duration, Instant}, +}; use flux::{ tile::{Tile, TileName}, utils::{ShortTypename, short_typename}, }; use flux_profiler::timed; +use flux_utils::SharedVector; use helix_common::{chain_info::ChainInfo, utils::utcnow_ns}; use helix_types::SignedValidatorRegistration; use tracing::info; -use crate::{HelixSpine, api::proposer::ProposerApiError, registration::handle::RegWorkerJob}; +use crate::{ + HelixSpine, api::proposer::ProposerApiError, housekeeper::SlotUpdate, + registration::handle::RegWorkerJob, spine::messages::SlotMsg, +}; -/// Registrations aren't slot-scoped (they arrive on their own schedule, not -/// tied to `bid_slot`), so this reports on the same wall-clock cadence as the -/// rest of `Telemetry` rather than on a slot boundary. -/// `registrations_seen == valid + invalid`; `batches_seen == completed + -/// aborted`. +/// `registrations_seen == valid + invalid`; `batches_seen == completed + aborted`. #[derive(Default)] struct RegStats { valid: u32, @@ -30,7 +33,6 @@ struct Telemetry { next_record: Instant, loop_start: Instant, loop_worked: Duration, - stats: RegStats, } impl Telemetry { @@ -63,18 +65,6 @@ impl Telemetry { WORKER_UTIL.with_label_values(&[id]).observe(util); WORKER_QUEUE_LEN.with_label_values(&[queue_type]).observe(rx.len() as f64); - - let stats = std::mem::take(&mut self.stats); - info!( - id, - registrations_seen = stats.valid + stats.invalid, - valid = stats.valid, - invalid = stats.invalid, - batches_seen = stats.completed + stats.aborted, - completed = stats.completed, - aborted = stats.aborted, - "registration tile stats" - ); } } } @@ -89,7 +79,6 @@ impl Default for Telemetry { Duration::from_millis(utcnow_ns() % 10 * 5), loop_start: Instant::now(), loop_worked: Default::default(), - stats: RegStats::default(), } } } @@ -100,6 +89,9 @@ pub struct RegistrationTile { chain_info: ChainInfo, tel: Telemetry, rx: crossbeam_channel::Receiver, + slot_events: Arc>, + bid_slot: u64, + stats: RegStats, } impl RegistrationTile { @@ -107,9 +99,47 @@ impl RegistrationTile { core_id: usize, chain_info: ChainInfo, rx: crossbeam_channel::Receiver, + slot_events: Arc>, ) -> Self { let id = ShortTypename::from_str_truncate(&format!("registration_{core_id}")); - Self { core_id, id, chain_info, tel: Default::default(), rx } + Self { + core_id, + id, + chain_info, + tel: Default::default(), + rx, + slot_events, + bid_slot: 0, + stats: RegStats::default(), + } + } + + fn on_slot_msg(&mut self, msg: SlotMsg) { + let Some(ev) = self.slot_events.get(msg.ix) else { return }; + let bid_slot = ev.bid_slot.as_u64(); + if bid_slot <= self.bid_slot { + return; + } + self.report_slot_stats(); + self.bid_slot = bid_slot; + } + + fn report_slot_stats(&mut self) { + if self.bid_slot == 0 { + return; + } + let stats = std::mem::take(&mut self.stats); + info!( + id = &*self.id, + bid_slot = self.bid_slot, + registrations_seen = stats.valid + stats.invalid, + valid = stats.valid, + invalid = stats.invalid, + batches_seen = stats.completed + stats.aborted, + completed = stats.completed, + aborted = stats.aborted, + "registration slot stats" + ); } fn handle_reg_task(&mut self, task: RegWorkerJob) { @@ -119,9 +149,9 @@ impl RegistrationTile { let completed = self.process_reg_task(task); let tag = if completed { "RegistrationBatch" } else { "RegistrationBatch_Aborted" }; if completed { - self.tel.stats.completed += 1; + self.stats.completed += 1; } else { - self.tel.stats.aborted += 1; + self.stats.aborted += 1; } let dur = start_task.elapsed(); @@ -146,9 +176,9 @@ impl RegistrationTile { let start = Instant::now(); let valid = validate_registration(&self.chain_info, ®s[i]); if valid.is_ok() { - self.tel.stats.valid += 1; + self.stats.valid += 1; } else { - self.tel.stats.invalid += 1; + self.stats.invalid += 1; } res.push((i, valid.is_ok())); @@ -164,7 +194,9 @@ impl RegistrationTile { } impl Tile for RegistrationTile { - fn loop_body(&mut self, _adapter: &mut flux::spine::SpineAdapter) { + fn loop_body(&mut self, adapter: &mut flux::spine::SpineAdapter) { + adapter.consume(|msg: SlotMsg, _| self.on_slot_msg(msg)); + if let Ok(task) = self.rx.recv_timeout(Duration::from_millis(50)) { self.handle_reg_task(task); } diff --git a/crates/relay/src/tcp_bid_recv/mod.rs b/crates/relay/src/tcp_bid_recv/mod.rs index f0f3cfe53..47f7d2fd4 100644 --- a/crates/relay/src/tcp_bid_recv/mod.rs +++ b/crates/relay/src/tcp_bid_recv/mod.rs @@ -1,9 +1,4 @@ -use std::{ - collections::HashMap, - net::SocketAddr, - sync::Arc, - time::{Duration, Instant}, -}; +use std::{collections::HashMap, net::SocketAddr, sync::Arc}; use bytes::Bytes; use dashmap::DashMap; @@ -23,7 +18,8 @@ use uuid::Uuid; use crate::{ HelixSpine, auctioneer::{InternalBidSubmissionHeader, SubmissionRef}, - spine::messages::{NewBidSubmission, SubmissionResultWithRef}, + housekeeper::SlotUpdate, + spine::messages::{NewBidSubmission, SlotMsg, SubmissionResultWithRef}, }; pub mod types; @@ -38,11 +34,7 @@ pub use crate::tcp_bid_recv::types::{ type SubmissionError = (Token, Option, Option, BidSubmissionError); -/// Connections aren't slot-scoped, so this reports on a wall-clock cadence -/// instead of a slot boundary. For a registered peer, every `Message` is -/// either a decoded submission or a header-parse error: -/// `submissions_received + header_parse_errors == messages_from_registered`. -/// For an unregistered peer, every `Message` is a registration attempt: +/// `submissions_received + header_parse_errors == messages_from_registered`, and /// `registration_ok + registration_invalid == registration_attempts`. #[derive(Default)] struct Stats { @@ -69,19 +61,19 @@ pub struct BidSubmissionTcpListener { // mutated out from under the decoder between publish and consume. http_submissions: Arc>, + slot_events: Arc>, + bid_slot: u64, stats: Stats, - next_report: Instant, } impl BidSubmissionTcpListener { - const REPORT_FREQ: Duration = Duration::from_millis(500); - pub fn new( listener_addr: SocketAddr, api_key_cache: Arc>>, max_connections: usize, dcache_ptr: DCachePtr, http_submissions: Arc>, + slot_events: Arc>, ) -> Self { // TODO: enable telemetry once the per-connection shm queue leak is fixed // Telemetry creates 4 shm queues per accepted connection keyed by peer @@ -100,20 +92,29 @@ impl BidSubmissionTcpListener { registered: HashMap::with_capacity(max_connections), submission_errors: Vec::with_capacity(max_connections), http_submissions, + slot_events, + bid_slot: 0, stats: Stats::default(), - next_report: Instant::now() + Self::REPORT_FREQ, } } - fn maybe_report_stats(&mut self) { - let now = Instant::now(); - if now < self.next_report { + fn on_slot_msg(&mut self, msg: SlotMsg) { + let Some(ev) = self.slot_events.get(msg.ix) else { return }; + let bid_slot = ev.bid_slot.as_u64(); + if bid_slot <= self.bid_slot { return; } - self.next_report = now + Self::REPORT_FREQ; + self.report_slot_stats(); + self.bid_slot = bid_slot; + } + fn report_slot_stats(&mut self) { + if self.bid_slot == 0 { + return; + } let stats = std::mem::take(&mut self.stats); info!( + bid_slot = self.bid_slot, accepted = stats.accepted, reconnected = stats.reconnected, disconnected = stats.disconnected, @@ -124,13 +125,15 @@ impl BidSubmissionTcpListener { submissions_received = stats.submissions_received, header_parse_errors = stats.header_parse_errors, results_sent = stats.results_sent, - "tcp bid recv tile stats" + "tcp bid recv slot stats" ); } } impl Tile for BidSubmissionTcpListener { fn loop_body(&mut self, adapter: &mut flux::spine::SpineAdapter) { + adapter.consume(|msg: SlotMsg, _| self.on_slot_msg(msg)); + self.listener.poll_with_produce(&mut adapter.producers, |event| match event { PollEvent::Accept { listener: _, stream, peer_addr } => { tracing::trace!("connected to new peer {:?} with token {:?}", peer_addr, stream); @@ -256,7 +259,5 @@ impl Tile for BidSubmissionTcpListener { response.ssz_append(buffer); }); }); - - self.maybe_report_stats(); } } From c095224482ad6f6d65652a52877f064f90e52497 Mon Sep 17 00:00:00 2001 From: owen Date: Wed, 9 Sep 2026 18:34:34 +0100 Subject: [PATCH 2/5] Accept a payload_attributes event with no parent block number Gloas drops `parent_block_number` from the event, so every event failed to parse and the relay never got a slot's attributes. The field is now optional, and stays quoted on the wire for the forks that still send it. Nothing reads it yet; a future consumer must take the number from the execution layer, because Gloas will never supply it. Co-Authored-By: Claude Opus 5 (1M context) --- crates/builder/src/building/assemble/tests.rs | 2 +- crates/builder/src/building/slot.rs | 37 +++++++++++++++++-- crates/builder/src/building/submit/tests.rs | 2 +- crates/common/src/beacon/types/chain.rs | 20 +++++++++- 4 files changed, 53 insertions(+), 8 deletions(-) diff --git a/crates/builder/src/building/assemble/tests.rs b/crates/builder/src/building/assemble/tests.rs index 987d72c3c..344aa49a3 100644 --- a/crates/builder/src/building/assemble/tests.rs +++ b/crates/builder/src/building/assemble/tests.rs @@ -74,7 +74,7 @@ impl Fixture { SlotContext { slot: 1, parent_hash: self.parent_hash, - parent_block_number: 0, + parent_block_number: Some(0), timestamp: self.parent_timestamp + 12, prev_randao: B256::repeat_byte(0xcc), withdrawals: Withdrawals::default(), diff --git a/crates/builder/src/building/slot.rs b/crates/builder/src/building/slot.rs index 444f3b929..acb0c1729 100644 --- a/crates/builder/src/building/slot.rs +++ b/crates/builder/src/building/slot.rs @@ -23,7 +23,7 @@ pub struct ProposerDuty { pub struct SlotContext { pub slot: u64, pub parent_hash: B256, - pub parent_block_number: u64, + pub parent_block_number: Option, pub timestamp: u64, pub prev_randao: B256, pub withdrawals: Withdrawals, @@ -144,7 +144,7 @@ mod tests { data: PayloadAttributesEventData { proposer_index: 1, proposal_slot: slot.into(), - parent_block_number: slot - 1, + parent_block_number: Some(slot - 1), parent_block_root: String::new(), parent_block_hash: parent, payload_attributes: PayloadAttributes { @@ -174,7 +174,7 @@ mod tests { assert_eq!(context.slot, 10); assert_eq!(context.parent_hash, B256::repeat_byte(0x11)); - assert_eq!(context.parent_block_number, 9); + assert_eq!(context.parent_block_number, Some(9)); assert_eq!(context.timestamp, 1_700_000_000 + 120); assert_eq!(context.prev_randao, B256::repeat_byte(0xcc)); assert_eq!(context.parent_beacon_block_root, B256::repeat_byte(0xdd)); @@ -337,11 +337,40 @@ mod tests { let context = tracker.on_payload_attributes(event).expect("a complete event must build"); assert_eq!(context.slot, 11111); - assert_eq!(context.parent_block_number, 999); + assert_eq!(context.parent_block_number, Some(999)); assert_eq!(context.timestamp, 1_700_000_000); assert_eq!(context.parent_hash, B256::repeat_byte(0x22)); assert_eq!(context.parent_beacon_block_root, B256::repeat_byte(0x44)); assert_eq!(context.withdrawals.len(), 1); assert_eq!(context.withdrawals[0].amount, 32_000_000_000); } + + #[test] + fn parses_a_gloas_payload_attributes_event_without_a_parent_block_number() { + let json = r#"{ + "version": "gloas", + "data": { + "proposer_index": "123", + "proposal_slot": "11111", + "parent_block_root": "0x1111111111111111111111111111111111111111111111111111111111111111", + "parent_block_hash": "0x2222222222222222222222222222222222222222222222222222222222222222", + "payload_attributes": { + "timestamp": "1700000000", + "prev_randao": "0x3333333333333333333333333333333333333333333333333333333333333333", + "suggested_fee_recipient": "0xbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", + "withdrawals": [], + "parent_beacon_block_root": "0x4444444444444444444444444444444444444444444444444444444444444444" + } + } + }"#; + + let event: PayloadAttributesEvent = serde_json::from_str(json).unwrap(); + let mut tracker = SlotTracker::default(); + tracker.on_duties(vec![duty_response(11111)]); + + let context = tracker.on_payload_attributes(event).expect("a Gloas event must build"); + + assert_eq!(context.slot, 11111); + assert_eq!(context.parent_block_number, None); + } } diff --git a/crates/builder/src/building/submit/tests.rs b/crates/builder/src/building/submit/tests.rs index fc85ac379..aece93ef6 100644 --- a/crates/builder/src/building/submit/tests.rs +++ b/crates/builder/src/building/submit/tests.rs @@ -23,7 +23,7 @@ fn slot_context() -> SlotContext { SlotContext { slot: 42, parent_hash: B256::repeat_byte(0x11), - parent_block_number: 41, + parent_block_number: Some(41), timestamp: 1_700_000_000, prev_randao: B256::repeat_byte(0xcc), withdrawals: Withdrawals::default(), diff --git a/crates/common/src/beacon/types/chain.rs b/crates/common/src/beacon/types/chain.rs index b68fbe244..490eddca8 100644 --- a/crates/common/src/beacon/types/chain.rs +++ b/crates/common/src/beacon/types/chain.rs @@ -81,13 +81,29 @@ pub struct PayloadAttributesEventData { #[serde(with = "serde_utils::quoted_u64")] pub proposer_index: u64, pub proposal_slot: Slot, - #[serde(with = "serde_utils::quoted_u64")] - pub parent_block_number: u64, + /// Absent from Gloas, which no longer carries it. + #[serde(default, with = "quoted_u64_opt")] + pub parent_block_number: Option, pub parent_block_root: String, pub parent_block_hash: B256, pub payload_attributes: PayloadAttributes, } +mod quoted_u64_opt { + use serde::{Deserialize, Deserializer, Serialize, Serializer}; + use serde_utils::quoted_u64::Quoted; + + pub fn serialize(value: &Option, serializer: S) -> Result { + value.map(|value| Quoted { value }).serialize(serializer) + } + + pub fn deserialize<'de, D: Deserializer<'de>>( + deserializer: D, + ) -> Result, D::Error> { + Ok(Option::>::deserialize(deserializer)?.map(|quoted| quoted.value)) + } +} + #[derive(Debug, Serialize, Deserialize, Clone, Default)] pub struct PayloadAttributes { #[serde(with = "serde_utils::quoted_u64")] From c1f707afdebb318d796bcee0ffd9b7dc3294ec7d Mon Sep 17 00:00:00 2001 From: owen Date: Wed, 9 Sep 2026 18:34:34 +0100 Subject: [PATCH 3/5] Skip the proposer duties write when there are no duties An empty duty list built `VALUES ON CONFLICT`, which Postgres refused, so a relay with no registered proposer logged a failure every slot. The aborted transaction changed nothing, so returning early keeps the same result without the error. Co-Authored-By: Claude Opus 5 (1M context) --- crates/database/src/postgres/postgres_db_service.rs | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/crates/database/src/postgres/postgres_db_service.rs b/crates/database/src/postgres/postgres_db_service.rs index 6bb1a8036..66ea43551 100644 --- a/crates/database/src/postgres/postgres_db_service.rs +++ b/crates/database/src/postgres/postgres_db_service.rs @@ -1219,6 +1219,11 @@ impl PostgresDatabaseService { ) -> Result<(), DatabaseError> { let mut record = DbMetricRecord::new("set_proposer_duties"); + if proposer_duties.is_empty() { + record.record_success(); + return Ok(()); + } + let mut client = self.high_priority_pool.get().await?; let transaction = client.transaction().await?; From 97fe2e3da469d697c91baf7b1c46b76e87b3f98a Mon Sep 17 00:00:00 2001 From: owen Date: Thu, 10 Sep 2026 01:03:29 +0100 Subject: [PATCH 4/5] Install the crypto provider before the builder boots The building role's HTTP client panicked on the main thread with "Could not automatically determine the process-level CryptoProvider", which killed the process and took the simulation role and the embedded node with it. The relay and the data API already install the provider at startup; the builder now does the same. Co-Authored-By: Claude Opus 5 (1M context) --- crates/builder/src/main.rs | 3 +++ 1 file changed, 3 insertions(+) diff --git a/crates/builder/src/main.rs b/crates/builder/src/main.rs index 8e27f6290..6a1470957 100644 --- a/crates/builder/src/main.rs +++ b/crates/builder/src/main.rs @@ -4,6 +4,7 @@ use flux::{ tile::{TileConfig, attach_tile}, utils::ThreadPriority, }; +use helix_common::utils::install_default_crypto_provider; use tracing::info; use tracing_subscriber::EnvFilter; @@ -31,6 +32,8 @@ use validation::{BlockValidator, server as validation_server}; static ALLOC: tikv_jemallocator::Jemalloc = tikv_jemallocator::Jemalloc; fn main() -> eyre::Result<()> { + install_default_crypto_provider(); + let cli = BuilderCli::parse(); init_tracing(&cli); From 92a2ace901223fe68851f285dac40dadbc6ba2cb Mon Sep 17 00:00:00 2001 From: owen Date: Thu, 10 Sep 2026 01:25:53 +0100 Subject: [PATCH 5/5] Announce the payment in gwei and leave the enshrined value at zero The bid's `value` and `execution_payment` are gwei, checked against the builder's on-chain balance, but the relay put a wei figure in both. A 0.001 ETH bid claimed a million ETH, and anything above 18.4 ETH pinned to u64::MAX. The proposer is paid in-block, so `value` is now 0 and only `execution_payment` carries the amount, converted to gwei. Co-Authored-By: Claude Opus 5 (1M context) --- .../src/auctioneer/get_execution_payload_bid.rs | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/crates/relay/src/auctioneer/get_execution_payload_bid.rs b/crates/relay/src/auctioneer/get_execution_payload_bid.rs index edbca72cd..429fecbf8 100644 --- a/crates/relay/src/auctioneer/get_execution_payload_bid.rs +++ b/crates/relay/src/auctioneer/get_execution_payload_bid.rs @@ -7,6 +7,8 @@ use tokio::sync::oneshot; use tracing::warn; use tree_hash::TreeHash; +const WEI_PER_GWEI: u64 = 1_000_000_000; + use crate::{ api::proposer::{GloasBuilderIdentity, ProposerApiError}, auctioneer::{ @@ -80,8 +82,9 @@ pub(super) fn build_signed_bid( let execution_requests = execution_requests_to_gloas(entry.bid_data_ref().execution_requests); let execution_requests_root = execution_requests.tree_hash_root(); - // Per gattaca-com/helix#489: no payment-split product need yet, so execution_payment = value. - let value = entry.value().saturating_to::(); + // The proposer is paid in-block, so the enshrined `value` stays 0. + let execution_payment = + (entry.value() / alloy_primitives::U256::from(WEI_PER_GWEI)).saturating_to::(); let bid = ExecutionPayloadBid { parent_block_hash: ExecutionBlockHash(params.parent_hash), @@ -92,8 +95,8 @@ pub(super) fn build_signed_bid( gas_limit: payload.gas_limit, builder_index: identity.builder_index, slot, - value, - execution_payment: value, + value: 0, + execution_payment, blob_kzg_commitments: convert_kzg_commitments_to_progressive( &entry.payload_and_blobs().blobs_bundle.commitments, ), @@ -279,7 +282,7 @@ mod tests { let block_hash = B256::repeat_byte(0x99); let parent_hash = B256::repeat_byte(0x11); let parent_root = B256::repeat_byte(0x22); - let entry = payload_entry(block_hash, 42); + let entry = payload_entry(block_hash, 42 * WEI_PER_GWEI); let identity = bid_identity(7); let params = params(parent_hash, parent_root); @@ -289,8 +292,8 @@ mod tests { assert_eq!(signed_bid.message.parent_block_hash.0, parent_hash); assert_eq!(signed_bid.message.parent_block_root, parent_root); assert_eq!(signed_bid.message.builder_index, 7); - assert_eq!(signed_bid.message.value, 42); - assert_eq!(signed_bid.message.execution_payment, 42); + assert_eq!(signed_bid.message.value, 0, "the proposer is paid in-block"); + assert_eq!(signed_bid.message.execution_payment, 42, "wei converts to gwei"); let epoch = signed_bid.message.slot.epoch(helix_types::MainnetEthSpec::slots_per_epoch()); let fork = chain_info.spec.fork_at_epoch(epoch);