diff --git a/Cargo.lock b/Cargo.lock index 4e216ed..f98c323 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1574,6 +1574,7 @@ dependencies = [ "prometheus", "reqwest", "rust_decimal", + "rustls", "sea-orm", "serde", "serde_json", @@ -1641,6 +1642,7 @@ dependencies = [ "lazy_static", "prometheus", "rust_decimal", + "rustls", "sea-orm", "serde", "serde_json", @@ -1653,6 +1655,7 @@ dependencies = [ "toml", "tracing", "tracing-subscriber", + "yellowstone-grpc-client", "yellowstone-grpc-proto", ] diff --git a/README.md b/README.md index 24664ce..bec2bc4 100644 --- a/README.md +++ b/README.md @@ -27,16 +27,12 @@ The API server exposes the following JSON-RPC methods: | `getLargestAccounts` | Returns the 20 largest accounts by lamport balance, with `filter: circulating\|nonCirculating` support. Optional; requires `[largest-accounts]` on both indexer and API. See [Largest Accounts](#largest-accounts-getlargestaccounts-gettokenlargestaccounts). | | `getSupply` | Returns the total and circulating supply in lamports plus the non-circulating account list. Optional; requires the `[supply]` section on the indexer and the `[supply]` section on the API. See [Supply](#supply-getsupply). | -Only **confirmed** and **finalized** commitment levels are fully supported. By default, requests with `processed` commitment return an error. This can be overridden via the `processed-commitment` configuration option (see [API Configuration](#api-server-cloudbreakapitoml)). +**Confirmed** and **finalized** commitment levels are supported for every method. With the optional `[processed-accounts]` section, `getAccountInfo`, `getMultipleAccounts`, `getBalance`, `getTokenAccountBalance`, `getTokenSupply` and `getSlot` also serve **processed** commitment from an in-memory block feed. Every other method handles `processed` through the `processed-commitment` option, which rejects it by default (see [API Configuration](#api-server-cloudbreakapitoml)). > **Note on `getVersion`.** The `solana-core` field returned by Cloudbreak is a *composite* string of the form `"-cloudbreak"` (e.g. `"2.0.21-cloudbreak0.1.0"`). The upstream half is the `solana-core` version reported by the gRPC source the indexer is subscribed to (persisted to the `environment_info` table on indexer startup); the suffix is Cloudbreak's own crate version. This lets clients see *both* what cluster they're effectively talking to and which Cloudbreak build is serving them. If the indexer has never written an upstream version, the prefix falls back to `"unknown"`. The response is cached in-process for 10 minutes. ## Roadmap -### Processed Commitment Level - -Full native support for the `processed` commitment level is planned as an **optional plugin**, allowing operators to enable it when low-latency reads of unconfirmed state are needed. In the meantime, operators can set `processed-commitment = "use-confirmed"` in the API config to respond with `confirmed` data instead of rejecting `processed` requests. - ### Paginated Responses Support for paginated responses to queries is planned, enabling clients to efficiently iterate over large result sets without loading all accounts into a single response. @@ -514,6 +510,27 @@ Example: processed-commitment = "use-confirmed" ``` +#### `[processed-accounts]` (optional) + +Serves `processed` commitment for `getAccountInfo`, `getMultipleAccounts`, `getBalance`, `getTokenAccountBalance`, `getTokenSupply` and `getSlot`. The API subscribes to Yellowstone blocks at processed commitment and keeps the blocks around the Postgres confirmed slot in memory. A key written in those blocks is answered from memory. Any other key reads Postgres at the confirmed slot. When the blocks cannot be linked to the confirmed slot, a processed request answers exactly as a confirmed request would, even with `processed-commitment = "reject"`. Other methods keep following `processed-commitment`. + +Requires `[slot-syncronizer]` with `enabled = true`. Startup fails without it. Each API instance carries its own block feed. + +| Field | Type | Default | Description | +| ---------- | -------- | ------- | ------------------------------------------------------------------------- | +| `enabled` | `bool` | `false` | Enable processed commitment for the methods above. | +| `endpoint` | `string` | `""` | Yellowstone gRPC endpoint that allows processed commitment and interslot updates. | +| `x-token` | `string` | none | Yellowstone gRPC access token. | + +Example: + +```toml +[processed-accounts] +enabled = true +endpoint = "https://grpc.example:443" +x-token = "..." +``` + #### `unhealthy-response` (top-level, optional) Controls how the API responds to requests while the node is unhealthy (the `slots.health` flag is unset / the slot syncronizer reports unhealthy). This is a top-level key (not inside any section). @@ -548,7 +565,7 @@ OpenTelemetry tracing configuration. If this section is omitted, OTel is disable | `max-batch-size` | `u64` | (none) | Max spans per export batch. | | `max-queue-size` | `u64` | (none) | Max queued spans before dropping. | | `track-idle-time` | `bool` | (none) | Include idle time in spans. | -| `span-filter` | `Vec` | (none) | Span names to export. See `example.cloudbreak.api.toml` for the current full list (covers gPA, gTABO/gTABD, JSON encoding, HTTP transport, mint lookups, and the cache finalize span). | +| `span-filter` | `Vec` | (none) | Span names to export. See `example.cloudbreak.api.toml` for the current full list (covers gPA, gTABO/gTABD, JSON encoding, HTTP transport, mint lookups, the cache finalize span, and the processed and account reads). | ### Query Tracker Service (`cloudbreak.query-tracker.toml`) @@ -843,7 +860,7 @@ All metrics are emitted in the Prometheus text exposition format on each service | Metric | Type | Labels | Description | | --------------------------------------------------- | ----------------- | ---------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | | `cloudbreak_api_requests_total` | Counter | `method`, `status` | Count of RPC method invocations grouped by outcome. `method` ∈ {`gPA`, `gTABO`, `gTABD`, `gTABM`, `gAI`, `getBalance`, `getMultipleAccounts`, `getTokenAccountBalance`, `http`}: `gPA` = `getProgramAccounts`, `gTABO` = `getTokenAccountsByOwner`, `gTABD` = `getTokenAccountsByDelegate`, `gTABM` = `getTokenAccountsByMint`, `gAI` = `getAccountInfo`, `http` is connection-level. `status` ∈ {`success`, `error`, `timeout`}: `error` is incremented on RPC-level failures (bad params, DB failure, stream-mid-error); `timeout` is incremented on `http` when the total `request-timeout` fires. The point-lookup methods (`gAI`, `getBalance`, `getMultipleAccounts`, `getTokenAccountBalance`) emit both `success` and `error`. The streaming methods (`gPA`, `gTABO`, `gTABD`, `gTABM`) emit `error` only — for their total throughput use `cloudbreak_api_request_duration_ms` instead. Note: `getSlot`, `getHealth`, `getVersion`, and `getGenesisHash` are not currently surfaced under this counter. | -| `cloudbreak_api_request_duration_ms` | Histogram | `method`, `bytes` | Per-stage request latency in milliseconds. `bytes` is the response-size bucket (`0-1KB`, `1-10KB`, `10-100KB`, `100KB-1MB`, `1MB-10MB`, `10MB-50MB`, `50MB-100MB`, `100MB-200MB`, `200MB-500MB`, `500MB+`). `method` values: `gpa` / `gpa_mint` (total in-handler time for `getProgramAccounts`, with `_mint` suffix when a token-mint filter is applied), `gpa_db` (Postgres query time), `gpa_db_first_row_time` (time-to-first-row), `gpa_encode` (account-encoding time), `gpa_json` (JSON serialization time); analogous `gtabo*` / `gtabd*` for the token-account methods; `gAI` / `getBalance` / `getMultipleAccounts` / `getTokenAccountBalance` (single observation per request — total handler + serialization time for the point-lookup methods); `http_with_transport` (end-to-end including body transport, label `bytes` reflects response size); `http_connection` (per-TCP-connection lifetime, label `bytes="0"`). | +| `cloudbreak_api_request_duration_ms` | Histogram | `method`, `bytes` | Per-stage request latency in milliseconds. `bytes` is the response-size bucket (`0-1KB`, `1-10KB`, `10-100KB`, `100KB-1MB`, `1MB-10MB`, `10MB-50MB`, `50MB-100MB`, `100MB-200MB`, `200MB-500MB`, `500MB+`). `method` values: `gpa` / `gpa_mint` (total in-handler time for `getProgramAccounts`, with `_mint` suffix when a token-mint filter is applied), `gpa_db` (Postgres query time), `gpa_db_first_row_time` (time-to-first-row), `gpa_encode` (account-encoding time), `gpa_json` (JSON serialization time); analogous `gtabo*` / `gtabd*` for the token-account methods; `gAI` / `getBalance` / `getMultipleAccounts` / `getTokenAccountBalance` / `getTokenSupply` (single observation per request — total handler + serialization time for the point-lookup methods, in fractional milliseconds); `http_with_transport` (end-to-end including body transport, label `bytes` reflects response size); `http_connection` (per-TCP-connection lifetime, label `bytes="0"`). | | `cloudbreak_api_requests_by_subscription_id` | Counter | `subscription_id_key` | Per-client request counter, attributed via the HTTP header configured by `[metrics].subscription-id-key` (default `x-subscription-id`). Only the four heavy methods report here: `getProgramAccounts`, `getTokenAccountsByMint`, `getTokenAccountsByOwner`, `getTokenAccountsByDelegate` — the point-lookup methods (`getAccountInfo`, `getBalance`, …) are not counted. Requests with the header absent fold into `unknown-subscription-id`. Batch requests count once per batch entry. | | `cloudbreak_api_data_fetched_by_subscription_id` | Counter | `subscription_id_key` | Per-client cumulative response size in bytes (JSON-encoded payload). Same method scope as `cloudbreak_api_requests_by_subscription_id`. | | `cloudbreak_api_duration_us_by_subscription_id` | Counter | `subscription_id_key` | Per-client cumulative request handling time in **microseconds** — divide by 1000 for milliseconds. Measures handler entry through the last encoded JSON chunk (the same duration the `json_encoding` span reports as `total_wall_time`), excluding HTTP body transport. Same method scope as `cloudbreak_api_requests_by_subscription_id`, so dividing the two yields a mean latency over those methods only. | @@ -856,6 +873,8 @@ All metrics are emitted in the Prometheus text exposition format on each service | `cloudbreak_gpa_cache_max_bytes` | IntGauge | — | Configured maximum size of the GPA cache in bytes (`[gpa-cache].max-total-bytes`). | | `cloudbreak_gpa_cache_evictions_total` | Counter | `used` | GPA cache entries evicted by cleanup to make room for a different query. `used` ∈ {`used`, `unused`} indicating whether the evicted entry had ever served a cache hit. A high rate of `unused` evictions indicates cache churn (e.g. `min-bytes-per-query` set too low). | | `cloudbreak_gpa_cache_evicted_bytes_total` | Counter | `used` | Total bytes evicted from the GPA cache by cleanup, with the same `used` labelling as `cloudbreak_gpa_cache_evictions_total`. | +| `cloudbreak_api_processed_requests_total` | Counter | `method`, `route`, `reason` | Processed commitment requests for the methods `[processed-accounts]` serves. `route` is `view` when the processed blocks answer, `degraded` when the request reads as confirmed. `reason` is `none`, `no_blocks`, `unhealthy`, `head_behind` or `finalized_above_anchor`. Registered only when `[processed-accounts]` is enabled. | +| `cloudbreak_api_processed_confirm_latency_ms` | Histogram | — | Time from receiving a processed block to the Postgres confirmed slot reaching it, in milliseconds. Registered only when `[processed-accounts]` is enabled. | ### Indexer (`cloudbreak-index`) diff --git a/crates/api/Cargo.toml b/crates/api/Cargo.toml index e141ff3..14ced55 100644 --- a/crates/api/Cargo.toml +++ b/crates/api/Cargo.toml @@ -16,6 +16,7 @@ authors = [ ] [dependencies] +rustls = { workspace = true } cloudbreak-core = { workspace = true } serde = { workspace = true } sea-orm = { workspace = true } diff --git a/crates/api/src/db/getBalance.sql b/crates/api/src/db/getBalance.sql index c97966c..989de57 100644 --- a/crates/api/src/db/getBalance.sql +++ b/crates/api/src/db/getBalance.sql @@ -4,11 +4,12 @@ */ -- $1 = pubkey (bytea literal) --- $2 = commitment level (integer) +-- $2 = slot bound: a slot literal, or a subquery on the slots table. +-- No row when the bound is NULL, so a missing slots entry answers no rows. WITH latest_slot AS ( SELECT slot - FROM slots - WHERE commitment = $2 + FROM (SELECT $2::bigint AS slot) AS bound + WHERE slot IS NOT NULL ), all_versions AS ( diff --git a/crates/api/src/db/getTokenAccountBalance.sql b/crates/api/src/db/getTokenAccountBalance.sql deleted file mode 100644 index e4e6252..0000000 --- a/crates/api/src/db/getTokenAccountBalance.sql +++ /dev/null @@ -1,106 +0,0 @@ --- SPDX-License-Identifier: AGPL-3.0-only -/* - * Copyright 2025-2026 Triton One Limited. All rights reserved. - */ - -WITH all_versions AS ( - SELECT - accounts.pubkey, - accounts.owner, - accounts.lamports, - accounts.slot, - accounts.data, - accounts.token_mint - FROM accounts - WHERE - accounts.pubkey = $1 - AND accounts.slot <= $2 - UNION ALL - SELECT - snapshot_accounts.pubkey, - snapshot_accounts.owner, - snapshot_accounts.lamports, - snapshot_accounts.slot, - snapshot_accounts.data, - snapshot_accounts.token_mint - FROM snapshot_accounts - WHERE - snapshot_accounts.pubkey = $1 - AND snapshot_accounts.slot <= $2 -), - -latest_account AS ( - SELECT - pubkey, - owner, - lamports, - slot, - data, - token_mint - FROM all_versions - ORDER BY slot DESC - LIMIT 1 -), - -needed_mint AS ( - SELECT token_mint AS mint_pubkey - FROM latest_account - WHERE - lamports > 0 - AND token_mint IS NOT NULL - AND ( - owner = '\x06ddf6e1d765a193d9cbe146ceeb79ac1cb485ed5f5b37913a8cf5857eff00a9'::bytea -- TokenkegQfeZyiNwAJbNbGKPFXCWuBvf9Ss623VQ5DA -- noqa: LT05 - OR owner = '\x06ddf6e1ee758fde18425dbce46ccddab61afc4d83b90d27febdf928d8a18bfc'::bytea -- TokenzQdBNbLqP5VEhdkAS6EPFLC1PHnBqCXEpPxuEb -- noqa: LT05 - ) -), - -all_mint_versions AS NOT MATERIALIZED ( - SELECT - accounts.pubkey, - accounts.data, - accounts.slot, - accounts.lamports - FROM accounts - INNER JOIN needed_mint ON accounts.pubkey = needed_mint.mint_pubkey - WHERE accounts.slot <= $2 - UNION ALL - SELECT - snapshot_accounts.pubkey, - snapshot_accounts.data, - snapshot_accounts.slot, - snapshot_accounts.lamports - FROM snapshot_accounts - INNER JOIN needed_mint ON snapshot_accounts.pubkey = needed_mint.mint_pubkey - WHERE snapshot_accounts.slot <= $2 -), - -mint AS ( - -- Same filter-after-DISTINCT-ON pattern as getAccountInfoWithMintData.sql. - SELECT - pubkey, - mint_data - FROM ( - SELECT DISTINCT ON (pubkey) - pubkey, - data AS mint_data, - lamports - FROM all_mint_versions - ORDER BY pubkey ASC, slot DESC - ) AS latest_per_pubkey - WHERE lamports > 0 -) - -SELECT - latest_account.owner, - latest_account.token_mint, - mint.mint_data, - CASE - WHEN - latest_account.owner = '\x06ddf6e1d765a193d9cbe146ceeb79ac1cb485ed5f5b37913a8cf5857eff00a9'::bytea -- noqa: LT05 - OR latest_account.owner = '\x06ddf6e1ee758fde18425dbce46ccddab61afc4d83b90d27febdf928d8a18bfc'::bytea -- noqa: LT05 - THEN SUBSTRING(latest_account.data FROM 65 FOR 8) - ELSE '\x0000000000000000'::bytea - END AS amount -FROM latest_account -LEFT JOIN mint ON latest_account.token_mint = mint.pubkey -WHERE latest_account.lamports > 0; diff --git a/crates/api/src/db_query.rs b/crates/api/src/db_query.rs index f173d1c..a234b9c 100644 --- a/crates/api/src/db_query.rs +++ b/crates/api/src/db_query.rs @@ -89,6 +89,19 @@ pub async fn get_slot_data(db: &DatabaseConnection) -> Option Result, sea_orm::sqlx::Error> { + sea_orm::sqlx::query_scalar::<_, String>( + "SELECT blockhash FROM recent_blockhashes WHERE slot = $1", + ) + .bind(slot as i64) + .fetch_optional(db.get_postgres_connection_pool()) + .await +} + /// # W3C traceparent format: /// 00-00000000000000000000000000000123-0000000000000123-01 /// ^^ ^^ diff --git a/crates/api/src/http/mod.rs b/crates/api/src/http/mod.rs index f5ebcf0..0fba46f 100644 --- a/crates/api/src/http/mod.rs +++ b/crates/api/src/http/mod.rs @@ -3,16 +3,21 @@ * Copyright 2025-2026 Triton One Limited. All rights reserved. */ +use crate::error::RpcError; use crate::http::server::HttpHandlerResponse; use crate::http::server::ResponseBody; use crate::modules::bandwidth; use crate::modules::cache::GpaProcessor; use crate::modules::supply_cache::SharedSupplySnapshot; use crate::modules::vote_accounts_cache::SharedStakesSnapshot; -use crate::error::RpcError; use crate::query_tracker_client::QueryTrackerClient; use crate::slot_syncronizer::SlotSyncronizerData; use agave_feature_set::FeatureSet; +use cloudbreak_core::modules::processed::ProcessedAccounts; +use cloudbreak_core::{ + AccountSelectorConfig, MethodSection, ProcessedCommitmentBehavior, UnhealthyResponseBehavior, +}; +use cloudbreak_entity::slots; use hyper::StatusCode; use sea_orm::{DatabaseConnection, EntityTrait}; use serde::{Deserialize, Serialize}; @@ -21,10 +26,6 @@ use solana_rpc_client_api::response::Response as RpcResponse; use std::sync::{Arc, RwLock}; use std::time::{Duration, Instant}; use tracing::Instrument; -use cloudbreak_core::{ - AccountSelectorConfig, MethodSection, ProcessedCommitmentBehavior, UnhealthyResponseBehavior, -}; -use cloudbreak_entity::slots; #[derive(Clone)] pub struct CachedFeatureSet { @@ -120,6 +121,9 @@ pub struct CloudbreakRpcState { /// The `[token-largest-accounts]` API section; getTokenLargestAccounts is /// served when its `enabled` flag is set. pub token_largest_accounts: MethodSection, + /// The `[processed-accounts]` handle. The disabled handle routes every + /// request through `resolve_commitment`. + pub processed: ProcessedAccounts, } impl CloudbreakRpcState { @@ -145,6 +149,7 @@ impl CloudbreakRpcState { supply_cache: SharedSupplySnapshot, largest_accounts: MethodSection, token_largest_accounts: MethodSection, + processed: ProcessedAccounts, ) -> Self { Self { database, @@ -168,6 +173,7 @@ impl CloudbreakRpcState { feature_set_cache: Arc::new(RwLock::new(None)), largest_accounts, token_largest_accounts, + processed, } } @@ -176,7 +182,8 @@ impl CloudbreakRpcState { /// response layer can decide the HTTP status purely from the error. pub fn node_unhealthy(&self) -> RpcError { RpcError::NodeUnhealthy { - service_unavailable: self.unhealthy_response == UnhealthyResponseBehavior::HttpUnavailable, + service_unavailable: self.unhealthy_response + == UnhealthyResponseBehavior::HttpUnavailable, } } diff --git a/crates/api/src/http/operational_endpoints.rs b/crates/api/src/http/operational_endpoints.rs index 76afa33..1e88f25 100644 --- a/crates/api/src/http/operational_endpoints.rs +++ b/crates/api/src/http/operational_endpoints.rs @@ -3,11 +3,11 @@ * Copyright 2025-2026 Triton One Limited. All rights reserved. */ +use cloudbreak_core::modules::rpc_filter_type::RpcFilterType; use hyper::body::Incoming; use hyper::{Request, StatusCode}; use serde::Serialize; use solana_pubkey::Pubkey; -use cloudbreak_core::modules::rpc_filter_type::RpcFilterType; use std::convert::Infallible; use std::str::FromStr; use tracing_subscriber::EnvFilter; diff --git a/crates/api/src/http/rpc.rs b/crates/api/src/http/rpc.rs index 1135ee6..61b1751 100644 --- a/crates/api/src/http/rpc.rs +++ b/crates/api/src/http/rpc.rs @@ -22,8 +22,8 @@ use crate::http::CloudbreakRpcState; use crate::http::server::{HttpHandlerResponse, ResponseBody}; use crate::http::streaming::gpa_streaming_response_body; use crate::http::{ - JsonRpcRequest, JsonRpcResponse, RequestContext, RpcRequestPayload, extract_optional_param, extract_param, - http_status_for_error, make_error_response, make_error_response_with_status, + JsonRpcRequest, JsonRpcResponse, RequestContext, RpcRequestPayload, extract_optional_param, + extract_param, http_status_for_error, make_error_response, make_error_response_with_status, }; use crate::methods::slot::RpcGetSlotConfig; use crate::methods::token::{ @@ -204,7 +204,7 @@ async fn process_single_request( metrics::CLOUDBREAK_API_REQUEST_DURATION_MS .with_label_values(&["gAI", metrics::bytes_bucket(json_response.0.len() as u64)]) - .observe(start_time.elapsed().as_millis() as f64); + .observe(start_time.elapsed().as_secs_f64() * 1000.0); json_response } @@ -237,7 +237,7 @@ async fn process_single_request( "getBalance", metrics::bytes_bucket(json_response.0.len() as u64), ]) - .observe(start_time.elapsed().as_millis() as f64); + .observe(start_time.elapsed().as_secs_f64() * 1000.0); json_response } @@ -275,7 +275,7 @@ async fn process_single_request( "getMultipleAccounts", metrics::bytes_bucket(json_response.0.len() as u64), ]) - .observe(start_time.elapsed().as_millis() as f64); + .observe(start_time.elapsed().as_secs_f64() * 1000.0); json_response } @@ -439,7 +439,7 @@ async fn process_single_request( "getTokenAccountBalance", metrics::bytes_bucket(json_response.0.len() as u64), ]) - .observe(start_time.elapsed().as_millis() as f64); + .observe(start_time.elapsed().as_secs_f64() * 1000.0); json_response } @@ -478,7 +478,7 @@ async fn process_single_request( "getTokenSupply", metrics::bytes_bucket(json_response.0.len() as u64), ]) - .observe(start_time.elapsed().as_millis() as f64); + .observe(start_time.elapsed().as_secs_f64() * 1000.0); json_response } diff --git a/crates/api/src/lib.rs b/crates/api/src/lib.rs index 906fae6..9f85d48 100644 --- a/crates/api/src/lib.rs +++ b/crates/api/src/lib.rs @@ -7,6 +7,7 @@ use futures::future; use sea_orm::{ConnectOptions, Database}; use std::sync::Arc; use std::time::Duration; +use cloudbreak_core::modules::processed::ProcessedAccounts; use cloudbreak_core::{ApiConfig, EnvironmentInfo, TryLoadConfig}; use crate::{ @@ -29,6 +30,13 @@ mod slot_syncronizer; pub async fn run(config: &str) -> cloudbreak_core::Result<()> { let config = ApiConfig::try_load(config)?; + config.validate_processed_accounts()?; + + if config.processed_accounts_enabled() { + rustls::crypto::aws_lc_rs::default_provider() + .install_default() + .expect("Failed to install rustls crypto provider"); + } setup_metrics(&config)?; @@ -70,8 +78,9 @@ pub async fn run(config: &str) -> cloudbreak_core::Result<()> { client_ip: config.metrics.client_ip_key.clone(), }; + let (anchor_tx, anchor_rx) = tokio::sync::watch::channel(None); let (mut slot_syncronizer_handle, slot_syncronizer_data) = - match slot_syncronizer::start_slot_syncronizer(database.clone(), &config) { + match slot_syncronizer::start_slot_syncronizer(database.clone(), &config, anchor_tx) { Some((handle, data)) => (future::Either::Left(handle), Some(data)), None => (future::Either::Right(future::pending()), None), }; @@ -165,6 +174,11 @@ pub async fn run(config: &str) -> cloudbreak_core::Result<()> { info!("getSupply: disabled (supply-enabled is false)"); } + let processed = + ProcessedAccounts::from_config(config.processed_accounts.as_ref(), indexer_filter.clone())?; + processed.spawn(anchor_rx); + info!("processed accounts: enabled: {}", processed.is_enabled()); + let state = CloudbreakRpcState::new( database, queries_timeout, @@ -186,6 +200,7 @@ pub async fn run(config: &str) -> cloudbreak_core::Result<()> { supply_cache, largest_accounts, token_largest_accounts, + processed, ); info!("Server is starting..."); diff --git a/crates/api/src/methods/get_account_info.rs b/crates/api/src/methods/get_account_info.rs index 45ae42e..ac6f075 100644 --- a/crates/api/src/methods/get_account_info.rs +++ b/crates/api/src/methods/get_account_info.rs @@ -3,27 +3,19 @@ * Copyright 2025-2026 Triton One Limited. All rights reserved. */ -use std::sync::Arc; - -use rust_decimal::prelude::ToPrimitive; -use sea_orm::sqlx::Row; -use sea_orm::sqlx::{self}; use solana_account::AccountSharedData; use solana_account_decoder::parse_account_data::AccountAdditionalDataV3; use solana_account_decoder::{UiAccountEncoding, encode_ui_account}; use solana_account_decoder_client_types::UiAccount; -use solana_commitment_config::CommitmentLevel; use solana_pubkey::Pubkey; use solana_rpc_client_api::config::RpcAccountInfoConfig; use solana_rpc_client_api::response::{Response as RpcResponse, RpcResponseContext}; -use tokio::time::timeout; -use tracing::Instrument; use crate::error::RpcError; use crate::http::CloudbreakRpcState; use crate::methods::token::{check_account_data_len_for_encoding, parse_additional_mint_data}; -use crate::methods::{is_token_program, resolve_commitment}; -use crate::{db_query, metrics}; +use crate::methods::{is_token_program, processed}; +use crate::metrics; #[tracing::instrument(name = "gai_rpc", skip_all, fields(pubkey = %pubkey))] pub async fn get_account_info( @@ -39,15 +31,9 @@ pub async fn get_account_info( .parse() .map_err(|_| RpcError::PubkeyValidationError(pubkey.clone()))?; - let commitment = config - .commitment - .map(|commitment_config| { - resolve_commitment(commitment_config.commitment, state.processed_commitment) - }) - .transpose()? - .unwrap_or(CommitmentLevel::Finalized); + let read = processed::read(state, config.commitment, "gAI")?; - let (latest_slot, block_time) = state.latest_slot_and_block_time(commitment).await?; + let (latest_slot, block_time) = read.slot_and_block_time(state).await?; if let Some(min_context_slot) = config.min_context_slot && latest_slot < min_context_slot @@ -68,30 +54,19 @@ pub async fn get_account_info( include_str!("../db/getAccountInfo.sql") }; - let pubkey_hex = format!("'\\x{}'::bytea", hex::encode(pubkey.as_ref())); - let sql = sql_template.replace("$1", &pubkey_hex); - let sql = sql.replace("$2", &latest_slot.to_string()); - let sql = db_query::add_trace_traceparent_to_query(&sql); - - tracing::debug!(target: "gai_sql", "## sql: {}", sql); - - let pool = state.database.get_postgres_connection_pool(); - let rows = timeout(state.queries_timeout, async { - let span = tracing::info_span!("gai_db"); - sqlx::raw_sql(&sql).fetch_all(pool).instrument(span).await - }) - .await - .map_err(|_elapsed| { - tracing::error!("getAccountInfo query timed out"); - RpcError::InternalError - })? - .map_err(|e| { - tracing::error!("Database query error: {}", e); - RpcError::InternalError - })?; - - let Some(row) = rows.first() else { - // No row for this pubkey (or its only versions had lamports = 0). Account not found. + let accounts = processed::read_accounts( + state, + &read, + std::slice::from_ref(&pubkey), + sql_template, + latest_slot, + with_mint, + "getAccountInfo", + ) + .await?; + + let Some(account) = accounts.into_iter().next().flatten() else { + // No account for this pubkey (absent, or its newest version is closed). Account not found. return Ok(RpcResponse { context: RpcResponseContext { slot: latest_slot, @@ -101,8 +76,7 @@ pub async fn get_account_info( }); }; - let owner_bytes: Vec = row.get("owner"); - let owner = Pubkey::try_from(owner_bytes.as_slice()).map_err(|_| RpcError::InternalError)?; + let owner = account.owner; // Post-query indexer-filter check: if this owner is excluded by the current indexer filter error. if !state.indexer_filter.is_program_selected(&owner) { @@ -112,24 +86,15 @@ pub async fn get_account_info( }); } - let lamports = row.get::("lamports") as u64; - let executable: bool = row.get("executable"); - let rent_epoch = row - .get::("rent_epoch") - .to_u64() - .unwrap_or(0); - let data: Vec = row.get("data"); - - // For jsonParsed encoding we may have fetched the mint's data in the same SQL roundtrip. + // For jsonParsed encoding the mint's data came with the account read. // // We pass the mint pubkey to parse_additional_mint_data unconditionally (with empty - // mint_data if the join didn't return a row): that way the function's native_mint + // mint_data if the mint was not found): that way the function's native_mint // short-circuit can still hardcode decimals for WSOL. let additional_mint_data: Option = if with_mint && is_token_program(&owner) { - if let Some(mint_pubkey) = get_token_mint_from_data(&data) { - let mint_data: Vec = row.try_get("mint_data").ok().unwrap_or_default(); - parse_additional_mint_data(&mint_pubkey, &mint_data, block_time) + if let Some(mint_pubkey) = get_token_mint_from_data(&account.data) { + parse_additional_mint_data(&mint_pubkey, account.mint_data(), block_time) } else { None } @@ -138,14 +103,14 @@ pub async fn get_account_info( }; let account_shared_data = AccountSharedData::create_from_existing_shared_data( - lamports, - Arc::new(data.clone()), + account.lamports, + account.data.clone(), owner, - executable, - rent_epoch, + account.executable, + account.rent_epoch, ); - check_account_data_len_for_encoding(encoding, data_slice, data.len(), &pubkey)?; + check_account_data_len_for_encoding(encoding, data_slice, account.data.len(), &pubkey)?; // encode_ui_account computes `space = data.len()` BEFORE applying dataSlice, so we pass // the full data and let it slice — keeps `space` honest, matching Agave. diff --git a/crates/api/src/methods/get_balance.rs b/crates/api/src/methods/get_balance.rs index 22f13ed..89b0943 100644 --- a/crates/api/src/methods/get_balance.rs +++ b/crates/api/src/methods/get_balance.rs @@ -3,9 +3,9 @@ * Copyright 2025-2026 Triton One Limited. All rights reserved. */ +use cloudbreak_core::modules::processed::ProcessedAccount; use sea_orm::sqlx::Row; use sea_orm::sqlx::{self}; -use solana_commitment_config::CommitmentLevel; use solana_pubkey::Pubkey; use solana_rpc_client_api::config::RpcContextConfig; use solana_rpc_client_api::response::{Response as RpcResponse, RpcResponseContext}; @@ -14,7 +14,7 @@ use tracing::Instrument; use crate::error::RpcError; use crate::http::CloudbreakRpcState; -use crate::methods::resolve_commitment; +use crate::methods::processed; use crate::{db_query, metrics}; #[tracing::instrument(name = "get_balance_rpc", skip_all, fields(pubkey = %pubkey))] @@ -31,18 +31,48 @@ pub async fn get_balance( .parse() .map_err(|_| RpcError::PubkeyValidationError(pubkey.clone()))?; - let commitment = config - .commitment - .map(|commitment_config| { - resolve_commitment(commitment_config.commitment, state.processed_commitment) - }) - .transpose()? - .unwrap_or(CommitmentLevel::Finalized); + let read = processed::read(state, config.commitment, "getBalance")?; + + // A key written in the blocks answers from memory at the head slot. + if let Some(blocks) = &read.blocks { + let span = blocks.read_span("getBalance"); + let account = span.in_scope(|| blocks.get_account(&pubkey)); + let lamports = match account { + ProcessedAccount::Live(account) => Some(account.lamports), + ProcessedAccount::Closed => Some(0), + ProcessedAccount::Unknown => None, + }; + if let Some(lamports) = lamports { + if let Some(min_context_slot) = config.min_context_slot + && blocks.slot < min_context_slot + { + return Err(RpcError::RpcSlotBehindMinContextSlot { + rpc_slot: blocks.slot, + }); + } + return Ok(RpcResponse { + context: RpcResponseContext { + slot: blocks.slot, + api_version: None, + }, + value: lamports, + }); + } + } + + // The slot bound is the blocks anchor, or the slot at the commitment. + let slot_bound = match &read.blocks { + Some(blocks) => blocks.anchor_slot.to_string(), + None => format!( + "(SELECT slot FROM slots WHERE commitment = {})", + read.commitment as i32 + ), + }; let sql_template = include_str!("../db/getBalance.sql"); let pubkey_hex = format!("'\\x{}'::bytea", hex::encode(pubkey.as_ref())); let sql = sql_template.replace("$1", &pubkey_hex); - let sql = sql.replace("$2", &(commitment as i32).to_string()); + let sql = sql.replace("$2", &slot_bound); let sql = db_query::add_trace_traceparent_to_query(&sql); tracing::debug!(target: "get_balance_sql", "## sql: {}", sql); @@ -65,12 +95,16 @@ pub async fn get_balance( let row = rows.first().ok_or_else(|| { tracing::error!( "getBalance: slots table missing entry for commitment {:?}", - commitment + read.commitment ); RpcError::InternalError })?; - let context_slot = row.get::("context_slot") as u64; + // With blocks the context slot is the head slot. + let context_slot = match &read.blocks { + Some(blocks) => blocks.slot, + None => row.get::("context_slot") as u64, + }; if let Some(min_context_slot) = config.min_context_slot && context_slot < min_context_slot diff --git a/crates/api/src/methods/get_multiple_accounts.rs b/crates/api/src/methods/get_multiple_accounts.rs index 6310b7a..1782053 100644 --- a/crates/api/src/methods/get_multiple_accounts.rs +++ b/crates/api/src/methods/get_multiple_accounts.rs @@ -3,28 +3,19 @@ * Copyright 2025-2026 Triton One Limited. All rights reserved. */ -use std::collections::HashMap; -use std::sync::Arc; - -use rust_decimal::prelude::ToPrimitive; -use sea_orm::sqlx::Row; -use sea_orm::sqlx::{self}; use solana_account::AccountSharedData; use solana_account_decoder::parse_account_data::AccountAdditionalDataV3; use solana_account_decoder::{UiAccountEncoding, encode_ui_account}; use solana_account_decoder_client_types::UiAccount; -use solana_commitment_config::CommitmentLevel; use solana_pubkey::Pubkey; use solana_rpc_client_api::config::RpcAccountInfoConfig; use solana_rpc_client_api::response::{Response as RpcResponse, RpcResponseContext}; -use tokio::time::timeout; -use tracing::Instrument; use crate::error::RpcError; use crate::http::CloudbreakRpcState; use crate::methods::token::{check_account_data_len_for_encoding, parse_additional_mint_data}; -use crate::methods::{is_token_program, resolve_commitment}; -use crate::{db_query, metrics}; +use crate::methods::{is_token_program, processed}; +use crate::metrics; #[tracing::instrument(name = "gma_rpc", skip_all, fields(num_pubkeys = pubkeys.len()))] pub async fn get_multiple_accounts( @@ -52,15 +43,9 @@ pub async fn get_multiple_accounts( }) .collect::, _>>()?; - let commitment = config - .commitment - .map(|commitment_config| { - resolve_commitment(commitment_config.commitment, state.processed_commitment) - }) - .transpose()? - .unwrap_or(CommitmentLevel::Finalized); + let read = processed::read(state, config.commitment, "getMultipleAccounts")?; - let (latest_slot, block_time) = state.latest_slot_and_block_time(commitment).await?; + let (latest_slot, block_time) = read.slot_and_block_time(state).await?; if let Some(min_context_slot) = config.min_context_slot && latest_slot < min_context_slot @@ -91,62 +76,28 @@ pub async fn get_multiple_accounts( include_str!("../db/getMultipleAccounts.sql") }; - // Build the bytea[] array literal - let mut array_literal = String::with_capacity(parsed_pubkeys.len() * 95 + 32); - array_literal.push_str("ARRAY["); - for (i, pk) in parsed_pubkeys.iter().enumerate() { - if i > 0 { - array_literal.push_str(", "); - } - array_literal.push_str(&format!("'\\x{}'::bytea", hex::encode(pk.as_ref()))); - } - array_literal.push(']'); - - let sql = sql_template.replace("$1", &array_literal); - let sql = sql.replace("$2", &latest_slot.to_string()); - let sql = db_query::add_trace_traceparent_to_query(&sql); - - tracing::debug!(target: "gma_sql", "## sql: {}", sql); - - let pool = state.database.get_postgres_connection_pool(); - let rows = timeout(state.queries_timeout, async { - let span = tracing::info_span!("gma_db"); - sqlx::raw_sql(&sql).fetch_all(pool).instrument(span).await - }) - .await - .map_err(|_elapsed| { - tracing::error!("getMultipleAccounts query timed out"); - RpcError::InternalError - })? - .map_err(|e| { - tracing::error!("Database query error: {}", e); - RpcError::InternalError - })?; - - // Build a (pubkey -> row) lookup. The SQL only returns rows for input pubkeys - // that exist AND are live (lamports > 0). - let mut row_by_pubkey: HashMap = HashMap::with_capacity(rows.len()); - for row in &rows { - let pubkey_bytes: Vec = row.get("pubkey"); - let row_pubkey = Pubkey::try_from(pubkey_bytes.as_slice()).map_err(|_| { - tracing::error!("getMultipleAccounts: invalid pubkey bytes returned by DB"); - RpcError::InternalError - })?; - row_by_pubkey.insert(row_pubkey, row); - } + // One entry per input pubkey, in order. None = the account does not exist + // (or its latest version is closed). + let accounts = processed::read_accounts( + state, + &read, + &parsed_pubkeys, + sql_template, + latest_slot, + with_mint, + "getMultipleAccounts", + ) + .await?; let mut result: Vec> = Vec::with_capacity(parsed_pubkeys.len()); - for pubkey in &parsed_pubkeys { - // Missing in the map = account doesn't exist (or its latest version is closed). - let Some(&row) = row_by_pubkey.get(pubkey) else { + for (pubkey, account) in parsed_pubkeys.iter().zip(accounts) { + let Some(account) = account else { result.push(None); continue; }; - let owner_bytes: Vec = row.get("owner"); - let owner = - Pubkey::try_from(owner_bytes.as_slice()).map_err(|_| RpcError::InternalError)?; + let owner = account.owner; // Per-position indexer-filter check: if the owner is excluded, we return None at that position and log a tracing error. if !state.indexer_filter.is_program_selected(&owner) { @@ -160,21 +111,12 @@ pub async fn get_multiple_accounts( continue; } - let lamports = row.get::("lamports") as u64; - let executable: bool = row.get("executable"); - let rent_epoch = row - .get::("rent_epoch") - .to_u64() - .unwrap_or(0); - let data: Vec = row.get("data"); - let additional_mint_data: Option = if with_mint && is_token_program(&owner) { - if data.len() >= 32 { - let mint_pubkey = - Pubkey::try_from(&data[..32]).map_err(|_| RpcError::InternalError)?; - let mint_data: Vec = row.try_get("mint_data").ok().unwrap_or_default(); - parse_additional_mint_data(&mint_pubkey, &mint_data, block_time) + if account.data.len() >= 32 { + let mint_pubkey = Pubkey::try_from(&account.data[..32]) + .map_err(|_| RpcError::InternalError)?; + parse_additional_mint_data(&mint_pubkey, account.mint_data(), block_time) } else { None } @@ -183,14 +125,14 @@ pub async fn get_multiple_accounts( }; let account_shared_data = AccountSharedData::create_from_existing_shared_data( - lamports, - Arc::new(data.clone()), + account.lamports, + account.data.clone(), owner, - executable, - rent_epoch, + account.executable, + account.rent_epoch, ); - check_account_data_len_for_encoding(encoding, data_slice, data.len(), pubkey)?; + check_account_data_len_for_encoding(encoding, data_slice, account.data.len(), pubkey)?; let ui_account = encode_ui_account( pubkey, diff --git a/crates/api/src/methods/get_token_account_balance.rs b/crates/api/src/methods/get_token_account_balance.rs index c2793f9..aa15c05 100644 --- a/crates/api/src/methods/get_token_account_balance.rs +++ b/crates/api/src/methods/get_token_account_balance.rs @@ -3,21 +3,17 @@ * Copyright 2025-2026 Triton One Limited. All rights reserved. */ -use sea_orm::sqlx::Row; -use sea_orm::sqlx::{self}; use solana_account_decoder::parse_token::token_amount_to_ui_amount_v3; use solana_account_decoder_client_types::token::UiTokenAmount; -use solana_commitment_config::{CommitmentConfig, CommitmentLevel}; +use solana_commitment_config::CommitmentConfig; use solana_pubkey::Pubkey; use solana_rpc_client_api::response::{Response as RpcResponse, RpcResponseContext}; -use tokio::time::timeout; -use tracing::Instrument; use crate::error::RpcError; use crate::http::CloudbreakRpcState; use crate::methods::token::parse_additional_mint_data; -use crate::methods::{is_token_program, resolve_commitment}; -use crate::{db_query, metrics}; +use crate::methods::{is_token_program, processed}; +use crate::metrics; #[tracing::instrument(name = "get_token_account_balance_rpc", skip_all, fields(pubkey = %pubkey))] pub async fn get_token_account_balance( @@ -31,47 +27,31 @@ pub async fn get_token_account_balance( .parse() .map_err(|_| RpcError::PubkeyValidationError(pubkey.clone()))?; - let commitment = commitment - .map(|commitment_config| { - resolve_commitment(commitment_config.commitment, state.processed_commitment) - }) - .transpose()? - .unwrap_or(CommitmentLevel::Finalized); + let read = processed::read(state, commitment, "getTokenAccountBalance")?; - let (latest_slot, block_time) = state.latest_slot_and_block_time(commitment).await?; + let (latest_slot, block_time) = read.slot_and_block_time(state).await?; - let sql_template = include_str!("../db/getTokenAccountBalance.sql"); - let pubkey_hex = format!("'\\x{}'::bytea", hex::encode(pubkey.as_ref())); - let sql = sql_template.replace("$1", &pubkey_hex); - let sql = sql.replace("$2", &latest_slot.to_string()); - let sql = db_query::add_trace_traceparent_to_query(&sql); + // The mint JOIN supplies the mint data the balance needs. + let sql_template = include_str!("../db/getAccountInfoWithMintData.sql"); + let accounts = processed::read_accounts( + state, + &read, + std::slice::from_ref(&pubkey), + sql_template, + latest_slot, + true, + "getTokenAccountBalance", + ) + .await?; - tracing::debug!(target: "get_token_account_balance_sql", "## sql: {}", sql); - - let pool = state.database.get_postgres_connection_pool(); - let rows = timeout(state.queries_timeout, async { - let span = tracing::info_span!("get_token_account_balance_db"); - sqlx::raw_sql(&sql).fetch_all(pool).instrument(span).await - }) - .await - .map_err(|_elapsed| { - tracing::error!("getTokenAccountBalance query timed out"); - RpcError::InternalError - })? - .map_err(|e| { - tracing::error!("Database query error: {}", e); - RpcError::InternalError - })?; - - let Some(row) = rows.first() else { + let Some(account) = accounts.into_iter().next().flatten() else { // Account not in DB (or its latest version was closed) return Err(RpcError::AccountNotFound { pubkey: pubkey.to_string(), }); }; - let owner_bytes: Vec = row.get("owner"); - let owner = Pubkey::try_from(owner_bytes.as_slice()).map_err(|_| RpcError::InternalError)?; + let owner = account.owner; if !state.indexer_filter.is_program_selected(&owner) { return Err(RpcError::AccountOwnerExcluded { @@ -86,11 +66,10 @@ pub async fn get_token_account_balance( }); } - // Amount: u64 LE at bytes 64..72 of the token account data. The SQL guarantees - // exactly 8 bytes for token-owned accounts (and 8 zero bytes for anything else, - // which we've already rejected above). - let amount_bytes: Vec = row.get("amount"); - let amount_array: [u8; 8] = amount_bytes.as_slice().try_into().map_err(|_| { + // Amount: u64 LE at bytes 64..72 of the token account data. Shorter data + // yields fewer bytes, as SUBSTRING does, and fails the conversion below. + let amount_bytes = sql_substring(&account.data, 64, 8); + let amount_array: [u8; 8] = amount_bytes.try_into().map_err(|_| { tracing::error!( "getTokenAccountBalance: unexpected amount length {} for pubkey {}", amount_bytes.len(), @@ -100,23 +79,19 @@ pub async fn get_token_account_balance( })?; let amount = u64::from_le_bytes(amount_array); - // Mint pubkey from the generated token_mint column (bytes 0..32 of data). - let mint_pubkey_bytes: Vec = row.try_get("token_mint").map_err(|e| { + // Mint pubkey from bytes 0..32 of the token account data. + let mint_pubkey = Pubkey::try_from(sql_substring(&account.data, 0, 32)).map_err(|_| { tracing::error!( - "getTokenAccountBalance: missing token_mint for pubkey {}: {}", - pubkey, - e + "getTokenAccountBalance: invalid token mint for pubkey {}", + pubkey ); RpcError::InternalError })?; - let mint_pubkey = - Pubkey::try_from(mint_pubkey_bytes.as_slice()).map_err(|_| RpcError::InternalError)?; // Pass mint_data (or empty) unconditionally so the WSOL native_mint short-circuit - // can hardcode decimals=9 even when the mint account itself isn't in our DB — - // same trick we use in gAI / gTABO. - let mint_data: Vec = row.try_get("mint_data").ok().unwrap_or_default(); - let additional_mint_data = parse_additional_mint_data(&mint_pubkey, &mint_data, block_time); + // can hardcode decimals=9 even when the mint account itself isn't in our DB. + let additional_mint_data = + parse_additional_mint_data(&mint_pubkey, account.mint_data(), block_time); let additional_data = additional_mint_data .as_ref() @@ -135,3 +110,22 @@ pub async fn get_token_account_balance( value: ui_token_amount, }) } + +/// The bytes `SUBSTRING(data FROM start + 1 FOR len)` returns: shorter when data ends early. +fn sql_substring(data: &[u8], start: usize, len: usize) -> &[u8] { + let end = (start + len).min(data.len()); + &data[start.min(end)..end] +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn sql_substring_matches_postgres_on_short_data() { + let data = [1u8, 2, 3, 4, 5]; + assert_eq!(sql_substring(&data, 1, 2), &[2, 3]); + assert_eq!(sql_substring(&data, 3, 8), &[4, 5]); + assert!(sql_substring(&data, 64, 8).is_empty()); + } +} diff --git a/crates/api/src/methods/get_token_supply.rs b/crates/api/src/methods/get_token_supply.rs index c4144e1..9945c25 100644 --- a/crates/api/src/methods/get_token_supply.rs +++ b/crates/api/src/methods/get_token_supply.rs @@ -3,23 +3,19 @@ * Copyright 2025-2026 Triton One Limited. All rights reserved. */ -use sea_orm::sqlx::Row; -use sea_orm::sqlx::{self}; use solana_account_decoder::parse_token::token_amount_to_ui_amount_v3; use solana_account_decoder_client_types::token::UiTokenAmount; -use solana_commitment_config::{CommitmentConfig, CommitmentLevel}; +use solana_commitment_config::CommitmentConfig; use solana_pubkey::Pubkey; use solana_rpc_client_api::response::{Response as RpcResponse, RpcResponseContext}; use spl_token_2022::extension::StateWithExtensions; use spl_token_2022::state::Mint; -use tokio::time::timeout; -use tracing::Instrument; use crate::error::RpcError; use crate::http::CloudbreakRpcState; use crate::methods::token::parse_additional_mint_data; -use crate::methods::{is_token_program, resolve_commitment}; -use crate::{db_query, metrics}; +use crate::methods::{is_token_program, processed}; +use crate::metrics; #[tracing::instrument(name = "get_token_supply_rpc", skip_all, fields(pubkey = %mint))] pub async fn get_token_supply( @@ -33,47 +29,30 @@ pub async fn get_token_supply( .parse() .map_err(|_| RpcError::PubkeyValidationError(mint.clone()))?; - let commitment = commitment - .map(|commitment_config| { - resolve_commitment(commitment_config.commitment, state.processed_commitment) - }) - .transpose()? - .unwrap_or(CommitmentLevel::Finalized); + let read = processed::read(state, commitment, "getTokenSupply")?; - let (latest_slot, block_time) = state.latest_slot_and_block_time(commitment).await?; + let (latest_slot, block_time) = read.slot_and_block_time(state).await?; let sql_template = include_str!("../db/getAccountInfo.sql"); - let pubkey_hex = format!("'\\x{}'::bytea", hex::encode(pubkey.as_ref())); - let sql = sql_template.replace("$1", &pubkey_hex); - let sql = sql.replace("$2", &latest_slot.to_string()); - let sql = db_query::add_trace_traceparent_to_query(&sql); - - tracing::debug!(target: "get_token_supply_sql", "## sql: {}", sql); - - let pool = state.database.get_postgres_connection_pool(); - let rows = timeout(state.queries_timeout, async { - let span = tracing::info_span!("get_token_supply_db"); - sqlx::raw_sql(&sql).fetch_all(pool).instrument(span).await - }) - .await - .map_err(|_elapsed| { - tracing::error!("getTokenSupply query timed out"); - RpcError::InternalError - })? - .map_err(|e| { - tracing::error!("Database query error: {}", e); - RpcError::InternalError - })?; - - let Some(row) = rows.first() else { + let accounts = processed::read_accounts( + state, + &read, + std::slice::from_ref(&pubkey), + sql_template, + latest_slot, + false, + "getTokenSupply", + ) + .await?; + + let Some(account) = accounts.into_iter().next().flatten() else { // Account not in DB (or its latest version was closed) return Err(RpcError::AccountNotFound { pubkey: pubkey.to_string(), }); }; - let owner_bytes: Vec = row.get("owner"); - let owner = Pubkey::try_from(owner_bytes.as_slice()).map_err(|_| RpcError::InternalError)?; + let owner = account.owner; if !state.indexer_filter.is_program_selected(&owner) { return Err(RpcError::AccountOwnerExcluded { @@ -88,15 +67,15 @@ pub async fn get_token_supply( }); } - let data: Vec = row.get("data"); + let data: &[u8] = &account.data; let mint_state = - StateWithExtensions::::unpack(&data).map_err(|_| RpcError::MintDataNotFound { + StateWithExtensions::::unpack(data).map_err(|_| RpcError::MintDataNotFound { mint: pubkey.to_string(), })?; let supply = mint_state.base.supply; - let additional_mint_data = parse_additional_mint_data(&pubkey, &data, block_time); + let additional_mint_data = parse_additional_mint_data(&pubkey, data, block_time); let additional_data = additional_mint_data .as_ref() .and_then(|d| d.spl_token_additional_data.as_ref()) diff --git a/crates/api/src/methods/mod.rs b/crates/api/src/methods/mod.rs index 6f3031b..c0d481d 100644 --- a/crates/api/src/methods/mod.rs +++ b/crates/api/src/methods/mod.rs @@ -21,6 +21,7 @@ pub mod get_token_largest_accounts; pub mod get_token_supply; pub mod mint; pub mod mint_accounts; +pub(crate) mod processed; pub mod program; pub mod simulate_transaction; pub mod slot; diff --git a/crates/api/src/methods/processed.rs b/crates/api/src/methods/processed.rs new file mode 100644 index 0000000..82e84b2 --- /dev/null +++ b/crates/api/src/methods/processed.rs @@ -0,0 +1,477 @@ +// SPDX-License-Identifier: AGPL-3.0-only +/* + * Copyright 2025-2026 Triton One Limited. All rights reserved. + */ + +//! Processed commitment for getAccountInfo, getMultipleAccounts, getBalance, +//! getTokenAccountBalance, getTokenSupply and getSlot. [`ProcessedBlocks`] and its +//! account rules are documented in `cloudbreak_core::modules::processed`. +//! +//! [`read`] gives each request a [`Read`]. A processed request +//! carries the latest [`ProcessedBlocks`] while the node is healthy, the head is +//! not below the cached confirmed slot and the cached finalized slot is not above +//! the anchor. Otherwise it reads as `Confirmed`, even with +//! `processed-commitment = "reject"`. Every other request resolves through +//! [`resolve_commitment`]. +//! +//! [`read_accounts`] serves getAccountInfo, getMultipleAccounts, +//! getTokenAccountBalance and getTokenSupply at every commitment with the +//! method's own SQL. With processed blocks, keys written in them answer from +//! memory and only the unknown keys go to that SQL, bounded at the anchor. The +//! context slot and block time come from the head block. A jsonParsed token +//! account resolves its mint through the blocks first: +//! +//! | Mint in the blocks | Account from the blocks | Account from Postgres | +//! |---|---|---| +//! | `Live` | mint data from the blocks | mint data from the blocks, overriding the joined mint | +//! | `Closed` | no mint data | no mint data | +//! | `Unknown` | one `getMultipleAccounts.sql` read at the anchor | the joined mint | +//! +//! getBalance keeps `getBalance.sql` and looks its key up in the blocks itself, +//! with the same anchor bound for an unknown key. getSlot answers the head slot +//! from the blocks, and otherwise reads the `slots` row at the commitment. +//! +//! The in-memory lookup runs in a `processed_read` span with the store size in +//! `stored_blocks` and `stored_bytes`. + +use std::collections::HashMap; +use std::sync::Arc; + +use cloudbreak_core::modules::processed::{LiveAccount, ProcessedAccount, ProcessedBlocks}; +use rust_decimal::prelude::ToPrimitive; +use sea_orm::sqlx::{self, Row, postgres::PgRow}; +use solana_commitment_config::{CommitmentConfig, CommitmentLevel}; +use solana_pubkey::Pubkey; +use tokio::time::timeout; +use tracing::Instrument; + +use crate::error::RpcError; +use crate::http::CloudbreakRpcState; +use crate::methods::{is_token_program, resolve_commitment}; +use crate::slot_syncronizer::SlotSyncronizerData; +use crate::{db_query, metrics}; + +/// How one request reads: at a commitment, or through processed blocks. +pub(crate) struct Read { + pub(crate) commitment: CommitmentLevel, + /// Set for a processed request the blocks can serve. `commitment` is then `Confirmed`. + pub(crate) blocks: Option>, +} + +impl Read { + /// The context slot and block time: the head block, or the cached slot at the commitment. + pub(crate) async fn slot_and_block_time( + &self, + state: &CloudbreakRpcState, + ) -> Result<(u64, i64), RpcError> { + match &self.blocks { + Some(blocks) => Ok((blocks.slot, blocks.block_time)), + None => state.latest_slot_and_block_time(self.commitment).await, + } + } + + /// The Postgres slot bound: the blocks anchor, or the request slot. + fn sql_slot_bound(&self, latest_slot: u64) -> u64 { + self.blocks + .as_ref() + .map_or(latest_slot, |blocks| blocks.anchor_slot) + } +} + +/// Resolves the request commitment. `method` is the metric label. +pub(crate) fn read( + state: &CloudbreakRpcState, + commitment: Option, + method: &str, +) -> Result { + let commitment = commitment.map(|config| config.commitment); + if commitment != Some(CommitmentLevel::Processed) || !state.processed.is_enabled() { + let commitment = commitment + .map(|commitment| resolve_commitment(commitment, state.processed_commitment)) + .transpose()? + .unwrap_or(CommitmentLevel::Finalized); + return Ok(Read { + commitment, + blocks: None, + }); + } + // Blocks first, so the cached slots are at least as new as the blocks. + let blocks = state.processed.blocks(); + let slots = state + .slot_syncronizer_data + .as_ref() + .map_or_else(Default::default, |data| { + data.read() + .expect("Failed to read slot syncronizer data") + .clone() + }); + let reason = match &blocks { + Some(blocks) => fallback_reason(blocks.slot, blocks.anchor_slot, &slots), + None => Some("no_blocks"), + }; + let blocks = match reason { + None => { + count_route(method, "view", "none"); + blocks + } + Some(reason) => { + count_route(method, "degraded", reason); + None + } + }; + Ok(Read { + commitment: CommitmentLevel::Confirmed, + blocks, + }) +} + +/// Why blocks with this head and anchor take the confirmed path. +fn fallback_reason( + head: u64, + anchor_slot: u64, + slots: &SlotSyncronizerData, +) -> Option<&'static str> { + if !slots.healthy { + Some("unhealthy") + } else if head < slots.confirmed_slot.slot { + Some("head_behind") + } else if slots.finalized_slot.slot > anchor_slot { + Some("finalized_above_anchor") + } else { + None + } +} + +fn count_route(method: &str, route: &str, reason: &str) { + metrics::CLOUDBREAK_API_PROCESSED_REQUESTS_TOTAL + .with_label_values(&[method, route, reason]) + .inc(); +} + +/// One live account, from the blocks or a Postgres row. +#[derive(Debug, Clone)] +pub(crate) struct Account { + pub(crate) lamports: u64, + pub(crate) owner: Pubkey, + pub(crate) executable: bool, + pub(crate) rent_epoch: u64, + pub(crate) data: Arc>, + /// The joined `mint_data` column, or the mint resolved through the blocks. + mint_data: Option>>, +} + +impl Account { + fn from_live(account: &LiveAccount) -> Self { + Self { + lamports: account.lamports, + owner: account.owner, + executable: account.executable, + rent_epoch: account.rent_epoch, + data: account.data.clone(), + mint_data: None, + } + } + + fn from_row(row: &PgRow, with_mint: bool) -> Result { + let mint_data = if with_mint { + row.try_get::, _>("mint_data").ok().map(Arc::new) + } else { + None + }; + Ok(Self { + lamports: row.get::("lamports") as u64, + owner: pubkey_column(row, "owner")?, + executable: row.get("executable"), + rent_epoch: row + .get::("rent_epoch") + .to_u64() + .unwrap_or(0), + data: Arc::new(row.get("data")), + mint_data, + }) + } + + /// The mint data of a jsonParsed token account. Empty when the mint was not found. + pub(crate) fn mint_data(&self) -> &[u8] { + self.mint_data.as_deref().map_or(&[], Vec::as_slice) + } + + /// The mint of a token-program account with at least 32 bytes of data. + fn token_mint(&self) -> Option { + if !is_token_program(&self.owner) || self.data.len() < 32 { + return None; + } + Pubkey::try_from(&self.data[..32]).ok() + } +} + +/// Reads `keys` in order, `None` for an absent or closed key. Without blocks +/// every key reads Postgres with `sql_template` at `latest_slot`. With blocks, +/// a live key comes from memory and an unknown key reads Postgres at the anchor. +/// `$1` in the template takes one bytea literal, or an array when the template +/// unnests it. `what` names the method in logs. +pub(crate) async fn read_accounts( + state: &CloudbreakRpcState, + read: &Read, + keys: &[Pubkey], + sql_template: &str, + latest_slot: u64, + with_mint: bool, + what: &str, +) -> Result>, RpcError> { + let in_blocks: Vec> = match &read.blocks { + Some(blocks) => blocks + .read_span(what) + .in_scope(|| keys.iter().map(|key| blocks.get_account(key)).collect()), + None => vec![ProcessedAccount::Unknown; keys.len()], + }; + let unknown: Vec = keys + .iter() + .zip(&in_blocks) + .filter(|(_, from_blocks)| matches!(from_blocks, ProcessedAccount::Unknown)) + .map(|(key, _)| *key) + .collect(); + + let mut found = HashMap::with_capacity(unknown.len()); + if !unknown.is_empty() { + let bound = read.sql_slot_bound(latest_slot); + for row in fetch_accounts(state, sql_template, &unknown, bound, what).await? { + found.insert( + pubkey_column(&row, "pubkey")?, + Account::from_row(&row, with_mint)?, + ); + } + } + + let mut accounts = in_key_order(keys, &in_blocks, found); + if let Some(blocks) = &read.blocks + && with_mint + { + resolve_mints(state, blocks, &mut accounts, &in_blocks).await?; + } + Ok(accounts) +} + +/// Pairs each key with its account from the blocks, from Postgres, or `None`. +fn in_key_order( + keys: &[Pubkey], + in_blocks: &[ProcessedAccount<'_>], + found: HashMap, +) -> Vec> { + keys.iter() + .zip(in_blocks) + .map(|(key, from_blocks)| match from_blocks { + ProcessedAccount::Live(account) => Some(Account::from_live(account)), + ProcessedAccount::Closed => None, + ProcessedAccount::Unknown => found.get(key).cloned(), + }) + .collect() +} + +async fn resolve_mints( + state: &CloudbreakRpcState, + blocks: &ProcessedBlocks, + accounts: &mut [Option], + in_blocks: &[ProcessedAccount<'_>], +) -> Result<(), RpcError> { + let mut queried: Vec<(usize, Pubkey)> = Vec::new(); + for (index, (account, from_blocks)) in accounts.iter_mut().zip(in_blocks).enumerate() { + let Some(account) = account else { + continue; + }; + let Some(mint) = account.token_mint() else { + continue; + }; + // A mint in the blocks overrides the joined one. An account from the blocks + // whose mint is unknown reads the mint at the anchor. + let account_from_postgres = matches!(from_blocks, ProcessedAccount::Unknown); + match blocks.get_account(&mint) { + ProcessedAccount::Live(mint) => account.mint_data = Some(mint.data.clone()), + ProcessedAccount::Closed => account.mint_data = None, + ProcessedAccount::Unknown if account_from_postgres => {} + ProcessedAccount::Unknown => queried.push((index, mint)), + } + } + if queried.is_empty() { + return Ok(()); + } + + let mut mints: Vec = queried.iter().map(|(_, mint)| *mint).collect(); + mints.sort_unstable(); + mints.dedup(); + let sql_template = include_str!("../db/getMultipleAccounts.sql"); + let rows = fetch_accounts( + state, + sql_template, + &mints, + blocks.anchor_slot, + "processed mint", + ) + .await?; + let mut found = HashMap::with_capacity(rows.len()); + for row in &rows { + let data = Arc::new(row.get::, _>("data")); + found.insert(pubkey_column(row, "pubkey")?, data); + } + for (index, mint) in queried { + if let Some(account) = &mut accounts[index] { + account.mint_data = found.get(&mint).cloned(); + } + } + Ok(()) +} + +/// Runs `sql_template` for `keys` bounded at `slot` under the API query timeout. +async fn fetch_accounts( + state: &CloudbreakRpcState, + sql_template: &str, + keys: &[Pubkey], + slot: u64, + what: &str, +) -> Result, RpcError> { + let keys_literal = if sql_template.contains("unnest($1)") { + bytea_array_literal(keys) + } else { + bytea_literal(&keys[0]) + }; + let sql = sql_template.replace("$1", &keys_literal); + let sql = sql.replace("$2", &slot.to_string()); + let sql = db_query::add_trace_traceparent_to_query(&sql); + + tracing::debug!(target: "account_sql", "## {what} sql: {}", sql); + + let pool = state.database.get_postgres_connection_pool(); + timeout(state.queries_timeout, async { + let span = tracing::info_span!("account_db", method = what); + sqlx::raw_sql(&sql).fetch_all(pool).instrument(span).await + }) + .await + .map_err(|_elapsed| { + tracing::error!("{what} query timed out"); + RpcError::InternalError + })? + .map_err(|e| { + tracing::error!("Database query error: {}", e); + RpcError::InternalError + }) +} + +fn pubkey_column(row: &PgRow, column: &str) -> Result { + let bytes: Vec = row.try_get(column).map_err(|e| { + tracing::error!("missing {column} column returned by DB: {e}"); + RpcError::InternalError + })?; + Pubkey::try_from(bytes.as_slice()).map_err(|_| { + tracing::error!("invalid {column} bytes returned by DB"); + RpcError::InternalError + }) +} + +fn bytea_literal(pubkey: &Pubkey) -> String { + format!("'\\x{}'::bytea", hex::encode(pubkey.as_ref())) +} + +fn bytea_array_literal(pubkeys: &[Pubkey]) -> String { + let literals: Vec = pubkeys.iter().map(bytea_literal).collect(); + format!("ARRAY[{}]", literals.join(", ")) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::methods::LEGACY_TOKEN_PROGRAM_ID; + use crate::slot_syncronizer::SlotData; + + fn slots(confirmed: u64, finalized: u64, healthy: bool) -> SlotSyncronizerData { + SlotSyncronizerData { + confirmed_slot: SlotData { + slot: confirmed, + block_time: 0, + }, + finalized_slot: SlotData { + slot: finalized, + block_time: 0, + }, + healthy, + } + } + + #[test] + fn fallback_checks_health_head_and_finalized_against_the_cached_slots() { + assert_eq!( + fallback_reason(103, 100, &slots(100, 68, false)), + Some("unhealthy") + ); + assert_eq!( + fallback_reason(99, 98, &slots(100, 68, true)), + Some("head_behind") + ); + assert_eq!( + fallback_reason(103, 100, &slots(100, 101, true)), + Some("finalized_above_anchor") + ); + assert_eq!(fallback_reason(100, 100, &slots(100, 100, true)), None); + assert_eq!(fallback_reason(103, 100, &slots(101, 68, true)), None); + } + + fn live(owner: Pubkey, lamports: u64, data: Vec) -> LiveAccount { + LiveAccount { + lamports, + owner, + executable: false, + rent_epoch: u64::MAX, + data: Arc::new(data), + } + } + + #[test] + fn in_key_order_keeps_order_across_live_closed_and_unknown() { + let mut keys: Vec = (0..5).map(|_| Pubkey::new_unique()).collect(); + // A repeated key answers at every position. + keys[4] = keys[2]; + let live_in_blocks = live(Pubkey::new_unique(), 7, vec![]); + let in_blocks = [ + ProcessedAccount::Live(&live_in_blocks), + ProcessedAccount::Closed, + ProcessedAccount::Unknown, + ProcessedAccount::Unknown, + ProcessedAccount::Unknown, + ]; + let mut found = HashMap::new(); + found.insert( + keys[2], + Account { + lamports: 9, + owner: Pubkey::new_unique(), + executable: false, + rent_epoch: 0, + data: Arc::new(vec![]), + mint_data: None, + }, + ); + + let accounts = in_key_order(&keys, &in_blocks, found); + let lamports: Vec> = accounts + .iter() + .map(|account| account.as_ref().map(|account| account.lamports)) + .collect(); + assert_eq!(lamports, vec![Some(7), None, Some(9), None, Some(9)]); + } + + #[test] + fn token_mint_needs_a_token_owner_and_32_bytes_of_data() { + let mint = Pubkey::new_unique(); + let mut data = vec![0u8; 165]; + data[..32].copy_from_slice(mint.as_ref()); + + let account = Account::from_live(&live(LEGACY_TOKEN_PROGRAM_ID, 1, data.clone())); + assert_eq!(account.token_mint(), Some(mint)); + assert!(account.mint_data().is_empty()); + + let short = Account::from_live(&live(LEGACY_TOKEN_PROGRAM_ID, 1, data[..10].to_vec())); + assert_eq!(short.token_mint(), None); + + let other = Account::from_live(&live(Pubkey::new_unique(), 1, data)); + assert_eq!(other.token_mint(), None); + } +} diff --git a/crates/api/src/methods/slot.rs b/crates/api/src/methods/slot.rs index 8b70f9a..615df9c 100644 --- a/crates/api/src/methods/slot.rs +++ b/crates/api/src/methods/slot.rs @@ -6,11 +6,11 @@ use crate::{ error::RpcError, http::{CloudbreakApiResponse, CloudbreakRpcState}, - methods::resolve_commitment, + methods::processed, }; use sea_orm::EntityTrait; use serde::{Deserialize, Serialize}; -use solana_commitment_config::{CommitmentConfig, CommitmentLevel}; +use solana_commitment_config::CommitmentConfig; use tokio::time::Instant; use cloudbreak_entity::slots; @@ -29,11 +29,20 @@ pub async fn get_slot( ) -> Result, RpcError> { let start_time = Instant::now(); - let commitment = if let Some(commitment) = config.as_ref().and_then(|c| c.commitment) { - resolve_commitment(commitment.commitment, state.processed_commitment)? - } else { - CommitmentLevel::Finalized - }; + let min_context_slot = config.as_ref().and_then(|c| c.min_context_slot); + let read = processed::read(state, config.as_ref().and_then(|c| c.commitment), "getSlot")?; + + if let Some(blocks) = &read.blocks { + if let Some(min_slot) = min_context_slot + && blocks.slot < min_slot + { + return Err(RpcError::RpcSlotBehindMinContextSlot { + rpc_slot: blocks.slot, + }); + } + return Ok(CloudbreakApiResponse::Response(blocks.slot)); + } + let commitment = read.commitment; let slot_model = slots::Entity::find_by_id(commitment as i32) .one(&state.database) @@ -50,14 +59,14 @@ pub async fn get_slot( }; if let Some(cached_slot_data) = cached_slot_data { - if rpc_latest_slot - cached_slot_data == 1 { + if rpc_latest_slot.saturating_sub(cached_slot_data) == 1 { tracing::warn!(target: "slot_mismatch", "Slot mismatch: cached slot: {} - rpc latest slot: {} - commitment: {}", cached_slot_data, rpc_latest_slot, commitment); - } else if rpc_latest_slot - cached_slot_data > 1 { + } else if rpc_latest_slot.saturating_sub(cached_slot_data) > 1 { tracing::error!(target: "slot_mismatch", "Slot mismatch: cached slot: {} - rpc latest slot: {} - commitment: {}", cached_slot_data, rpc_latest_slot, commitment); } } - if let Some(min_slot) = config.as_ref().and_then(|c| c.min_context_slot) + if let Some(min_slot) = min_context_slot && rpc_latest_slot < min_slot { return Err(RpcError::RpcSlotBehindMinContextSlot { diff --git a/crates/api/src/metrics.rs b/crates/api/src/metrics.rs index 1b0e1f3..e0ab9fc 100644 --- a/crates/api/src/metrics.rs +++ b/crates/api/src/metrics.rs @@ -4,6 +4,7 @@ */ use cloudbreak_core::ApiConfig; +use cloudbreak_core::metrics::PROCESSED_CONFIRM_LATENCY_MS; use hyper::StatusCode; use prometheus::{ HistogramOpts, HistogramVec, IntCounterVec, IntGauge, IntGaugeVec, Opts, Registry, TextEncoder, @@ -33,7 +34,7 @@ lazy_static::lazy_static! { "Total API request latency in milliseconds, labeled by method." ) .buckets(vec![ - 1.0, 2.0, 5.0, 10.0, 25.0, 50.0, 100.0, 150.0, 200.0, 300.0, 400.0, 500.0, 650.0, 800.0, + 0.05, 0.1, 0.25, 0.5, 1.0, 2.0, 5.0, 10.0, 25.0, 50.0, 100.0, 150.0, 200.0, 300.0, 400.0, 500.0, 650.0, 800.0, 1000.0, 1500.0, 2000.0, 3000.0, 4000.0, 5000.0, 6000.0, 7000.0, 8000.0, 9000.0, 10000.0, 12000.0, 14000.0, 16000.0, 18000.0, 20000.0, 25000.0, 30000.0, 40000.0, 50000.0, 80000.0, 100000.0, 150000.0, 200000.0, 300000.0 @@ -159,6 +160,15 @@ lazy_static::lazy_static! { &["used"], ).unwrap(); + /// Processed commitment requests for the methods served from processed blocks, + /// labelled by `route` (`view` or `degraded`) and `reason` (`none` or the fallback reason). + pub static ref CLOUDBREAK_API_PROCESSED_REQUESTS_TOTAL: IntCounterVec = IntCounterVec::new( + Opts::new( + "cloudbreak_api_processed_requests_total", + "Processed commitment requests, labelled by method, route and degrade reason" + ), + &["method", "route", "reason"], + ).unwrap(); /// Queries accepted for caching that are queued on, or running on, the /// blocking pool. Expected to sit near 0; a sustained value means insertion /// is falling behind the requests producing it, which delays entries becoming @@ -304,6 +314,11 @@ pub fn setup_metrics(config: &ApiConfig) -> anyhow::Result<()> { &METRICS_REGISTRY, config.metrics.client_ip_bandwidth_enabled, ); + + if config.processed_accounts_enabled() { + register!(CLOUDBREAK_API_PROCESSED_REQUESTS_TOTAL); + register!(PROCESSED_CONFIRM_LATENCY_MS); + } }); // Set the max connections as a reference metric at startup diff --git a/crates/api/src/slot_syncronizer.rs b/crates/api/src/slot_syncronizer.rs index 6eea3f0..c1c728c 100644 --- a/crates/api/src/slot_syncronizer.rs +++ b/crates/api/src/slot_syncronizer.rs @@ -4,14 +4,15 @@ */ use crate::db_query; +use cloudbreak_core::ApiConfig; +use cloudbreak_core::modules::processed::Anchor; use sea_orm::DatabaseConnection; use solana_commitment_config::CommitmentLevel; use std::{ sync::{Arc, RwLock}, time::Duration, }; -use tokio::{task::JoinHandle, time::Instant}; -use cloudbreak_core::ApiConfig; +use tokio::{sync::watch, task::JoinHandle, time::Instant}; /// Data structure to store the confirmed and finalized slots from the slot data /// syncronizer background task @@ -52,9 +53,15 @@ pub struct SlotData { pub block_time: i64, } +/// Floor for the blockhash read timeout when the poll interval is very short. +const MIN_BLOCKHASH_READ_TIMEOUT: Duration = Duration::from_millis(100); + +/// With `[processed-accounts]` enabled, every successful poll also publishes the +/// confirmed slot and its blockhash as the processed [`Anchor`] on `anchor_tx`. pub fn start_slot_syncronizer( db: DatabaseConnection, config: &ApiConfig, + anchor_tx: watch::Sender>, ) -> Option<(JoinHandle<()>, Arc>)> { if !config.slot_syncronizer.enabled { return None; @@ -62,6 +69,7 @@ pub fn start_slot_syncronizer( let slot_syncronizer_data = Arc::new(RwLock::new(SlotSyncronizerData::default())); let delay = Duration::from_millis(config.slot_syncronizer.interval_ms); + let anchor_tx = config.processed_accounts_enabled().then_some(anchor_tx); let slot_data_clone = slot_syncronizer_data.clone(); let join_handle = tokio::spawn(async move { @@ -70,6 +78,7 @@ pub fn start_slot_syncronizer( tokio::time::sleep(delay).await; tracing::debug!(target: "slot_syncronizer", "Slot syncronizer: last time sync: {:?}", last_time_sync.elapsed().as_secs_f32()); let query_start_time = Instant::now(); + let mut confirmed = None; if let Some(db_slot_data) = db_query::get_slot_data(&db).await { let mut cached_slot_data = @@ -100,9 +109,88 @@ pub fn start_slot_syncronizer( ); last_time_sync = Instant::now(); + confirmed = Some(cached_slot_data.confirmed_slot.slot); + } + + // Published after the block so the slot cache lock is released first. + if let (Some(anchor_tx), Some(confirmed_slot)) = (&anchor_tx, confirmed) + && let Some(confirmed_blockhash) = read_blockhash(&db, confirmed_slot, delay).await + { + publish_anchor( + anchor_tx, + Anchor { + confirmed_slot, + confirmed_blockhash, + }, + ); } } }); Some((join_handle, slot_syncronizer_data)) } + +/// Reads the blockhash of `slot`, bounded by the poll interval so a slow read +/// delays the next poll by at most one interval. +async fn read_blockhash(db: &DatabaseConnection, slot: u64, delay: Duration) -> Option { + let read_timeout = delay.max(MIN_BLOCKHASH_READ_TIMEOUT); + match tokio::time::timeout(read_timeout, db_query::get_blockhash_at_slot(db, slot)).await { + Ok(Ok(blockhash)) => blockhash, + Ok(Err(e)) => { + tracing::warn!(target: "slot_syncronizer", "Confirmed blockhash read failed for slot {slot}: {e}"); + None + } + Err(_elapsed) => { + tracing::warn!(target: "slot_syncronizer", "Confirmed blockhash read for slot {slot} timed out after {read_timeout:?}"); + None + } + } +} + +/// Publishes the anchor when it differs from the last one published. +fn publish_anchor(anchor_tx: &watch::Sender>, anchor: Anchor) { + anchor_tx.send_if_modified(|current| { + if current.as_ref() == Some(&anchor) { + return false; + } + *current = Some(anchor); + true + }); +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn anchor_is_published_once_per_change() { + let (anchor_tx, mut anchor_rx) = watch::channel(None); + let anchor = Anchor { + confirmed_slot: 100, + confirmed_blockhash: "h100".to_string(), + }; + publish_anchor(&anchor_tx, anchor.clone()); + assert!(anchor_rx.has_changed().unwrap()); + assert_eq!(anchor_rx.borrow_and_update().as_ref(), Some(&anchor)); + + publish_anchor(&anchor_tx, anchor.clone()); + assert!(!anchor_rx.has_changed().unwrap()); + + publish_anchor( + &anchor_tx, + Anchor { + confirmed_slot: 101, + ..anchor + }, + ); + assert!(anchor_rx.has_changed().unwrap()); + assert_eq!( + anchor_rx + .borrow_and_update() + .as_ref() + .unwrap() + .confirmed_slot, + 101 + ); + } +} diff --git a/crates/core/Cargo.toml b/crates/core/Cargo.toml index 65bb82d..05fd177 100644 --- a/crates/core/Cargo.toml +++ b/crates/core/Cargo.toml @@ -28,6 +28,7 @@ tokio = { workspace = true } toml = { workspace = true } cloudbreak-entity = { workspace = true } yellowstone-grpc-proto = { workspace = true } +yellowstone-grpc-client = { workspace = true } solana-pubkey = { workspace = true } solana-program = { workspace = true } solana-stake-interface = { workspace = true, features = ["serde"] } @@ -43,3 +44,4 @@ lazy_static = { workspace = true } [dev-dependencies] serde_json = { workspace = true } +rustls = { workspace = true } diff --git a/crates/core/src/config.rs b/crates/core/src/config.rs index 9d07622..1c2b522 100644 --- a/crates/core/src/config.rs +++ b/crates/core/src/config.rs @@ -753,6 +753,25 @@ pub struct ApiConfig { /// `[token-largest-accounts]` state. #[serde(rename = "token-largest-accounts", default)] pub token_largest_accounts: Option, + /// Serves processed commitment for getAccountInfo, getMultipleAccounts, getBalance, + /// getTokenAccountBalance, getTokenSupply and getSlot. Requires `[slot-syncronizer]`. + #[serde(rename = "processed-accounts", default)] + pub processed_accounts: Option, +} + +/// The in-memory processed blocks around the Postgres confirmed slot, fed by a +/// Yellowstone block subscription. See `modules::processed`. +#[derive(Deserialize, Debug, Clone)] +#[serde(deny_unknown_fields)] +pub struct ProcessedAccountsConfig { + #[serde(default)] + pub enabled: bool, + /// Yellowstone gRPC endpoint that allows processed commitment and interslot updates. + #[serde(default)] + pub endpoint: String, + /// Sent as the `x-token` header. + #[serde(rename = "x-token", default)] + pub x_token: Option, } /// Config section for an optional API method; the method is served only when @@ -1414,6 +1433,21 @@ impl ApiConfig { pub fn supply_enabled(&self) -> bool { self.supply.as_ref().is_some_and(|supply| supply.enabled) } + + pub fn processed_accounts_enabled(&self) -> bool { + self.processed_accounts + .as_ref() + .is_some_and(|processed| processed.enabled) + } + + /// Checks the `[slot-syncronizer]` requirement of an enabled `[processed-accounts]`. + pub fn validate_processed_accounts(&self) -> Result<()> { + anyhow::ensure!( + !self.processed_accounts_enabled() || self.slot_syncronizer.enabled, + "processed-accounts requires [slot-syncronizer] enabled = true" + ); + Ok(()) + } } #[derive(Deserialize, Debug)] @@ -1671,4 +1705,50 @@ mod tests { // Neutral value guard by default: candidate and incumbent scores compared as-is. assert_eq!(c.value_guard_creation_bias, 1.0); } + + const API_BASE: &str = r#" +[database] +url = "postgres://localhost/cloudbreak" + +[server] + +[metrics] +"#; + + fn api_config(extra: &str) -> Result { + Ok(toml::from_str(&format!("{API_BASE}\n{extra}"))?) + } + + #[test] + fn processed_accounts_absent_or_disabled_skips_validation() { + for extra in [ + "", + "[slot-syncronizer]\nenabled = false\ninterval_ms = 200\n\n[processed-accounts]\nenabled = false\n", + ] { + let config = api_config(extra).unwrap(); + assert!(!config.processed_accounts_enabled()); + config.validate_processed_accounts().unwrap(); + } + } + + #[test] + fn processed_accounts_rejects_unknown_field() { + let err = api_config("[processed-accounts]\nenabled = true\nretain-slots = 1\n") + .unwrap_err() + .to_string(); + assert!(err.contains("unknown field"), "{err}"); + } + + #[test] + fn processed_accounts_requires_slot_syncronizer() { + let config = api_config( + "[slot-syncronizer]\nenabled = false\ninterval_ms = 200\n\n[processed-accounts]\nenabled = true\nendpoint = \"http://grpc\"\n", + ) + .unwrap(); + let err = config + .validate_processed_accounts() + .unwrap_err() + .to_string(); + assert!(err.contains("slot-syncronizer"), "{err}"); + } } diff --git a/crates/core/src/grpc.rs b/crates/core/src/grpc.rs new file mode 100644 index 0000000..f6d6f75 --- /dev/null +++ b/crates/core/src/grpc.rs @@ -0,0 +1,296 @@ +// SPDX-License-Identifier: AGPL-3.0-only +/* + * Copyright 2025-2026 Triton One Limited. All rights reserved. + */ + +//! Yellowstone gRPC subscription with reconnection, shared by the indexer and +//! the API processed feed. +//! +//! [`subscribe_with_reconnection`] connects, runs [`Subscriber::on_connect`], +//! subscribes with [`Subscriber::request`] and hands the stream to +//! [`Subscriber::session`] until it ends. A failed connect or subscribe, or a +//! session that errored before it delivered a block, opens the give-up window. +//! While the window is open every attempt first waits `reconnect_backoff`, +//! lets the request replay only while the window is younger than +//! `reconnect_from_slot_retain`, and, with `reconnect_give_up` set, panics +//! once the window is older than that. `reconnect_give_up = None` keeps +//! retrying. A session that delivered a block, or ended without an error, +//! closes the window and reconnects at once. The loop returns only when +//! [`Subscriber::cancelled`] is true. + +use std::collections::HashMap; +use std::panic::AssertUnwindSafe; +use std::time::Duration; + +use futures::{FutureExt, Stream}; +use tokio::time::Instant; +use yellowstone_grpc_client::{ClientTlsConfig, GeyserGrpcClient}; +use yellowstone_grpc_proto::geyser::{ + CommitmentLevel, SubscribeRequest, SubscribeRequestFilterBlocks, SubscribeRequestFilterSlots, + SubscribeUpdate, +}; +use yellowstone_grpc_proto::tonic::{Status, codec::CompressionEncoding}; + +const KEEPALIVE: Duration = Duration::from_secs(10); + +#[derive(Debug, Clone)] +pub struct GrpcClientOptions { + pub endpoint: String, + /// Sent as the `x-token` header when set. + pub x_token: Option, + /// Bound on connecting and on each request. + pub timeout: Duration, + pub max_decoding_message_size: usize, + /// Wait before an attempt while the give-up window is open. + pub reconnect_backoff: Duration, + /// Panics when the give-up window is older than this. `None` keeps retrying. + pub reconnect_give_up: Option, + /// The request may replay from a slot only while the give-up window is younger than this. + pub reconnect_from_slot_retain: Duration, +} + +/// How a session ended. `Failed` is an error before any block, which keeps the +/// give-up window open. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum SessionEnd { + Healthy, + Failed, +} + +/// The caller side of one subscription. +pub trait Subscriber: Send { + /// The request for one session. `replay` is true when the loop allows a + /// reconnect to set `from_slot`. + fn request(&self, replay: bool) -> SubscribeRequest; + + /// True when the loop must stop. + fn cancelled(&self) -> bool; + + /// Runs after a connect or subscribe failure. + fn on_connect_failed(&mut self); + + /// Runs on every new connection, before the subscribe. + fn on_connect(&mut self, client: &mut GeyserGrpcClient) -> impl Future + Send; + + /// Consumes one stream until it ends, errors or stalls. + fn session( + &mut self, + stream: impl Stream> + Send, + ) -> impl Future + Send; +} + +/// Runs the subscriber until it cancels, reconnecting as the module doc describes. +pub async fn subscribe_with_reconnection( + options: GrpcClientOptions, + mut subscriber: S, +) { + let mut reconnect_failed_since: Option = None; + let mut is_reconnect = false; + + loop { + if subscriber.cancelled() { + tracing::info!("GRPC subscription cancelled"); + return; + } + + if let Some(started) = reconnect_failed_since { + if options + .reconnect_give_up + .is_some_and(|give_up| started.elapsed() >= give_up) + { + tracing::error!( + "Failed to (re)connect to Yellowstone GRPC after {:?}", + started.elapsed() + ); + panic!( + "Failed to (re)connect to Yellowstone GRPC after {:?}", + started.elapsed() + ); + } + tokio::time::sleep(options.reconnect_backoff).await; + } + + let mut client = match connect(&options).await { + Ok(client) => client, + Err(e) => { + reconnect_failed_since.get_or_insert_with(Instant::now); + tracing::error!("Failed to connect to Yellowstone GRPC: {:?}", e); + subscriber.on_connect_failed(); + continue; + } + }; + subscriber.on_connect(&mut client).await; + + // Replay only while the window is young. The server may not have older slots buffered. + let keep_from_slot = reconnect_failed_since + .is_none_or(|started| started.elapsed() < options.reconnect_from_slot_retain); + let request = subscriber.request(is_reconnect && keep_from_slot); + let from_slot = request.from_slot; + + let (_subscribe_tx, stream) = match client.subscribe_with_request(Some(request)).await { + Ok(subscription) => { + if let Some(slot) = from_slot { + tracing::info!( + "Reconnected to Yellowstone GRPC replaying from slot {}", + slot + ); + } + subscription + } + Err(e) => { + reconnect_failed_since.get_or_insert_with(Instant::now); + tracing::error!( + "Failed to subscribe to Yellowstone GRPC (from_slot {:?}): {:?}", + from_slot, + e + ); + subscriber.on_connect_failed(); + continue; + } + }; + + match AssertUnwindSafe(subscriber.session(stream)) + .catch_unwind() + .await + { + Ok(SessionEnd::Failed) => { + reconnect_failed_since.get_or_insert_with(Instant::now); + } + Ok(SessionEnd::Healthy) => reconnect_failed_since = None, + Err(_) => { + tracing::error!("GRPC subscription session panicked"); + reconnect_failed_since.get_or_insert_with(Instant::now); + } + } + + is_reconnect = true; + } +} + +/// The blocks-with-accounts subscription with a slot status stream. +pub fn blocks_with_accounts_request( + commitment: CommitmentLevel, + interslot_updates: bool, + from_slot: Option, +) -> SubscribeRequest { + SubscribeRequest { + accounts: HashMap::new(), + slots: HashMap::from([( + "accounts_slots".to_string(), + SubscribeRequestFilterSlots { + filter_by_commitment: Some(false), + interslot_updates: Some(interslot_updates), + }, + )]), + transactions: HashMap::new(), + transactions_status: HashMap::new(), + blocks: HashMap::from([( + "accounts_blocks".to_string(), + SubscribeRequestFilterBlocks { + account_include: vec![], + include_transactions: Some(false), + include_accounts: Some(true), + include_entries: Some(false), + cuckoo_account_include: None, + }, + )]), + blocks_meta: HashMap::new(), + entry: HashMap::new(), + commitment: Some(commitment as i32), + accounts_data_slice: Vec::new(), + ping: None, + from_slot, + } +} + +async fn connect( + options: &GrpcClientOptions, +) -> Result { + GeyserGrpcClient::build_from_shared(options.endpoint.clone()) + .expect("Failed to build GeyserGrpcClient") + .x_token(options.x_token.clone()) + .expect("Failed to set x-token") + .max_decoding_message_size(options.max_decoding_message_size) + .accept_compressed(CompressionEncoding::Zstd) + .connect_timeout(options.timeout) + .timeout(options.timeout) + .tls_config(ClientTlsConfig::new().with_native_roots()) + .expect("Failed to set tls config") + .tcp_keepalive(Some(KEEPALIVE)) + .http2_keep_alive_interval(KEEPALIVE) + .keep_alive_timeout(KEEPALIVE) + .connect() + .await +} + +#[cfg(test)] +mod tests { + use super::*; + + /// Counts failed attempts against a closed port and cancels after `stop_after`. + struct Counting { + attempts: usize, + stop_after: usize, + } + + impl Subscriber for Counting { + fn request(&self, _replay: bool) -> SubscribeRequest { + SubscribeRequest::default() + } + + fn cancelled(&self) -> bool { + self.attempts >= self.stop_after + } + + fn on_connect_failed(&mut self) { + self.attempts += 1; + } + + async fn on_connect(&mut self, _client: &mut GeyserGrpcClient) {} + + async fn session( + &mut self, + _stream: impl Stream> + Send, + ) -> SessionEnd { + SessionEnd::Healthy + } + } + + fn options(reconnect_give_up: Option) -> GrpcClientOptions { + // The binaries install the provider at startup. Tests connect without one. + let _ = rustls::crypto::aws_lc_rs::default_provider().install_default(); + GrpcClientOptions { + endpoint: "http://127.0.0.1:1".to_string(), + x_token: None, + timeout: Duration::from_millis(500), + max_decoding_message_size: 1024, + reconnect_backoff: Duration::from_millis(1), + reconnect_give_up, + reconnect_from_slot_retain: Duration::from_secs(1), + } + } + + #[tokio::test] + async fn without_give_up_the_loop_keeps_retrying_until_cancelled() { + let counting = Counting { + attempts: 0, + stop_after: 3, + }; + let done = tokio::time::timeout( + Duration::from_secs(10), + subscribe_with_reconnection(options(None), counting), + ) + .await; + assert!(done.is_ok(), "the loop must return once cancelled"); + } + + #[tokio::test] + #[should_panic(expected = "Failed to (re)connect to Yellowstone GRPC")] + async fn with_give_up_the_loop_panics_once_the_window_expires() { + let counting = Counting { + attempts: 0, + stop_after: 100, + }; + subscribe_with_reconnection(options(Some(Duration::ZERO)), counting).await; + } +} diff --git a/crates/core/src/lib.rs b/crates/core/src/lib.rs index 2ef78db..ef32f7c 100644 --- a/crates/core/src/lib.rs +++ b/crates/core/src/lib.rs @@ -8,6 +8,7 @@ use std::sync::OnceLock; use anyhow::Result as AnyhowResult; mod config; +pub mod grpc; pub mod metrics; pub mod modules; diff --git a/crates/core/src/metrics.rs b/crates/core/src/metrics.rs index a634124..af7e610 100644 --- a/crates/core/src/metrics.rs +++ b/crates/core/src/metrics.rs @@ -109,6 +109,18 @@ lazy_static::lazy_static! { &["kind"], ) .expect("Failed to create non-circulating changes counter"); + + /// Time from receiving a processed block to the store applying the Postgres + /// anchor that covers it, for blocks on the confirmed chain. Includes the + /// slot syncronizer poll. + pub static ref PROCESSED_CONFIRM_LATENCY_MS: Histogram = Histogram::with_opts( + HistogramOpts::new("cloudbreak_api_processed_confirm_latency_ms", "Processed block receipt to Postgres confirmed, in milliseconds") + .buckets(vec![ + 100.0, 200.0, 300.0, 400.0, 500.0, 750.0, 1_000.0, 1_500.0, 2_000.0, 3_000.0, + 5_000.0, 10_000.0, 20_000.0, + ]), + ) + .expect("Failed to create processed confirm latency histogram"); } /// We use a guard to increment the current tokio tasks metric when a task is created and diff --git a/crates/core/src/modules/mod.rs b/crates/core/src/modules/mod.rs index c113b25..cf03bad 100644 --- a/crates/core/src/modules/mod.rs +++ b/crates/core/src/modules/mod.rs @@ -7,6 +7,7 @@ pub mod account_owner_map; pub mod index_identity; pub mod largest_accounts; pub mod non_circulating; +pub mod processed; pub mod query_tracker_api; pub mod rpc_filter_type; pub mod service_health; diff --git a/crates/core/src/modules/processed/ingest.rs b/crates/core/src/modules/processed/ingest.rs new file mode 100644 index 0000000..b1f3dae --- /dev/null +++ b/crates/core/src/modules/processed/ingest.rs @@ -0,0 +1,189 @@ +// SPDX-License-Identifier: AGPL-3.0-only +/* + * Copyright 2025-2026 Triton One Limited. All rights reserved. + */ + +//! Builds a [`SlotBlock`] from one `SubscribeUpdateBlock`. +//! +//! Each pubkey is inserted once with no per-block dedup. The plugin seals at +//! most one entry per pubkey per block (see the upstream invariants in `mod.rs`). + +use std::collections::HashMap; +use std::sync::Arc; +use std::time::Instant; + +use solana_pubkey::Pubkey; +use yellowstone_grpc_proto::prelude::SubscribeUpdateBlock; + +use super::{AccountEntry, LiveAccount}; +use crate::config::AccountSelectorConfig; + +/// One sealed processed block, keyed in the store by slot. +pub(crate) struct SlotBlock { + pub slot: u64, + pub blockhash: String, + pub parent_slot: u64, + pub parent_blockhash: String, + pub block_time: Option, + pub received_at: Instant, + pub accounts: HashMap, + /// Estimated heap bytes: account data plus the map table. + pub heap_bytes: usize, +} + +impl SlotBlock { + /// Classifies every account. A zero-lamport write and a write whose owner + /// is outside the program filter are both closes. + pub(crate) fn from_update( + block: SubscribeUpdateBlock, + program_filter: &AccountSelectorConfig, + received_at: Instant, + ) -> Self { + let mut accounts = HashMap::with_capacity(block.accounts.len()); + let mut skipped = 0usize; + let mut data_bytes = 0usize; + + for account in block.accounts { + let (Ok(pubkey), Ok(owner)) = ( + Pubkey::try_from(account.pubkey.as_slice()), + Pubkey::try_from(account.owner.as_slice()), + ) else { + skipped += 1; + continue; + }; + let entry = if account.lamports == 0 || !program_filter.is_program_selected(&owner) { + AccountEntry::Closed + } else { + data_bytes += account.data.len(); + AccountEntry::Live(LiveAccount { + lamports: account.lamports, + owner, + executable: account.executable, + rent_epoch: account.rent_epoch, + data: Arc::new(account.data), + }) + }; + accounts.insert(pubkey, entry); + } + + if skipped > 0 { + tracing::warn!( + slot = block.slot, + skipped, + "processed block has accounts with a malformed pubkey or owner" + ); + } + + Self { + slot: block.slot, + blockhash: block.blockhash, + parent_slot: block.parent_slot, + parent_blockhash: block.parent_blockhash, + block_time: block.block_time.map(|t| t.timestamp), + received_at, + heap_bytes: data_bytes + accounts.capacity() * size_of::<(Pubkey, AccountEntry)>(), + accounts, + } + } +} + +#[cfg(test)] +pub(crate) mod tests { + use super::*; + use crate::config::PubkeyDef; + use yellowstone_grpc_proto::prelude::{SubscribeUpdateAccountInfo, UnixTimestamp}; + + pub(crate) fn account_info( + pubkey: Pubkey, + owner: Pubkey, + lamports: u64, + data: Vec, + ) -> SubscribeUpdateAccountInfo { + SubscribeUpdateAccountInfo { + pubkey: pubkey.to_bytes().to_vec(), + lamports, + owner: owner.to_bytes().to_vec(), + executable: false, + rent_epoch: u64::MAX, + data, + write_version: 0, + txn_signature: None, + } + } + + pub(crate) fn update( + slot: u64, + hash: &str, + parent_slot: u64, + parent_hash: &str, + accounts: Vec, + ) -> SubscribeUpdateBlock { + SubscribeUpdateBlock { + slot, + blockhash: hash.to_string(), + parent_slot, + parent_blockhash: parent_hash.to_string(), + block_time: Some(UnixTimestamp { + timestamp: 1_700_000_000 + slot as i64, + }), + accounts, + ..Default::default() + } + } + + fn build( + accounts: Vec, + filter: &AccountSelectorConfig, + ) -> SlotBlock { + SlotBlock::from_update(update(10, "h10", 9, "h9", accounts), filter, Instant::now()) + } + + #[test] + fn classifies_live_and_treats_excluded_as_closed() { + let included = Pubkey::new_unique(); + let excluded = Pubkey::new_unique(); + let filter = AccountSelectorConfig { + include: vec![PubkeyDef(included)], + exclude: vec![], + }; + let live_key = Pubkey::new_unique(); + let closed_key = Pubkey::new_unique(); + let excluded_key = Pubkey::new_unique(); + let block = build( + vec![ + account_info(live_key, included, 5, vec![1, 2, 3]), + account_info(closed_key, included, 0, vec![9; 100]), + account_info(excluded_key, excluded, 7, vec![9; 100]), + ], + &filter, + ); + match &block.accounts[&live_key] { + AccountEntry::Live(account) => { + assert_eq!(account.lamports, 5); + assert_eq!(account.owner, included); + assert_eq!(*account.data, vec![1, 2, 3]); + } + other => panic!("expected live, got {other:?}"), + } + assert_eq!(block.accounts[&closed_key], AccountEntry::Closed); + assert_eq!(block.accounts[&excluded_key], AccountEntry::Closed); + assert!(block.heap_bytes > 3); + assert_eq!(block.block_time, Some(1_700_000_010)); + } + + #[test] + fn skips_malformed_pubkeys() { + let owner = Pubkey::new_unique(); + let mut bad = account_info(Pubkey::new_unique(), owner, 1, vec![]); + bad.pubkey.truncate(31); + let mut bad_owner = account_info(Pubkey::new_unique(), owner, 1, vec![]); + bad_owner.owner.push(0); + let good = Pubkey::new_unique(); + let block = build( + vec![bad, bad_owner, account_info(good, owner, 1, vec![])], + &AccountSelectorConfig::default(), + ); + assert_eq!(block.accounts.len(), 1); + assert!(block.accounts.contains_key(&good)); + } +} diff --git a/crates/core/src/modules/processed/mod.rs b/crates/core/src/modules/processed/mod.rs new file mode 100644 index 0000000..ba5b695 --- /dev/null +++ b/crates/core/src/modules/processed/mod.rs @@ -0,0 +1,264 @@ +// SPDX-License-Identifier: AGPL-3.0-only +/* + * Copyright 2025-2026 Triton One Limited. All rights reserved. + */ + +//! Processed commitment for getAccountInfo, getMultipleAccounts, getBalance, +//! getTokenAccountBalance, getTokenSupply and getSlot. +//! +//! Postgres holds confirmed data only. This module subscribes to Yellowstone +//! blocks with accounts at processed commitment and keeps the blocks around the +//! Postgres confirmed slot in memory. Each request takes one [`ProcessedBlocks`]: +//! the highest stored block and its parents down through the confirmed slot, +//! linked by parent slot and parent blockhash. A key written in the blocks is +//! answered from memory, and an unknown key reads Postgres at `slot <= anchor_slot`. +//! When no chain can be proven, [`ProcessedAccounts::blocks`] returns `None` and +//! the request takes the confirmed path. The module has no Postgres dependency. +//! The confirmed slot and its blockhash arrive as an [`Anchor`] on a watch +//! channel from the API slot syncronizer. +//! +//! # Layout +//! +//! - `mod.rs`: the public surface and the constants. +//! - `ingest.rs`: builds a `SlotBlock` and classifies each account as live or +//! closed. A write by an owner outside the API program filter is a close. +//! - `store.rs`: the block store. One block per slot, slot statuses, the +//! anchor, conflicts, retention, the slot cap and the latest chained blocks. +//! - `read.rs`: [`ProcessedBlocks::get_account`], the one read function the API calls, +//! and the `processed_read` span with the store size. +//! - `subscribe.rs`: the feed thread. The shared gRPC client in `crate::grpc` +//! drives the single writer, which owns the store without a lock. +//! +//! # Runtime model +//! +//! [`ProcessedAccounts::spawn`] starts the `processed-feed` thread. After every +//! block, slot status or anchor change the writer prunes, selects the latest +//! chained blocks and publishes them behind an `RwLock`. A request clones one +//! `Arc` and holds no lock while it reads. A block leaves memory when the last +//! [`ProcessedBlocks`] that pins it drops, on the feed thread after the publish +//! lock or in a request. +//! +//! The feed subscribes without `from_slot`, keeps retrying on every failure +//! and never changes node health. While it is down, requests take the +//! confirmed path. +//! +//! # Latest chained blocks +//! +//! The head is the highest stored slot. Its walk follows parent links, each +//! checked by blockhash, down to the confirmed slot, whose blockhash comes from +//! `recent_blockhashes`. The walk fails, and no blocks are served, when a link is +//! missing, a blockhash differs, the head has no `block_time`, or a slot above +//! the confirmed slot is poisoned: dead, restarted by a second +//! `SLOT_CREATED_BANK`, without a `SLOT_CREATED_BANK` in this session, or at or +//! below the highest slot seen before the last reconnect. A second blockhash +//! for a stored slot, or a parent blockhash that does not match the stored +//! parent, deletes the conflicting block and its descendants and serves nothing +//! until the confirmed slot passes that slot. +//! +//! # Retention +//! +//! Only the Postgres confirmed slot prunes. Blocks below it stay for +//! `RETAINED_SLOTS_BELOW_CONFIRMED` slots and serve hot keys from memory. The +//! Postgres bound stays at the confirmed slot, so a retained write is the +//! newest one whenever the chain is linked. Above the confirmed slot at most +//! `MAX_SLOTS_ABOVE_CONFIRMED` slots are kept. Past that the lowest goes, which +//! breaks the walk until Postgres catches up. +//! +//! # Enable rules +//! +//! The API `[processed-accounts]` section with `enabled = true` turns it on. +//! Otherwise [`ProcessedAccounts::default()`] is a no-op handle: `spawn` does +//! nothing and `blocks` returns `None`. +//! +//! # Node requirements +//! +//! - A Yellowstone endpoint that allows processed commitment and +//! `interslot_updates`, plus its x-token. +//! - `[slot-syncronizer]` enabled, which publishes the [`Anchor`]. +//! - No owner map, no program filter change, no indexer change, no +//! `replay_stored_slots`. Each API instance carries a full block feed. +//! +//! # Upstream invariants +//! +//! Correctness leans on yellowstone-grpc plugin and Agave behaviour. Recheck +//! each one on every plugin major version bump. +//! +//! - At most one entry per pubkey per block, the one with the highest +//! `write_version`. The plugin's `ProcessingSlot::seal()` enforces it. Ingest +//! inserts each pubkey without dedup, so if it breaks, a repeated pubkey keeps +//! an arbitrary version from that block. +//! - One block per slot per stream. The plugin's block assembly enforces it. A +//! second version is a conflict. +//! - Account writes are sent before the block seals. The plugin's message +//! ordering enforces it. If it breaks, a block misses writes and serves stale +//! data. +//! - `SLOT_CREATED_BANK` is sent for every bank creation. The plugin's +//! `update_slot_status` enforces it. If it breaks, a restarted slot goes +//! undetected and a block that mixes two attempts can be served. +//! - `parent_blockhash` equals the parent block's `blockhash`. Agave bank +//! construction enforces it. If it breaks, nothing links and no blocks are served. + +mod ingest; +mod read; +mod store; +mod subscribe; + +pub use read::{ProcessedAccount, ProcessedBlocks}; + +use std::sync::{Arc, RwLock}; +use std::time::Duration; + +use solana_pubkey::Pubkey; +use yellowstone_grpc_client::GeyserGrpcClient; + +use crate::config::{AccountSelectorConfig, ProcessedAccountsConfig}; + +/// Bound on connecting and on each request. +const CONNECT_TIMEOUT: Duration = Duration::from_secs(10); +/// Reconnects when the stream is silent this long. Above the 10 s server ping. +const STALL_TIMEOUT: Duration = Duration::from_secs(30); +const RECONNECT_BACKOFF: Duration = Duration::from_secs(5); +const MAX_DECODING_MESSAGE_SIZE: usize = 256 * 1024 * 1024; +/// Stored slots above the confirmed slot. +pub(crate) const MAX_SLOTS_ABOVE_CONFIRMED: usize = 32; +/// Slots kept below the confirmed slot. +pub(crate) const RETAINED_SLOTS_BELOW_CONFIRMED: u64 = 4; + +/// The Postgres confirmed slot and its blockhash, published by the API slot syncronizer. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct Anchor { + pub confirmed_slot: u64, + /// Base58, as in `recent_blockhashes.blockhash` and `SubscribeUpdateBlock.blockhash`. + pub confirmed_blockhash: String, +} + +/// One account version held in memory. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct LiveAccount { + pub lamports: u64, + pub owner: Pubkey, + pub executable: bool, + pub rent_epoch: u64, + pub data: Arc>, +} + +/// An account write in one block. +#[derive(Debug, PartialEq, Eq)] +pub(crate) enum AccountEntry { + Live(LiveAccount), + /// Zero lamports, or an owner outside the program filter. Shadows every older version. + Closed, +} + +struct Shared { + config: ProcessedAccountsConfig, + program_filter: Arc, + latest: RwLock>>, +} + +impl Shared { + /// Swaps in the latest blocks. The old ones drop after the write lock is released. + fn set_latest(&self, next: Option>) { + let _previous = { + let mut guard = self.latest.write().unwrap_or_else(|e| e.into_inner()); + std::mem::replace(&mut *guard, next) + }; + } +} + +/// Cheap-clone handle to the latest processed blocks. `None` is the disabled handle. +#[derive(Clone, Default)] +pub struct ProcessedAccounts(Option>); + +impl ProcessedAccounts { + /// Returns the disabled handle when the section is absent or disabled. + /// Otherwise checks the endpoint and x-token. Does not connect. + pub fn from_config( + config: Option<&ProcessedAccountsConfig>, + program_filter: Arc, + ) -> anyhow::Result { + let Some(config) = config.filter(|c| c.enabled) else { + return Ok(Self::default()); + }; + GeyserGrpcClient::build_from_shared(config.endpoint.clone())? + .x_token(config.x_token.clone())?; + Ok(Self(Some(Arc::new(Shared { + config: config.clone(), + program_filter, + latest: RwLock::new(None), + })))) + } + + pub fn is_enabled(&self) -> bool { + self.0.is_some() + } + + /// Starts the `processed-feed` thread. No-op when disabled. + pub fn spawn(&self, anchor_rx: tokio::sync::watch::Receiver>) { + if let Some(shared) = &self.0 { + subscribe::spawn_feed(shared.clone(), anchor_rx); + } + } + + /// The blocks for one request, or `None` when no chain can be proven. + pub fn blocks(&self) -> Option> { + self.0 + .as_ref()? + .latest + .read() + .unwrap_or_else(|e| e.into_inner()) + .clone() + } +} + +#[cfg(test)] +mod tests { + use super::store::tests::{TestChain, anchor_at}; + use super::*; + + pub(super) fn config(endpoint: &str) -> ProcessedAccountsConfig { + ProcessedAccountsConfig { + enabled: true, + endpoint: endpoint.to_string(), + x_token: None, + } + } + + pub(super) fn handle(config: &ProcessedAccountsConfig) -> anyhow::Result { + ProcessedAccounts::from_config(Some(config), Arc::new(AccountSelectorConfig::default())) + } + + #[test] + fn from_config_gates_on_enabled_and_checks_the_endpoint() { + let disabled = ProcessedAccounts::default(); + assert!(!disabled.is_enabled()); + let (_tx, rx) = tokio::sync::watch::channel(None); + disabled.spawn(rx); + assert!(disabled.blocks().is_none()); + + let mut off = config(""); + off.enabled = false; + assert!(!handle(&off).unwrap().is_enabled()); + let none = ProcessedAccounts::from_config(None, Arc::new(AccountSelectorConfig::default())); + assert!(!none.unwrap().is_enabled()); + + assert!(handle(&config("not a uri")).is_err()); + + let enabled = handle(&config("http://grpc:10000")).unwrap(); + assert!(enabled.is_enabled()); + assert!(enabled.blocks().is_none()); + } + + #[test] + fn blocks_returns_the_latest_blocks() { + let handle = handle(&config("http://grpc:10000")).unwrap(); + let shared = handle.0.as_ref().unwrap(); + let mut chain = TestChain::new(); + chain.linear(101, 101); + chain.store.set_anchor(anchor_at(100)); + shared.set_latest(chain.event().map(Arc::new)); + assert_eq!(handle.blocks().unwrap().slot, 101); + shared.set_latest(None); + assert!(handle.blocks().is_none()); + } +} diff --git a/crates/core/src/modules/processed/read.rs b/crates/core/src/modules/processed/read.rs new file mode 100644 index 0000000..6beacde --- /dev/null +++ b/crates/core/src/modules/processed/read.rs @@ -0,0 +1,135 @@ +// SPDX-License-Identifier: AGPL-3.0-only +/* + * Copyright 2025-2026 Triton One Limited. All rights reserved. + */ + +//! The processed blocks a request reads and their one read function. + +use std::sync::Arc; + +use solana_pubkey::Pubkey; + +use super::ingest::SlotBlock; +use super::{AccountEntry, LiveAccount}; + +/// The newest state of a key along the processed blocks. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum ProcessedAccount<'a> { + Live(&'a LiveAccount), + /// Closed, or written by an owner outside the program filter. + Closed, + /// Not written in the blocks. Read Postgres at `slot <= anchor_slot`. + Unknown, +} + +/// The immutable chain of blocks from the head down through the confirmed slot +/// and the retained slots below it. Every read in one request uses the same blocks. +pub struct ProcessedBlocks { + /// Head slot, the response `context.slot`. + pub slot: u64, + /// Head block time in unix seconds. + pub block_time: i64, + /// Postgres bound for unknown keys: `slot <= anchor_slot`. + pub anchor_slot: u64, + /// Blocks in the store when these blocks were published. + pub stored_blocks: usize, + /// Estimated heap bytes of those stored blocks, without per-account headers. + pub stored_bytes: usize, + /// Newest first. + pub(super) blocks: Vec>, +} + +impl ProcessedBlocks { + /// The newest write of `pubkey` along the blocks, or [`ProcessedAccount::Unknown`]. + pub fn get_account(&self, pubkey: &Pubkey) -> ProcessedAccount<'_> { + for block in &self.blocks { + if let Some(entry) = block.accounts.get(pubkey) { + return match entry { + AccountEntry::Live(account) => ProcessedAccount::Live(account), + AccountEntry::Closed => ProcessedAccount::Closed, + }; + } + } + ProcessedAccount::Unknown + } + + /// The `processed_read` span for the in-memory lookup of `method`, with the store size. + pub fn read_span(&self, method: &str) -> tracing::Span { + tracing::info_span!( + "processed_read", + method, + stored_blocks = self.stored_blocks, + stored_bytes = self.stored_bytes + ) + } +} + +#[cfg(test)] +mod tests { + use super::super::store::tests::{TestChain, anchor_at}; + use super::*; + use crate::config::{AccountSelectorConfig, PubkeyDef}; + + fn lamports(account: ProcessedAccount<'_>) -> Option { + match account { + ProcessedAccount::Live(LiveAccount { lamports, .. }) => Some(*lamports), + _ => None, + } + } + + #[test] + fn newest_version_wins_across_blocks() { + let key = Pubkey::new_unique(); + let other = Pubkey::new_unique(); + let mut chain = TestChain::new(); + chain.block_with(101, 100, vec![(key, 1), (other, 7)]); + chain.block_with(102, 101, vec![(key, 2)]); + chain.block_with(103, 102, vec![]); + chain.store.set_anchor(anchor_at(100)); + let blocks = chain.event().unwrap(); + assert_eq!(blocks.slot, 103); + assert_eq!(lamports(blocks.get_account(&key)), Some(2)); + assert_eq!(lamports(blocks.get_account(&other)), Some(7)); + assert_eq!( + blocks.get_account(&Pubkey::new_unique()), + ProcessedAccount::Unknown + ); + } + + #[test] + fn closed_and_excluded_shadow_older_versions() { + let closed = Pubkey::new_unique(); + let moved = Pubkey::new_unique(); + let selected = Pubkey::new_unique(); + let other = Pubkey::new_unique(); + let mut chain = TestChain::new(); + chain.filter = AccountSelectorConfig { + include: vec![PubkeyDef(selected)], + exclude: vec![], + }; + chain.block_owned(101, 100, vec![(closed, selected, 5), (moved, selected, 5)]); + chain.block_owned(102, 101, vec![(closed, selected, 0), (moved, other, 6)]); + chain.store.set_anchor(anchor_at(100)); + let blocks = chain.event().unwrap(); + // Closed, not Unknown, so the caller never falls back to a Postgres row. + assert_eq!(blocks.get_account(&closed), ProcessedAccount::Closed); + assert_eq!(blocks.get_account(&moved), ProcessedAccount::Closed); + + chain.block_owned(103, 102, vec![(moved, selected, 7)]); + let blocks = chain.event().unwrap(); + assert_eq!(blocks.slot, 103); + assert_eq!(lamports(blocks.get_account(&moved)), Some(7)); + } + + #[test] + fn retained_blocks_below_the_confirmed_slot_answer_from_memory() { + let key = Pubkey::new_unique(); + let mut chain = TestChain::new(); + chain.block_with(98, 97, vec![(key, 3)]); + chain.linear(99, 103); + chain.store.set_anchor(anchor_at(100)); + let blocks = chain.event().unwrap(); + assert_eq!(blocks.anchor_slot, 100); + assert_eq!(lamports(blocks.get_account(&key)), Some(3)); + } +} diff --git a/crates/core/src/modules/processed/store.rs b/crates/core/src/modules/processed/store.rs new file mode 100644 index 0000000..d123333 --- /dev/null +++ b/crates/core/src/modules/processed/store.rs @@ -0,0 +1,553 @@ +// SPDX-License-Identifier: AGPL-3.0-only +/* + * Copyright 2025-2026 Triton One Limited. All rights reserved. + */ + +//! The block store state machine. Pure and synchronous: it takes events, keeps +//! one block per slot, prunes, and selects the latest chained blocks. + +use std::cmp::Ordering; +use std::collections::{BTreeMap, BTreeSet, HashSet}; +use std::sync::Arc; +use std::time::Instant; + +use super::ingest::SlotBlock; +use super::read::ProcessedBlocks; +use super::{Anchor, MAX_SLOTS_ABOVE_CONFIRMED, RETAINED_SLOTS_BELOW_CONFIRMED}; +use crate::metrics; + +pub(crate) struct BlockStore { + pub(super) blocks: BTreeMap>, + pub(super) dead: BTreeSet, + /// Slots with a `SLOT_CREATED_BANK` in this session. + pub(super) created: HashSet, + /// No blocks are served until the confirmed slot passes this slot. + pub(super) conflict_until: Option, + pub(super) session_floor: u64, + pub(super) max_slot_seen: u64, + pub(super) anchor: Option, +} + +impl BlockStore { + pub(crate) fn new() -> Self { + Self { + blocks: BTreeMap::new(), + dead: BTreeSet::new(), + created: HashSet::new(), + conflict_until: None, + session_floor: 0, + max_slot_seen: 0, + anchor: None, + } + } + + pub(crate) fn anchor_slot(&self) -> Option { + self.anchor.as_ref().map(|a| a.confirmed_slot) + } + + fn find(&self, slot: u64, hash: &str) -> Option<&Arc> { + self.blocks + .get(&slot) + .filter(|block| block.blockhash == hash) + } + + fn parent_of(&self, block: &SlotBlock) -> Option<&Arc> { + if block.parent_slot >= block.slot { + return None; + } + self.find(block.parent_slot, &block.parent_blockhash) + } + + /// Starts a subscription session. Slots seen before it are ineligible until + /// the confirmed slot passes them, and births are forgotten. + pub(crate) fn new_session(&mut self) { + self.session_floor = self.max_slot_seen; + self.created.clear(); + } + + pub(crate) fn on_created_bank(&mut self, slot: u64, parent: Option) { + self.max_slot_seen = self.max_slot_seen.max(slot); + if !self.created.insert(slot) && self.dead.insert(slot) { + tracing::info!(slot, ?parent, "processed slot restarted, marked dead"); + } + } + + pub(crate) fn on_dead(&mut self, slot: u64) { + self.max_slot_seen = self.max_slot_seen.max(slot); + self.dead.insert(slot); + } + + /// Stores a block. A replay of a stored block is ignored. A second + /// blockhash for a stored slot, or a parent blockhash that does not match + /// the stored parent, is a conflict above the confirmed slot. At or below + /// it the stored block is the confirmed one and the newcomer is dropped. + pub(crate) fn on_block(&mut self, block: SlotBlock) { + self.max_slot_seen = self.max_slot_seen.max(block.slot); + let confirmed = |slot: u64| self.anchor_slot().is_some_and(|a| slot <= a); + if let Some(stored) = self.blocks.get(&block.slot) { + if stored.blockhash != block.blockhash && !confirmed(block.slot) { + self.conflict(block.slot, "second blockhash for slot"); + } + return; + } + if let Some(parent) = self.blocks.get(&block.parent_slot) + && parent.blockhash != block.parent_blockhash + { + if !confirmed(block.parent_slot) { + self.conflict(block.parent_slot, "parent blockhash mismatch"); + } + return; + } + self.blocks.insert(block.slot, Arc::new(block)); + } + + /// Deletes the block at `slot` and its descendants, and serves nothing + /// until the confirmed slot passes `slot`. + fn conflict(&mut self, slot: u64, cause: &str) { + let mut doomed = HashSet::from([slot]); + for (&child_slot, child) in self.blocks.range(slot + 1..) { + if doomed.contains(&child.parent_slot) && self.parent_of(child).is_some() { + doomed.insert(child_slot); + } + } + for doomed_slot in &doomed { + self.blocks.remove(doomed_slot); + } + self.conflict_until = Some(self.conflict_until.map_or(slot, |c| c.max(slot))); + tracing::warn!(slot, cause, deleted = doomed.len(), "processed conflict"); + } + + /// Applies the Postgres confirmed slot and blockhash. + pub(crate) fn set_anchor(&mut self, anchor: Anchor) { + if let Some(previous) = self.anchor_slot() + && let Some(confirmed) = self.find(anchor.confirmed_slot, &anchor.confirmed_blockhash) + { + let now = Instant::now(); + let newly_confirmed = std::iter::successors(Some(confirmed), |b| self.parent_of(b)) + .take_while(|block| block.slot > previous); + for block in newly_confirmed { + let latency = now.saturating_duration_since(block.received_at); + metrics::PROCESSED_CONFIRM_LATENCY_MS.observe(latency.as_secs_f64() * 1_000.0); + } + } + if self + .conflict_until + .is_some_and(|until| anchor.confirmed_slot > until) + { + self.conflict_until = None; + } + self.anchor = Some(anchor); + } + + /// Drops slots below the retained window under the confirmed slot, poison + /// entries at or below it, and the lowest slots over the cap above it. + /// Before the first anchor the cap below the highest slot seen is the floor. + pub(crate) fn prune(&mut self) { + if let Some(a) = self.anchor_slot() { + let floor = a.saturating_sub(RETAINED_SLOTS_BELOW_CONFIRMED); + while self + .blocks + .first_key_value() + .is_some_and(|(s, _)| *s < floor) + { + self.blocks.pop_first(); + } + } + let poison_floor = self.anchor_slot().unwrap_or_else(|| { + self.max_slot_seen + .saturating_sub(MAX_SLOTS_ABOVE_CONFIRMED as u64) + }); + self.dead.retain(|slot| *slot > poison_floor); + self.created.retain(|slot| *slot > poison_floor); + let above = self.anchor_slot().map_or(0, |a| a + 1); + while self.blocks.range(above..).count() > MAX_SLOTS_ABOVE_CONFIRMED { + let lowest = *self.blocks.range(above..).next().map(|(s, _)| s).unwrap(); + self.blocks.remove(&lowest); + } + } + + /// A slot above the confirmed slot that must not be served: dead, + /// restarted, without a birth in this session, or seen before this session. + fn poisoned(&self, slot: u64) -> bool { + self.dead.contains(&slot) || !self.created.contains(&slot) || slot <= self.session_floor + } + + /// The blocks from the highest stored slot, when its walk reaches the + /// confirmed slot and blockhash with no poisoned slot on the way. + pub(crate) fn latest_chained_blocks(&self) -> Option { + let anchor = self.anchor.as_ref()?; + let a = anchor.confirmed_slot; + if self.conflict_until.is_some_and(|until| a <= until) { + return None; + } + let head = self.blocks.values().next_back()?; + let block_time = head.block_time?; + + let mut blocks = Vec::new(); + let mut block = head; + while block.slot > a { + if self.poisoned(block.slot) { + return None; + } + blocks.push(block.clone()); + if block.parent_slot == a { + if block.parent_blockhash != anchor.confirmed_blockhash { + return None; + } + break; + } + block = self.parent_of(block)?; + } + // The confirmed block and its stored parents extend the chain below the anchor. + let at_anchor = match block.slot.cmp(&a) { + Ordering::Greater => self.parent_of(block), + Ordering::Equal => Some(self.find(a, &anchor.confirmed_blockhash)?), + Ordering::Less => return None, + }; + blocks.extend(std::iter::successors(at_anchor, |b| self.parent_of(b)).cloned()); + + Some(ProcessedBlocks { + slot: head.slot, + block_time, + anchor_slot: a, + stored_blocks: self.blocks.len(), + stored_bytes: self.blocks.values().map(|block| block.heap_bytes).sum(), + blocks, + }) + } +} + +#[cfg(test)] +pub(crate) mod tests { + use super::super::ingest::tests::{account_info, update}; + use super::*; + use crate::config::AccountSelectorConfig; + use solana_pubkey::Pubkey; + + pub(crate) fn hash(slot: u64) -> String { + format!("h{slot}") + } + + pub(crate) fn anchor_at(slot: u64) -> Anchor { + anchor_with(slot, &hash(slot)) + } + + pub(crate) fn anchor_with(slot: u64, blockhash: &str) -> Anchor { + Anchor { + confirmed_slot: slot, + confirmed_blockhash: blockhash.to_string(), + } + } + + /// Drives a store with blocks built through ingest. Hashes are named `h`. + pub(crate) struct TestChain { + pub store: BlockStore, + pub filter: AccountSelectorConfig, + owner: Pubkey, + } + + impl TestChain { + pub fn new() -> Self { + let mut store = BlockStore::new(); + store.new_session(); + Self { + store, + filter: AccountSelectorConfig::default(), + owner: Pubkey::new_unique(), + } + } + + fn birth(&mut self, slot: u64, parent: u64) { + if !self.store.created.contains(&slot) { + self.store.on_created_bank(slot, Some(parent)); + } + } + + /// Birth plus block, every account owned by the chain's owner. + pub fn block_with(&mut self, slot: u64, parent: u64, accounts: Vec<(Pubkey, u64)>) { + let owner = self.owner; + let accounts = accounts.into_iter().map(|(k, l)| (k, owner, l)).collect(); + self.block_owned(slot, parent, accounts); + } + + pub fn block_owned( + &mut self, + slot: u64, + parent: u64, + accounts: Vec<(Pubkey, Pubkey, u64)>, + ) { + self.birth(slot, parent); + let infos = accounts + .into_iter() + .map(|(key, owner, lamports)| account_info(key, owner, lamports, vec![0; 8])) + .collect(); + self.raw(update(slot, &hash(slot), parent, &hash(parent), infos)); + } + + pub fn block_without_time(&mut self, slot: u64, parent: u64) { + self.birth(slot, parent); + let mut block = update(slot, &hash(slot), parent, &hash(parent), vec![]); + block.block_time = None; + self.raw(block); + } + + pub fn raw(&mut self, block: yellowstone_grpc_proto::prelude::SubscribeUpdateBlock) { + let built = SlotBlock::from_update(block, &self.filter, Instant::now()); + self.store.on_block(built); + } + + /// Birth plus an empty block for every slot in `from..=to`, each on the previous slot. + pub fn linear(&mut self, from: u64, to: u64) { + for slot in from..=to { + self.block_with(slot, slot - 1, vec![]); + } + } + + /// Birth plus an empty block with explicit hashes. + pub fn fork(&mut self, slot: u64, hash: &str, parent: u64, parent_hash: &str) { + self.birth(slot, parent); + self.raw(update(slot, hash, parent, parent_hash, vec![])); + } + + pub fn event(&mut self) -> Option { + self.store.prune(); + self.store.latest_chained_blocks() + } + + pub fn head(&mut self) -> Option { + self.event().map(|blocks| blocks.slot) + } + + pub fn slots(&self) -> Vec { + self.store.blocks.keys().copied().collect() + } + } + + #[test] + fn linear_chain_serves_the_highest_slot_down_to_the_confirmed_block() { + let mut chain = TestChain::new(); + chain.linear(100, 105); + chain.store.set_anchor(anchor_at(100)); + let latest = chain.event().unwrap(); + assert_eq!(latest.slot, 105); + assert_eq!(latest.anchor_slot, 100); + assert_eq!(latest.block_time, 1_700_000_105); + // Five blocks above the anchor plus the confirmed block itself. + assert_eq!(latest.blocks.len(), 6); + assert_eq!(latest.blocks.last().unwrap().slot, 100); + } + + #[test] + fn chain_links_to_the_anchor_by_parent_hash_when_the_confirmed_block_is_absent() { + let mut chain = TestChain::new(); + chain.linear(101, 103); + chain.store.set_anchor(anchor_at(100)); + let latest = chain.event().unwrap(); + assert_eq!(latest.slot, 103); + assert_eq!(latest.blocks.len(), 3); + + // A different confirmed blockhash breaks the walk. + chain.store.set_anchor(anchor_with(100, "other100")); + assert_eq!(chain.head(), None); + } + + #[test] + fn broken_walk_serves_nothing_until_the_confirmed_slot_passes_the_gap() { + let mut chain = TestChain::new(); + chain.linear(101, 102); + chain.linear(105, 106); + chain.store.set_anchor(anchor_at(100)); + assert_eq!(chain.head(), None); + chain.store.set_anchor(anchor_at(102)); + assert_eq!(chain.head(), None); + chain.store.set_anchor(anchor_at(104)); + assert_eq!(chain.head(), Some(106)); + } + + #[test] + fn highest_slot_on_a_losing_fork_serves_nothing() { + let mut chain = TestChain::new(); + chain.linear(101, 102); + chain.fork(103, "x103", 100, "other100"); + chain.store.set_anchor(anchor_at(100)); + assert_eq!(chain.head(), None); + // Once a higher block on the confirmed chain arrives it is the head again. + chain.block_with(104, 102, vec![]); + assert_eq!(chain.head(), Some(104)); + } + + #[test] + fn head_at_or_below_the_confirmed_slot() { + let mut chain = TestChain::new(); + chain.linear(100, 101); + chain.store.set_anchor(anchor_at(101)); + assert_eq!(chain.head(), Some(101)); + chain.store.set_anchor(anchor_at(102)); + assert_eq!(chain.head(), None); + chain.store.set_anchor(anchor_with(101, "other101")); + assert_eq!(chain.head(), None); + } + + #[test] + fn no_blocks_before_the_first_anchor() { + let mut chain = TestChain::new(); + chain.linear(101, 103); + assert_eq!(chain.head(), None); + chain.store.set_anchor(anchor_at(100)); + assert_eq!(chain.head(), Some(103)); + } + + #[test] + fn second_blockhash_deletes_the_slot_and_descendants_until_confirmed_passes_it() { + let mut chain = TestChain::new(); + chain.linear(101, 104); + chain.fork(105, "y105", 103, "h103"); + chain.store.set_anchor(anchor_at(100)); + assert_eq!(chain.head(), Some(105)); + + chain.fork(102, "other102", 101, "h101"); + assert_eq!(chain.store.conflict_until, Some(102)); + // 102 and its descendants 103, 104 and 105 are gone. 101 stays. + assert_eq!(chain.slots(), vec![101]); + assert_eq!(chain.head(), None); + + chain.store.set_anchor(anchor_at(102)); + assert_eq!(chain.store.conflict_until, Some(102)); + assert_eq!(chain.head(), None); + chain.linear(103, 104); + chain.store.set_anchor(anchor_at(103)); + assert_eq!(chain.store.conflict_until, None); + assert_eq!(chain.head(), Some(104)); + } + + #[test] + fn parent_mismatch_deletes_the_stored_parent_and_its_descendants() { + let mut chain = TestChain::new(); + chain.linear(101, 104); + chain.store.set_anchor(anchor_at(100)); + assert_eq!(chain.head(), Some(104)); + + chain.fork(105, "h105", 103, "r103"); + assert_eq!(chain.store.conflict_until, Some(103)); + assert_eq!(chain.slots(), vec![101, 102]); + assert_eq!(chain.head(), None); + chain.store.set_anchor(anchor_at(103)); + assert_eq!(chain.head(), None); + chain.store.set_anchor(anchor_at(104)); + assert_eq!(chain.store.conflict_until, None); + } + + #[test] + fn newcomer_conflicting_at_or_below_the_confirmed_slot_is_dropped() { + let mut chain = TestChain::new(); + chain.linear(99, 103); + chain.store.set_anchor(anchor_at(100)); + assert_eq!(chain.head(), Some(103)); + + chain.fork(100, "other100", 99, "h99"); + chain.fork(99, "other99", 98, "h98"); + chain.fork(104, "x104", 100, "other100"); + assert_eq!(chain.store.conflict_until, None); + assert_eq!(chain.slots(), vec![99, 100, 101, 102, 103]); + assert_eq!(chain.head(), Some(103)); + } + + #[test] + fn replay_of_a_stored_block_is_not_a_conflict() { + let mut chain = TestChain::new(); + chain.linear(101, 102); + chain.store.set_anchor(anchor_at(100)); + chain.linear(102, 102); + assert_eq!(chain.store.conflict_until, None); + assert_eq!(chain.head(), Some(102)); + } + + #[test] + fn retention_keeps_a_window_below_the_confirmed_slot() { + let mut chain = TestChain::new(); + chain.linear(90, 105); + chain.store.on_dead(93); + chain.store.set_anchor(anchor_at(100)); + chain.store.prune(); + let floor = 100 - RETAINED_SLOTS_BELOW_CONFIRMED; + assert_eq!(chain.slots(), (floor..=105).collect::>()); + assert!(chain.store.dead.is_empty()); + assert!(chain.store.created.iter().all(|s| *s > 100)); + let latest = chain.event().unwrap(); + assert_eq!(latest.blocks.last().unwrap().slot, floor); + } + + #[test] + fn slot_cap_evicts_the_lowest_slot_above_the_confirmed_slot() { + let cap = MAX_SLOTS_ABOVE_CONFIRMED as u64; + let mut chain = TestChain::new(); + chain.linear(101, 100 + cap + 2); + chain.store.set_anchor(anchor_at(100)); + assert_eq!(chain.head(), None); + assert_eq!(chain.slots().first(), Some(&103)); + chain.store.set_anchor(anchor_at(102)); + assert_eq!(chain.head(), Some(100 + cap + 2)); + + // Before the first anchor the cap bounds the whole store and the poison sets. + let mut chain = TestChain::new(); + chain.linear(101, 100 + cap + 1); + chain.store.on_dead(101); + chain.store.prune(); + assert_eq!(chain.slots().len(), MAX_SLOTS_ABOVE_CONFIRMED); + assert!(chain.store.dead.is_empty()); + assert!(chain.store.created.iter().all(|s| *s > 101)); + } + + #[test] + fn head_without_block_time_serves_nothing() { + let mut chain = TestChain::new(); + chain.linear(101, 102); + chain.block_without_time(103, 102); + chain.store.set_anchor(anchor_at(100)); + assert_eq!(chain.head(), None); + chain.linear(104, 104); + assert_eq!(chain.head(), Some(104)); + } + + #[test] + fn dead_slot_on_the_walk_serves_nothing_until_confirmed_passes_it() { + let mut chain = TestChain::new(); + chain.linear(101, 104); + chain.store.set_anchor(anchor_at(100)); + chain.store.on_dead(102); + assert_eq!(chain.head(), None); + chain.store.set_anchor(anchor_at(102)); + assert_eq!(chain.head(), Some(104)); + } + + #[test] + fn repeated_bank_creation_marks_the_slot_dead() { + let mut chain = TestChain::new(); + chain.linear(101, 103); + chain.store.set_anchor(anchor_at(100)); + assert_eq!(chain.head(), Some(103)); + chain.store.on_created_bank(103, Some(102)); + assert!(chain.store.dead.contains(&103)); + assert_eq!(chain.head(), None); + } + + #[test] + fn missing_birth_and_session_floor_are_ineligible() { + let mut chain = TestChain::new(); + chain.linear(101, 101); + chain.raw(update(102, "h102", 101, "h101", vec![])); + chain.store.set_anchor(anchor_at(100)); + assert_eq!(chain.head(), None); + chain.store.on_created_bank(102, Some(101)); + assert_eq!(chain.head(), Some(102)); + + chain.store.new_session(); + assert_eq!(chain.store.session_floor, 102); + chain.block_with(103, 102, vec![]); + chain.store.on_created_bank(101, Some(100)); + chain.store.on_created_bank(102, Some(101)); + assert_eq!(chain.head(), None); + chain.store.set_anchor(anchor_at(102)); + assert_eq!(chain.head(), Some(103)); + } +} diff --git a/crates/core/src/modules/processed/subscribe.rs b/crates/core/src/modules/processed/subscribe.rs new file mode 100644 index 0000000..3d7ae9f --- /dev/null +++ b/crates/core/src/modules/processed/subscribe.rs @@ -0,0 +1,210 @@ +// SPDX-License-Identifier: AGPL-3.0-only +/* + * Copyright 2025-2026 Triton One Limited. All rights reserved. + */ + +//! The feed thread and the single writer. +//! +//! The writer runs the shared gRPC client on its own OS thread with a +//! current-thread runtime. One session subscribes to processed blocks with +//! accounts and to slot statuses with interslot updates. Every block, +//! `SLOT_CREATED_BANK`, `SLOT_DEAD` and anchor change is applied to the store, +//! followed by a prune, a selection of the latest chained blocks and a +//! publish. A stream end, +//! error or stall ends the session, and the client reconnects with no +//! `from_slot`. The client never gives up. + +use std::sync::Arc; +use std::time::{Duration, Instant}; + +use futures::{Stream, StreamExt}; +use tokio::sync::watch; +use yellowstone_grpc_client::GeyserGrpcClient; +use yellowstone_grpc_proto::geyser::{ + CommitmentLevel, SlotStatus, SubscribeRequest, SubscribeUpdate, subscribe_update::UpdateOneof, +}; +use yellowstone_grpc_proto::tonic::Status; + +use super::ingest::SlotBlock; +use super::store::BlockStore; +use super::{ + Anchor, CONNECT_TIMEOUT, MAX_DECODING_MESSAGE_SIZE, RECONNECT_BACKOFF, STALL_TIMEOUT, Shared, +}; +use crate::grpc::{ + GrpcClientOptions, SessionEnd, Subscriber, blocks_with_accounts_request, + subscribe_with_reconnection, +}; + +/// Starts `processed-feed`. Logs and returns when the thread cannot start. +pub(super) fn spawn_feed(shared: Arc, anchor_rx: watch::Receiver>) { + let options = GrpcClientOptions { + endpoint: shared.config.endpoint.clone(), + x_token: shared.config.x_token.clone(), + timeout: CONNECT_TIMEOUT, + max_decoding_message_size: MAX_DECODING_MESSAGE_SIZE, + reconnect_backoff: RECONNECT_BACKOFF, + reconnect_give_up: None, + reconnect_from_slot_retain: Duration::ZERO, + }; + let writer = FeedWriter { + shared, + store: BlockStore::new(), + anchor_rx, + }; + let spawned = std::thread::Builder::new() + .name("processed-feed".to_string()) + .spawn(move || { + let runtime = match tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + { + Ok(runtime) => runtime, + Err(e) => { + tracing::error!("Failed to build processed-feed runtime: {e}"); + return; + } + }; + runtime.block_on(subscribe_with_reconnection(options, writer)); + }); + if let Err(e) = spawned { + tracing::error!("Failed to start processed-feed thread: {e}"); + } +} + +struct FeedWriter { + shared: Arc, + store: BlockStore, + anchor_rx: watch::Receiver>, +} + +impl Subscriber for FeedWriter { + fn request(&self, _replay: bool) -> SubscribeRequest { + blocks_with_accounts_request(CommitmentLevel::Processed, true, None) + } + + fn cancelled(&self) -> bool { + false + } + + fn on_connect_failed(&mut self) {} + + async fn on_connect(&mut self, _client: &mut GeyserGrpcClient) {} + + async fn session( + &mut self, + stream: impl Stream> + Send, + ) -> SessionEnd { + tracing::info!("processed feed subscribed"); + self.store.new_session(); + self.apply_anchor(); + let mut stream = std::pin::pin!(stream); + let stall = tokio::time::sleep(STALL_TIMEOUT); + tokio::pin!(stall); + let mut received_block = false; + loop { + tokio::select! { + message = stream.next() => match message { + Some(Ok(update)) => { + stall.as_mut().reset(tokio::time::Instant::now() + STALL_TIMEOUT); + received_block |= self.apply_update(update); + } + Some(Err(status)) => { + tracing::warn!("processed feed stream error: {status}"); + break if received_block { + SessionEnd::Healthy + } else { + SessionEnd::Failed + }; + } + None => { + tracing::warn!("processed feed stream ended"); + break SessionEnd::Healthy; + } + }, + changed = self.anchor_rx.changed() => { + if changed.is_ok() { + self.apply_anchor(); + } + } + () = &mut stall => { + tracing::warn!("processed feed stalled for {STALL_TIMEOUT:?}"); + break SessionEnd::Healthy; + } + } + } + } +} + +impl FeedWriter { + /// Applies one update. True when it was a block. + fn apply_update(&mut self, update: SubscribeUpdate) -> bool { + match update.update_oneof { + Some(UpdateOneof::Block(block)) => { + let received_at = Instant::now(); + let block = SlotBlock::from_update(block, &self.shared.program_filter, received_at); + self.store.on_block(block); + self.publish_latest(); + true + } + Some(UpdateOneof::Slot(slot)) => { + match SlotStatus::try_from(slot.status) { + Ok(SlotStatus::SlotCreatedBank) => { + self.store.on_created_bank(slot.slot, slot.parent) + } + Ok(SlotStatus::SlotDead) => self.store.on_dead(slot.slot), + _ => return false, + } + self.publish_latest(); + false + } + _ => false, + } + } + + /// Applies the current anchor when there is one. + fn apply_anchor(&mut self) { + let anchor = self.anchor_rx.borrow_and_update().clone(); + if let Some(anchor) = anchor { + self.store.set_anchor(anchor); + self.publish_latest(); + } + } + + /// Prunes, selects and publishes. + fn publish_latest(&mut self) { + self.store.prune(); + self.shared + .set_latest(self.store.latest_chained_blocks().map(Arc::new)); + } +} + +#[cfg(test)] +mod tests { + use super::super::tests::{config, handle}; + use super::*; + + fn writer() -> FeedWriter { + let handle = handle(&config("http://grpc:10000")).unwrap(); + FeedWriter { + shared: handle.0.expect("enabled handle"), + store: BlockStore::new(), + anchor_rx: watch::channel(None).1, + } + } + + #[test] + fn request_subscribes_processed_blocks_and_interslot_slots() { + let request = writer().request(false); + assert_eq!(request.commitment, Some(CommitmentLevel::Processed as i32)); + assert_eq!(request.from_slot, None); + let blocks = &request.blocks["accounts_blocks"]; + assert_eq!(blocks.include_accounts, Some(true)); + assert_eq!(blocks.include_transactions, Some(false)); + assert_eq!(blocks.include_entries, Some(false)); + assert!(blocks.account_include.is_empty()); + let slots = &request.slots["accounts_slots"]; + assert_eq!(slots.interslot_updates, Some(true)); + assert_eq!(slots.filter_by_commitment, Some(false)); + assert!(request.accounts.is_empty() && request.transactions.is_empty()); + } +} diff --git a/crates/index/src/modules/grpc.rs b/crates/index/src/modules/grpc.rs index 96ca29b..8e867d3 100644 --- a/crates/index/src/modules/grpc.rs +++ b/crates/index/src/modules/grpc.rs @@ -3,11 +3,14 @@ * Copyright 2025-2026 Triton One Limited. All rights reserved. */ +use cloudbreak_core::grpc::{ + GrpcClientOptions, SessionEnd, Subscriber, blocks_with_accounts_request, + subscribe_with_reconnection, +}; use cloudbreak_core::{EnvironmentInfo, IndexConfig}; -use futures::StreamExt; +use futures::{Stream, StreamExt}; use sea_orm::DatabaseConnection; use std::{ - collections::HashMap, ops::Add, sync::{ Arc, Mutex, @@ -20,13 +23,10 @@ use tokio::{ task::JoinHandle, time::{Instant, timeout}, }; -use yellowstone_grpc_client::{ClientTlsConfig, GeyserGrpcClient}; +use yellowstone_grpc_client::GeyserGrpcClient; use yellowstone_grpc_proto::{ - geyser::{ - CommitmentLevel, SubscribeRequest, SubscribeRequestFilterBlocks, - SubscribeRequestFilterSlots, SubscribeUpdate, subscribe_update::UpdateOneof, - }, - tonic::codec::CompressionEncoding, + geyser::{CommitmentLevel, SubscribeRequest, SubscribeUpdate, subscribe_update::UpdateOneof}, + tonic::Status, }; use crate::metrics; @@ -57,19 +57,16 @@ async fn store_grpc_version(version_json: &str, db: &DatabaseConnection) { /// Creates a persistent Yellowstone GRPC connection with automatic reconnection. /// Spawns a background task to handle the stream and forwards updates to the buffer channel. /// Automatically reconnects on stream timeouts , stream `None` or errors (only after exceeding -/// the `max_grpc_errors` count). It also resets the is_startup flag when the connection is lost. +/// the `max_grpc_errors` count). /// -/// Reconnection is governed by a single give-up window: `reconnect_failed_since` records when we -/// started failing and is only cleared once a reconnection is proven *healthy* — i.e. its stream -/// actually delivers a block. Connect/subscribe failures, and streams that error before delivering +/// The reconnect loop, the give-up window, the backoff and the `from_slot` replay live in +/// `cloudbreak_core::grpc`. Connect/subscribe failures, and streams that error before delivering /// any block (e.g. the server returns "failed to get replay response" for a `from_slot` it no /// longer has), all keep the window open: the loop backs off by `config.grpc.reconnect_backoff` /// between attempts, drops `from_slot` after `reconnect_from_slot_retain` (resubscribing from the -/// live tip), and panics once the failing window exceeds `config.grpc.reconnect_give_up`. Clearing -/// the window only on a delivered block (rather than on subscribe success) is what stops a -/// subscribe-then-immediately-error stream from hot-looping with no delay. A stream that received -/// blocks and then ended (error, inactivity timeout, or stream `None`) is treated as a healthy run -/// and reconnects immediately. +/// live tip), and panics once the failing window exceeds `config.grpc.reconnect_give_up`. A stream +/// that received blocks and then ended (error, inactivity timeout, or stream `None`) is treated as +/// a healthy run and reconnects immediately. pub fn subscribe_grpc_with_reconnection( config: IndexConfig, buffer_channel_tx: Sender, @@ -78,294 +75,194 @@ pub fn subscribe_grpc_with_reconnection( cancel: Arc, db: DatabaseConnection, ) -> JoinHandle<()> { + let grpc_timeout = Duration::from_secs(config.grpc.timeout); + let options = GrpcClientOptions { + endpoint: config.grpc.endpoint.clone(), + x_token: Some(config.grpc.x_token.clone().unwrap_or_default()), + timeout: grpc_timeout, + max_decoding_message_size: usize::MAX, + reconnect_backoff: config.grpc.reconnect_backoff, + reconnect_give_up: Some(config.grpc.reconnect_give_up), + reconnect_from_slot_retain: config.grpc.reconnect_from_slot_retain, + }; + let subscriber = IndexerSubscriber { + grpc_timeout, + max_grpc_errors: config.grpc.max_grpc_errors, + buffer_channel_tx, + buffer_channel_rx_len, + last_slot_received, + cancel, + db, + }; tokio::spawn(async move { let _guard = metrics::TokioTaskCounterGuard::new("grpc"); - let mut log_first_message = true; - let mut reconnect_failed_since: Option = None; - let mut is_reconnect = false; - - let give_up = config.grpc.reconnect_give_up; - let backoff = config.grpc.reconnect_backoff; - let from_slot_retain = config.grpc.reconnect_from_slot_retain; - - loop { - if cancel.load(Ordering::SeqCst) { - tracing::info!("GRPC subscription cancelled"); - return; - } - - // Centralized reconnection backoff / give-up: `reconnect_failed_since` is `Some` while - // we are failing to (re)connect or subscribe and only cleared once both succeed. - if let Some(started) = reconnect_failed_since { - if started.elapsed() >= give_up { - tracing::error!( - "Failed to (re)connect to Yellowstone GRPC after {:?}", - started.elapsed() - ); - panic!( - "Failed to (re)connect to Yellowstone GRPC after {:?}", - started.elapsed() - ); - } - tokio::time::sleep(backoff).await; - } + subscribe_with_reconnection(options, subscriber).await; + }) +} - let grpc_timeout = Duration::from_secs(config.grpc.timeout); +struct IndexerSubscriber { + grpc_timeout: Duration, + max_grpc_errors: usize, + buffer_channel_tx: Sender, + buffer_channel_rx_len: Arc>, + last_slot_received: Arc>, + cancel: Arc, + db: DatabaseConnection, +} - let mut client = match GeyserGrpcClient::build_from_shared(config.grpc.endpoint.clone()) - .expect("Failed to build GeyserGrpcClient") - .x_token(Some(config.grpc.x_token.clone().unwrap_or_default())) - .expect("Failed to set x-token") - .max_decoding_message_size(usize::MAX) - .accept_compressed(CompressionEncoding::Zstd) - .connect_timeout(grpc_timeout) - .timeout(grpc_timeout) - .tls_config(ClientTlsConfig::new().with_native_roots()) - .expect("Failed to set tls config") - .tcp_keepalive(Some(Duration::from_secs(10))) - .http2_keep_alive_interval(Duration::from_secs(10)) - .keep_alive_timeout(Duration::from_secs(10)) - // .http2_adaptive_window(true) - // .initial_stream_window_size(8 * 1024 * 1024) // 8MB - // .initial_connection_window_size(8 * 1024 * 1024) // 8MB - .connect() - .await - { - Ok(mut c) => { - match c.get_version().await { - Ok(response) => store_grpc_version(&response.version, &db).await, - Err(e) => tracing::error!("Failed to get grpc version: {:?}", e), - } - c - } - Err(e) => { - reconnect_failed_since.get_or_insert_with(Instant::now); - tracing::error!("Failed to connect to Yellowstone GRPC: {:?}", e); - metrics::increment_grpc_errors(); - continue; - } - }; +impl Subscriber for IndexerSubscriber { + /// A replay starts at the slot after the last received one. + fn request(&self, replay: bool) -> SubscribeRequest { + let from_slot = replay.then(|| { + let last = *self + .last_slot_received + .lock() + .expect("Failed to lock last_slot_received"); + (last != 0).then_some(last + 1) + }); + blocks_with_accounts_request(CommitmentLevel::Confirmed, false, from_slot.flatten()) + } - // let account_include = config - // .programs - // .include - // .iter() - // .map(|pubkey| pubkey.0.to_string()) - // .collect(); + fn cancelled(&self) -> bool { + self.cancel.load(Ordering::SeqCst) + } - // tracing::debug!("Account include: {:?}", account_include); + fn on_connect_failed(&mut self) { + metrics::increment_grpc_errors(); + } - // Replay from the last received slot on reconnect, but only while we have been failing - // for less than `from_slot_retain`; after that, drop it since the server may no longer - // have that slot buffered. - let keep_from_slot = - reconnect_failed_since.is_none_or(|started| started.elapsed() < from_slot_retain); - let from_slot = if is_reconnect && keep_from_slot { - let last = *last_slot_received - .lock() - .expect("Failed to lock last_slot_received"); - (last != 0).then_some(last + 1) - } else { - None - }; + async fn on_connect(&mut self, client: &mut GeyserGrpcClient) { + match client.get_version().await { + Ok(response) => store_grpc_version(&response.version, &self.db).await, + Err(e) => tracing::error!("Failed to get grpc version: {:?}", e), + } + } - let blocks_subscribe_request: SubscribeRequest = SubscribeRequest { - accounts: HashMap::new(), - slots: HashMap::from([( - "accounts_slots".to_string(), - SubscribeRequestFilterSlots { - filter_by_commitment: Some(false), - interslot_updates: Some(false), - }, - )]), - transactions: HashMap::new(), - transactions_status: HashMap::new(), - blocks: HashMap::from([( - "accounts_blocks".to_string(), - SubscribeRequestFilterBlocks { - account_include: vec![], - include_transactions: Some(false), - include_accounts: Some(true), - include_entries: Some(false), - cuckoo_account_include: None, - }, - )]), - blocks_meta: HashMap::new(), - entry: HashMap::new(), - commitment: Some(CommitmentLevel::Confirmed as i32), - accounts_data_slice: Vec::new(), - ping: None, - from_slot, - }; + async fn session( + &mut self, + stream: impl Stream> + Send, + ) -> SessionEnd { + let _guard = metrics::TokioTaskCounterGuard::new("grpc"); - let (_sub_tx, stream) = match client - .subscribe_with_request(Some(blocks_subscribe_request)) - .await - { - Ok(subscription) => { - if let Some(slot) = from_slot { - tracing::info!( - "Reconnected to Yellowstone GRPC replaying from slot {}", - slot - ); - } - subscription - } - Err(e) => { - reconnect_failed_since.get_or_insert_with(Instant::now); - tracing::error!( - "Failed to subscribe to Yellowstone GRPC (from_slot {:?}): {:?}", - from_slot, - e - ); - metrics::increment_grpc_errors(); - continue; - } - }; + let mut stream = std::pin::pin!(stream); - let buffer_channel_rx_len_clone = buffer_channel_rx_len.clone(); - let mut grpc_current_errors = 0; + let mut log_first_message = true; + let mut last_block_received_at = Instant::now(); + let mut grpc_current_errors = 0; + + // Health signals for the outer give-up window: a run only clears the + // window if it delivered a block; a stream that *errored* before any + // block is a failed attempt (engages backoff). A no-block inactivity + // timeout / stream `None` is neither (reconnects immediately, as-is). + let mut received_block = false; + let mut stream_errored = false; + + let mut buffer_channel_size = + self.buffer_channel_tx.max_capacity() - self.buffer_channel_tx.capacity(); + + // Add a timeout in case we stop receiving updates for 30 more seconds than the grpc timeout + // If we reach it, we break the loop and try to reconnect + while let Some(update) = timeout( + self.grpc_timeout.add(Duration::from_secs(30)), + stream.next(), + ) + .await + .unwrap_or_else(|elapsed| { + tracing::error!( + "GRPC timeout: {:?} - grpc_errors_count: {}", + elapsed, + grpc_current_errors, + ); + metrics::increment_grpc_timeout_errors(); + + // If the timeout is reached, we return None to break the loop + None + }) { + if self.cancelled() { + tracing::info!("GRPC subscription cancelled mid-stream"); + // Shutting down, not a failed attempt. + return SessionEnd::Healthy; + } - let buffer_channel_tx_clone = buffer_channel_tx.clone(); - let last_slot_received = last_slot_received.clone(); - let cancel_clone = cancel.clone(); + metrics::GRPC_TOTAL_UPDATES_RECEIVED.inc(); - let handle = tokio::spawn(async move { - let _guard = metrics::TokioTaskCounterGuard::new("grpc"); + if Instant::now().duration_since(last_block_received_at) > Duration::from_secs(30) { + tracing::error!("No block received in the last 30 seconds"); + grpc_current_errors += 1; + metrics::increment_grpc_errors(); - let mut stream = std::pin::pin!(stream); + if grpc_current_errors >= self.max_grpc_errors { + break; + } + } - let mut last_block_received_at = Instant::now(); + buffer_channel_size = + self.buffer_channel_tx.max_capacity() - self.buffer_channel_tx.capacity(); - // Health signals for the outer give-up window: a run only clears the - // window if it delivered a block; a stream that *errored* before any - // block is a failed attempt (engages backoff). A no-block inactivity - // timeout / stream `None` is neither (reconnects immediately, as-is). - let mut received_block = false; - let mut stream_errored = false; + metrics::GRPC_BUFFER_CHANNEL_SIZE_SENDER.set(buffer_channel_size as i64); - let mut buffer_channel_size = - buffer_channel_tx_clone.max_capacity() - buffer_channel_tx_clone.capacity(); + match update { + Ok(update) => { + if let Some(UpdateOneof::Block(block)) = &update.update_oneof { + last_block_received_at = Instant::now(); + received_block = true; - // Add a timeout in case we stop receiving updates for 30 more seconds than the grpc timeout - // If we reach it, we break the loop and try to reconnect - while let Some(update) = - timeout(grpc_timeout.add(Duration::from_secs(30)), stream.next()) - .await - .unwrap_or_else(|elapsed| { - tracing::error!( - "GRPC timeout: {:?} - grpc_errors_count: {}", - elapsed, - grpc_current_errors, + if log_first_message { + tracing::info!( + "Starting a new indexer service run - slot: {}", + block.slot ); - metrics::increment_grpc_timeout_errors(); - - // If the timeout is reached, we return None to break the loop - None - }) - { - if cancel_clone.load(Ordering::SeqCst) { - tracing::info!("GRPC subscription cancelled mid-stream"); - // Shutting down, not a failed attempt. - return false; - } - - metrics::GRPC_TOTAL_UPDATES_RECEIVED.inc(); - - if Instant::now().duration_since(last_block_received_at) - > Duration::from_secs(30) - { - tracing::error!("No block received in the last 30 seconds"); - grpc_current_errors += 1; - metrics::increment_grpc_errors(); - - if grpc_current_errors >= config.grpc.max_grpc_errors { - break; + log_first_message = false; } } - buffer_channel_size = - buffer_channel_tx_clone.max_capacity() - buffer_channel_tx_clone.capacity(); - - metrics::GRPC_BUFFER_CHANNEL_SIZE_SENDER.set(buffer_channel_size as i64); - - match update { - Ok(update) => { - if let Some(UpdateOneof::Block(block)) = &update.update_oneof { - last_block_received_at = Instant::now(); - received_block = true; - - if log_first_message { - tracing::info!( - "Starting a new indexer service run - slot: {}", - block.slot - ); - log_first_message = false; - } - } - - buffer_channel_tx_clone - .send(update) - .await - .expect("Failed to send update to buffer channel"); - } - Err(e) => { - stream_errored = true; - tracing::error!( - "GRPC error: {:?} buffer_channel_size: {} (sender: {}) - grpc_errors_count: {}", - e, - *buffer_channel_rx_len_clone - .lock() - .expect("Failed to lock buffer_channel_rx_len"), - buffer_channel_size, - grpc_current_errors, - ); - grpc_current_errors += 1; - metrics::increment_grpc_errors(); - - if grpc_current_errors >= config.grpc.max_grpc_errors { - break; - } - } - } + self.buffer_channel_tx + .send(update) + .await + .expect("Failed to send update to buffer channel"); } + Err(e) => { + stream_errored = true; + tracing::error!( + "GRPC error: {:?} buffer_channel_size: {} (sender: {}) - grpc_errors_count: {}", + e, + *self + .buffer_channel_rx_len + .lock() + .expect("Failed to lock buffer_channel_rx_len"), + buffer_channel_size, + grpc_current_errors, + ); + grpc_current_errors += 1; + metrics::increment_grpc_errors(); - tracing::error!( - "Breaking out of grpc subscription loop at slot: {} - buffer_channel_size: {} (sender: {})", - *last_slot_received - .lock() - .expect("Failed to lock last_slot_received"), - *buffer_channel_rx_len_clone - .lock() - .expect("Failed to lock buffer_channel_rx_len"), - buffer_channel_size, - ); - - // A failed attempt = the stream errored before delivering any block. - // A run that saw a block, or ended via inactivity timeout / stream - // `None` without erroring, is not penalized (reconnects immediately). - stream_errored && !received_block - }); - - match handle.await { - Ok(failed_without_data) => { - if failed_without_data { - // Keep the give-up window open so the next iteration backs off, - // eventually drops `from_slot`, and gives up if it never recovers. - reconnect_failed_since.get_or_insert_with(Instant::now); - } else { - // Healthy run (delivered a block) or a benign timeout/None end: - // clear the window so a transient blip reconnects immediately. - reconnect_failed_since = None; + if grpc_current_errors >= self.max_grpc_errors { + break; } } - Err(e) => { - tracing::error!("GRPC subscription handle panicked: {:?}", e); - reconnect_failed_since.get_or_insert_with(Instant::now); - } } + } - is_reconnect = true; + tracing::error!( + "Breaking out of grpc subscription loop at slot: {} - buffer_channel_size: {} (sender: {})", + *self + .last_slot_received + .lock() + .expect("Failed to lock last_slot_received"), + *self + .buffer_channel_rx_len + .lock() + .expect("Failed to lock buffer_channel_rx_len"), + buffer_channel_size, + ); + + // A failed attempt = the stream errored before delivering any block. + // A run that saw a block, or ended via inactivity timeout / stream + // `None` without erroring, is not penalized (reconnects immediately). + if stream_errored && !received_block { + SessionEnd::Failed + } else { + SessionEnd::Healthy } - }) + } } diff --git a/crates/integration_tests/src/benchmark.rs b/crates/integration_tests/src/benchmark.rs index 0658df1..b6ef4ca 100644 --- a/crates/integration_tests/src/benchmark.rs +++ b/crates/integration_tests/src/benchmark.rs @@ -1119,7 +1119,7 @@ fn size_category_ord(cat: &str) -> u8 { } } -fn percentile(sorted: &[u128], pct: f64) -> u128 { +pub(crate) fn percentile(sorted: &[u128], pct: f64) -> u128 { if sorted.is_empty() { return 0; } diff --git a/crates/integration_tests/src/compare_processed_accounts.rs b/crates/integration_tests/src/compare_processed_accounts.rs new file mode 100644 index 0000000..f32aa5b --- /dev/null +++ b/crates/integration_tests/src/compare_processed_accounts.rs @@ -0,0 +1,605 @@ +// SPDX-License-Identifier: AGPL-3.0-only +/* + * Copyright 2025-2026 Triton One Limited. All rights reserved. + */ + +//! `compare-processed-accounts`: correctness and speed checks for processed commitment. +//! +//! Every call is read-only and each check is bounded by a sample or key count. Keys come from +//! `--pubkeys-file` and from writable keys of non-vote transactions in recent confirmed blocks of +//! the first reference, which is also the canonical chain oracle through `getBlocks(S, S)`. +//! +//! - Cross-source: the same processed getMultipleAccounts goes to cloudbreak and every reference +//! at once. At equal context slots every value must match. +//! - Processed matches confirmed: a processed answer at slot S must equal any confirmed answer, +//! from cloudbreak or the first reference, whose context slot is exactly S. Both are the state of +//! bank S, so no write history is needed. A sample with no such answer is a fork or a miss. +//! - Token methods: getBalance, getTokenAccountBalance and getTokenSupply at processed must match +//! the first reference at equal context slots. An error on one side only is a mismatch. +//! - getSlot: cloudbreak processed getSlot against its confirmed getSlot read just before, and +//! against the highest reference processed slot. Processed can trail confirmed by the slot +//! syncronizer interval, so both are reported and only a cloudbreak error fails. +//! +//! A mismatch on a non-canonical slot is a fork, and one whose status stays unknown fails. A +//! cloudbreak null is an excluded key when a confirmed getAccountInfo on cloudbreak at S or later +//! answers -32010, or null while the reference serves the account. A token method -32010 is +//! excluded too. A check fails when it compares nothing, or when cloudbreak errors where the +//! reference answers. + +use crate::benchmark::RequestType::{GetBalance, GetMultipleAccounts}; +use crate::benchmark::percentile; +use crate::config::RpcEndpoint; +use crate::response_comparison::compare_responses; +use crate::utils::{get_slot, send_rpc_request}; +use anyhow::{Result, anyhow}; +use base64::Engine as _; +use base64::engine::general_purpose::STANDARD as BASE64; +use clap::Parser; +use rand::seq::SliceRandom; +use serde_json::{Value as JsonValue, json}; +use std::collections::{HashMap, HashSet}; +use std::time::{Duration, Instant}; + +const TOKEN_PROGRAMS: [&str; 2] = [ + "TokenkegQfeZyiNwAJbNbGKPFXCWuBvf9Ss623VQ5DA", + "TokenzQdBNbLqP5VEhdkAS6EPFLC1PHnBqCXEpPxuEb", +]; +const VOTE_PROGRAM: &str = "Vote111111111111111111111111111111111111111"; +/// Owner-excluded error of the Postgres read path. +const EXCLUDED_CODE: i64 = -32010; +const MAX_KEYS: usize = 2_000; +const MAX_FAILURES: usize = 40; +/// Tries per token method call to land both sources on one context slot. +const SLOT_TRIES: usize = 5; +const POLL: Duration = Duration::from_millis(100); + +#[derive(Parser, Debug)] +#[command(name = "compare-processed-accounts")] +#[command(about = "\ +Validate processed getMultipleAccounts, getBalance, getTokenAccountBalance, getTokenSupply and getSlot \ +against reference Agave RPCs at equal context slots and against confirmed answers at the same \ +slot, and report latency per source. A mismatch on a slot that is not canonical is a fork.")] +pub struct Args { + /// Cloudbreak RPC endpoint URL, pinned to one API instance + #[arg(long, default_value = "http://10.43.10.2:26722")] + pub rpc: String, + /// Agave reference RPC URL, repeatable. The first one is the canonical chain oracle + #[arg(long = "reference", required = true)] + pub references: Vec, + /// File with one base58 pubkey per line + #[arg(long)] + pub pubkeys_file: Option, + /// Recent confirmed blocks on the first reference to take writable keys from + #[arg(long, default_value_t = 4)] + pub discover_blocks: usize, + /// Cross-source samples + #[arg(long, default_value_t = 100)] + pub samples: usize, + /// Processed matches confirmed samples + #[arg(long, default_value_t = 20)] + pub confirm_samples: usize, + /// Keys per method in the token methods pass + #[arg(long, default_value_t = 20)] + pub smoke_keys: usize, + #[arg(long, default_value_t = 20)] + pub keys_per_sample: usize, + /// Pause between samples in milliseconds + #[arg(long, default_value_t = 200)] + pub interval_ms: u64, + /// HTTP timeout, and the wait for a slot to confirm, in seconds + #[arg(long, default_value_t = 30)] + pub timeout: u64, +} + +struct Ctx { + client: reqwest::Client, + cloudbreak: RpcEndpoint, + references: Vec, + wait: Duration, +} + +/// One differing value, held until exclusion and canonical status are known. +struct Mismatch { + slot: u64, + key: String, + detail: String, + /// Cloudbreak answered null, so an excluded key may explain it. + maybe_excluded: bool, + /// The slot is known to be on the confirmed chain. + settled: bool, +} + +pub async fn run(args: &Args) -> Result<()> { + let endpoint = |name: String, url: &String| RpcEndpoint { + url: url.clone(), + name, + }; + let ctx = Ctx { + client: reqwest::Client::builder() + .timeout(Duration::from_secs(args.timeout)) + .build()?, + cloudbreak: endpoint("cloudbreak".to_string(), &args.rpc), + references: (args.references.iter().enumerate()) + .map(|(i, url)| endpoint(format!("reference-{i}"), url)) + .collect(), + wait: Duration::from_secs(args.timeout), + }; + let keys = load_keys(&ctx, args).await?; + if keys.is_empty() { + return Err(anyhow!("no keys, set --pubkeys-file or --discover-blocks")); + } + println!("{} keys, {} reference(s)", keys.len(), ctx.references.len()); + + let (mut mismatches, mut failures) = (Vec::new(), Vec::new()); + let tokens = cross_source(&ctx, args, &keys, &mut mismatches, &mut failures).await; + processed_matches_confirmed(&ctx, args, &keys, &mut mismatches, &mut failures).await; + token_methods(&ctx, args, &keys, &tokens, &mut mismatches, &mut failures).await; + slots_check(&ctx, args, &mut failures).await; + + let (mut excluded, mut forks, mut unclassified) = (0, 0, 0); + let (mut slots, mut probes) = (HashMap::new(), HashMap::new()); + for m in &mismatches { + if failures.len() >= MAX_FAILURES { + unclassified += 1; + continue; + } + let status = match (m.settled, slots.get(&m.slot).copied()) { + (true, _) => Some(true), + (false, Some(status)) => status, + (false, None) => *slots.entry(m.slot).or_insert(canonical(&ctx, m.slot).await), + }; + if status == Some(false) { + forks += 1; + continue; + } + let is_excluded = match (m.maybe_excluded, probes.get(&m.key).copied()) { + (false, _) => false, + (true, Some(known)) => known, + (true, None) => *probes + .entry(&m.key) + .or_insert(probe_excluded(&ctx, &m.key, m.slot).await), + }; + if is_excluded { + excluded += 1; + continue; + } + let note = status.map_or(" (canonical unknown)", |_| ""); + failures.push(format!("{} at slot {}: {}{note}", m.key, m.slot, m.detail)); + } + println!("mismatches: {excluded} excluded keys, {forks} on forks, {unclassified} unclassified"); + + if failures.is_empty() { + println!("PASS: processed reads consistent with references and confirmed data"); + Ok(()) + } else { + for f in &failures { + println!(" FAIL {f}"); + } + Err(anyhow!("{} check(s) failed", failures.len() + unclassified)) + } +} + +/// Check 1. Returns token accounts and mints seen in cloudbreak's answers. +async fn cross_source( + ctx: &Ctx, + args: &Args, + keys: &[String], + mismatches: &mut Vec, + failures: &mut Vec, +) -> (Vec, Vec) { + let sources: Vec<&RpcEndpoint> = std::iter::once(&ctx.cloudbreak) + .chain(&ctx.references) + .collect(); + let (mut compared, mut cb_errors) = (0, 0); + let mut latencies = vec![Vec::new(); sources.len()]; + let mut highest = vec![0usize; sources.len()]; + let (mut accounts, mut mints) = (HashSet::new(), HashSet::new()); + for _ in 0..args.samples { + let sample = sample_keys(keys, args.keys_per_sample); + let params = json!([sample, {"commitment": "processed", "encoding": "base64"}]); + let calls = (sources.iter()).map(|s| call(ctx, s, "getMultipleAccounts", params.clone())); + let replies: Vec> = futures::future::join_all(calls) + .await + .into_iter() + .map(|r| r.ok().filter(|(json, _)| get_slot(json).is_some())) + .collect(); + let top = replies.iter().flatten().map(|r| get_slot(&r.0)).max(); + for (i, reply) in replies.iter().enumerate() { + if let Some((json, ms)) = reply { + latencies[i].push(*ms); + highest[i] += usize::from(Some(get_slot(json)) == top); + } + } + if replies[0].is_none() && replies[1..].iter().any(Option::is_some) { + cb_errors += 1; + } + if let Some((cb, _)) = &replies[0] + && let Some(slot) = get_slot(cb) + { + let values = cb["result"]["value"].as_array().into_iter().flatten(); + for (key, value) in sample.iter().zip(values) { + let owner = value["owner"].as_str().unwrap_or_default(); + let data = BASE64.decode(value["data"][0].as_str().unwrap_or_default()); + let data = data.unwrap_or_default(); + // A mint is 82 bytes, a token account 165, and a longer one has a type byte at 165. + match (TOKEN_PROGRAMS.contains(&owner), data.len(), data.get(165)) { + (true, 82, _) | (true, 166.., Some(1)) => mints.insert(key.clone()), + (true, 165, _) | (true, 166.., Some(2)) => accounts.insert(key.clone()), + _ => false, + }; + } + for (i, reply) in replies.iter().enumerate().skip(1) { + if let Some((reference, _)) = reply + && get_slot(reference) == Some(slot) + { + compared += 1; + let found = + differing_keys(&sample, slot, cb, reference, &sources[i].name, false); + mismatches.extend(found); + } + } + } + tokio::time::sleep(Duration::from_millis(args.interval_ms)).await; + } + + println!("cross-source: {} samples", args.samples); + println!(" source ok p50ms p90ms p99ms highest"); + for (i, source) in sources.iter().enumerate() { + latencies[i].sort_unstable(); + let [p50, p90, p99] = [50.0, 90.0, 99.0].map(|p| percentile(&latencies[i], p)); + let (name, ok) = (&source.name, latencies[i].len()); + let share = 100.0 * highest[i] as f64 / args.samples.max(1) as f64; + println!(" {name:<14} {ok:>6} {p50:>6} {p90:>6} {p99:>6} {share:>7.1}%"); + } + let n = args.samples; + if compared == 0 { + failures.push(format!("cross-source: no equal-slot pair in {n} samples")); + } + if cb_errors > 0 { + failures.push(format!( + "cross-source: cloudbreak errored on {cb_errors} samples" + )); + } + (accounts.into_iter().collect(), mints.into_iter().collect()) +} + +/// Check 2. A processed answer at S against confirmed answers at exactly S. +async fn processed_matches_confirmed( + ctx: &Ctx, + args: &Args, + keys: &[String], + mismatches: &mut Vec, + failures: &mut Vec, +) { + let oracles = [&ctx.cloudbreak, &ctx.references[0]]; + let mut classes: HashMap<&str, usize> = HashMap::new(); + for _ in 0..args.confirm_samples { + tokio::time::sleep(Duration::from_millis(args.interval_ms)).await; + let sample = sample_keys(keys, args.keys_per_sample); + let params = json!([sample, {"commitment": "processed", "encoding": "base64"}]); + let reply = call(ctx, &ctx.cloudbreak, "getMultipleAccounts", params).await; + let Some((s, processed)) = reply.ok().and_then(|(j, _)| Some((get_slot(&j)?, j))) else { + *classes.entry("error").or_default() += 1; + continue; + }; + let config = json!({"commitment": "confirmed", "encoding": "base64", "minContextSlot": s}); + let params = json!([sample, config]); + let (deadline, mut passed, mut hits) = (Instant::now() + ctx.wait, [false; 2], 0); + while passed.contains(&false) && Instant::now() < deadline { + for (i, oracle) in oracles.iter().enumerate() { + if passed[i] { + continue; + } + let reply = call(ctx, oracle, "getMultipleAccounts", params.clone()).await; + // Below S the node answers a minContextSlot error with no slot. + let Some((c, confirmed)) = reply.ok().and_then(|(j, _)| Some((get_slot(&j)?, j))) + else { + continue; + }; + passed[i] = true; + if c == s { + hits += 1; + let against = format!("{} confirmed", oracle.name); + let found = differing_keys(&sample, s, &processed, &confirmed, &against, true); + mismatches.extend(found); + } + } + tokio::time::sleep(POLL).await; + } + let status = if hits == 0 { + canonical(ctx, s).await + } else { + None + }; + *classes.entry(sample_class(hits, status)).or_default() += 1; + } + println!("processed-confirmed: {classes:?}"); + if !classes.contains_key("compared") { + let n = args.confirm_samples; + failures.push(format!( + "processed-confirmed: no exact-S answer in {n} samples" + )); + } +} + +/// Check 3. Single-key methods at processed against the first reference. +async fn token_methods( + ctx: &Ctx, + args: &Args, + keys: &[String], + (accounts, mints): &(Vec, Vec), + mismatches: &mut Vec, + failures: &mut Vec, +) { + let methods = [ + ("getBalance", keys), + ("getTokenAccountBalance", &accounts[..]), + ("getTokenSupply", &mints[..]), + ]; + let reference = &ctx.references[0]; + for (method, keys) in methods { + let keys = &keys[..keys.len().min(args.smoke_keys)]; + let (mut equal, mut excluded, mut cb_errors) = (0, 0, 0); + for key in keys { + let params = json!([key, {"commitment": "processed"}]); + for _ in 0..SLOT_TRIES { + let (cb, rf) = tokio::join!( + call(ctx, &ctx.cloudbreak, method, params.clone()), + call(ctx, reference, method, params.clone()) + ); + cb_errors += usize::from(cb.is_err() && rf.is_ok()); + let (Ok((cb, _)), Ok((rf, _))) = (cb, rf) else { + continue; + }; + if cb["error"]["code"] == EXCLUDED_CODE { + excluded += 1; + break; + } + let slot = match (get_slot(&cb), get_slot(&rf)) { + (Some(a), Some(b)) if a == b => a, + (Some(_), Some(_)) => continue, + (None, None) => break, + // One side errored, so the pair is a mismatch at the slot the other side names. + (Some(slot), None) | (None, Some(slot)) => slot, + }; + equal += 1; + // All three methods compare result.value directly, so one request type fits. + if !compare_responses(&cb, &rf, "none", GetBalance).matches { + let shown = + |r: &JsonValue| r.get("error").unwrap_or(&r["result"]["value"]).to_string(); + let (c, r, name) = (shown(&cb), shown(&rf), &reference.name); + mismatches.push(Mismatch { + slot, + key: format!("{method} {key}"), + detail: format!("cloudbreak {c}, {name} {r}"), + maybe_excluded: false, + settled: false, + }); + } + break; + } + } + let n = keys.len(); + println!("{method}: {n} keys, {equal} compared at equal slots, {excluded} excluded"); + if equal == 0 { + failures.push(format!("{method}: nothing compared for {n} keys")); + } + if cb_errors > 0 { + failures.push(format!("{method}: cloudbreak errored on {cb_errors} calls")); + } + } +} + +/// Check 4. Processed getSlot against confirmed getSlot and the references. +async fn slots_check(ctx: &Ctx, args: &Args, failures: &mut Vec) { + let slot_of = |reply: Result<(JsonValue, u128)>| reply.ok()?.0["result"].as_u64(); + let (mut below_confirmed, mut behind) = (0, Vec::new()); + for _ in 0..args.confirm_samples { + tokio::time::sleep(Duration::from_millis(args.interval_ms)).await; + let confirmed = json!([{"commitment": "confirmed"}]); + let confirmed = slot_of(call(ctx, &ctx.cloudbreak, "getSlot", confirmed).await); + let processed = json!([{"commitment": "processed"}]); + let calls = std::iter::once(&ctx.cloudbreak) + .chain(&ctx.references) + .map(|source| call(ctx, source, "getSlot", processed.clone())); + let slots: Vec> = futures::future::join_all(calls) + .await + .into_iter() + .map(slot_of) + .collect(); + let (Some(confirmed), Some(cb)) = (confirmed, slots[0]) else { + failures.push("getSlot: cloudbreak errored".to_string()); + continue; + }; + below_confirmed += usize::from(cb < confirmed); + if let Some(top) = slots[1..].iter().flatten().max() { + behind.push(top.saturating_sub(cb) as u128); + } + } + behind.sort_unstable(); + let [p50, p90] = [50.0, 90.0].map(|p| percentile(&behind, p)); + println!("getSlot: processed below confirmed in {below_confirmed} samples"); + if !behind.is_empty() { + println!("getSlot: slots behind the highest reference p50 {p50} p90 {p90}"); + } +} + +/// Keys whose values differ between two getMultipleAccounts answers at the same slot. +fn differing_keys( + keys: &[String], + slot: u64, + cloudbreak: &JsonValue, + other: &JsonValue, + against: &str, + settled: bool, +) -> Vec { + let mismatch = |key: &str, detail: String, maybe_excluded| Mismatch { + slot, + key: key.to_string(), + detail, + maybe_excluded, + settled, + }; + if compare_responses(cloudbreak, other, "base64", GetMultipleAccounts).matches { + return Vec::new(); + } + let (cb, other) = (&cloudbreak["result"]["value"], &other["result"]["value"]); + match (cb.as_array(), other.as_array()) { + (Some(a), Some(b)) if a.len() == keys.len() && b.len() == keys.len() => (keys.iter()) + .zip(a.iter().zip(b)) + .filter(|(_, (a, b))| a != b) + .map(|(key, (a, b))| mismatch(key, format!("{a} against {against} {b}"), a.is_null())) + .collect(), + _ => vec![mismatch( + "response", + format!("shape differs from {against}"), + false, + )], + } +} + +/// Class of a processed matches confirmed sample. +fn sample_class(hits: usize, canonical: Option) -> &'static str { + match (hits, canonical) { + (1.., _) => "compared", + (0, Some(false)) => "fork", + (0, Some(true)) => "miss", + (0, None) => "unresolved", + } +} + +/// True when cloudbreak's confirmed getAccountInfo at `slot` or later answers -32010, or answers +/// null while the reference serves the account, so the node does not index the key. +async fn probe_excluded(ctx: &Ctx, key: &str, slot: u64) -> bool { + let config = json!({"commitment": "confirmed", "encoding": "base64", "minContextSlot": slot, + "dataSlice": {"offset": 0, "length": 0}}); + let params = json!([key, config]); + let deadline = Instant::now() + ctx.wait; + loop { + let (cb, rf) = tokio::join!( + call(ctx, &ctx.cloudbreak, "getAccountInfo", params.clone()), + call(ctx, &ctx.references[0], "getAccountInfo", params.clone()) + ); + let (cb, rf) = (cb.unwrap_or_default().0, rf.unwrap_or_default().0); + if cb["error"]["code"] == EXCLUDED_CODE { + return true; + } + // Both answer with a slot once their confirmed slot reaches `slot`. + if get_slot(&cb).is_some() && get_slot(&rf).is_some() { + return cb["result"]["value"].is_null() && rf["result"]["value"].is_object(); + } + if Instant::now() >= deadline { + return false; + } + tokio::time::sleep(Duration::from_millis(500)).await; + } +} + +/// Canonical status of `slot` from the first reference once its confirmed slot reaches it. +async fn canonical(ctx: &Ctx, slot: u64) -> Option { + let oracle = &ctx.references[0]; + let deadline = Instant::now() + ctx.wait; + loop { + let tip = call(ctx, oracle, "getSlot", json!([{"commitment": "confirmed"}])).await; + if tip.ok().and_then(|(json, _)| json["result"].as_u64()) >= Some(slot) { + let params = json!([slot, slot, {"commitment": "confirmed"}]); + let (blocks, _) = call(ctx, oracle, "getBlocks", params).await.ok()?; + return Some(blocks["result"].as_array()?.contains(&json!(slot))); + } + if Instant::now() >= deadline { + return None; + } + tokio::time::sleep(Duration::from_millis(500)).await; + } +} + +/// Keys from `--pubkeys-file`, then writable keys of non-vote transactions in recent blocks. +async fn load_keys(ctx: &Ctx, args: &Args) -> Result> { + let mut keys: Vec = match &args.pubkeys_file { + Some(path) => std::fs::read_to_string(path)? + .lines() + .filter_map(|line| line.split('#').next().map(str::trim)) + .filter(|key| !key.is_empty()) + .map(str::to_string) + .collect(), + None => Vec::new(), + }; + let oracle = &ctx.references[0]; + let (tip, _) = call(ctx, oracle, "getSlot", json!([{"commitment": "confirmed"}])).await?; + let mut slot = tip["result"].as_u64().unwrap_or_default(); + let mut found = 0; + // Walks back past skipped slots and blocks not yet available. + for _ in 0..args.discover_blocks * 4 { + if found == args.discover_blocks || slot == 0 { + break; + } + let config = json!({"commitment": "confirmed", "maxSupportedTransactionVersion": 0, + "transactionDetails": "accounts", "rewards": false}); + let (block, _) = call(ctx, oracle, "getBlock", json!([slot, config])).await?; + slot -= 1; + let Some(txs) = block["result"]["transactions"].as_array() else { + continue; + }; + found += 1; + let tx_keys = txs + .iter() + .filter_map(|tx| tx["transaction"]["accountKeys"].as_array()); + let non_vote = tx_keys.filter(|ks| !ks.iter().any(|k| k["pubkey"] == VOTE_PROGRAM)); + let writable = non_vote.flatten().filter(|k| k["writable"] == true); + keys.extend(writable.filter_map(|k| k["pubkey"].as_str().map(str::to_string))); + } + let mut seen = HashSet::new(); + keys.retain(|k| seen.insert(k.clone())); + keys.truncate(MAX_KEYS); + Ok(keys) +} + +fn sample_keys(keys: &[String], n: usize) -> Vec { + let picks = keys.choose_multiple(&mut rand::thread_rng(), n); + picks.cloned().collect() +} + +async fn call( + ctx: &Ctx, + endpoint: &RpcEndpoint, + method: &str, + params: JsonValue, +) -> Result<(JsonValue, u128)> { + let request = json!({"jsonrpc": "2.0", "id": 1, "method": method, "params": params}); + send_rpc_request(&ctx.client, endpoint, &request, None).await +} + +#[cfg(test)] +mod tests { + use super::*; + + fn gma(slot: u64, values: JsonValue) -> JsonValue { + json!({"result": {"context": {"slot": slot}, "value": values}}) + } + + #[test] + fn equal_slot_comparison_flags_each_differing_key() { + let account = |lamports: u64| json!({"lamports": lamports, "owner": VOTE_PROGRAM}); + let keys = ["a", "b", "c", "d"].map(str::to_string); + let cb = gma(7, json!([account(1), null, account(2), account(3)])); + let other = gma(7, json!([account(1), account(5), null, account(4)])); + assert!(differing_keys(&keys, 7, &cb, &cb, "ref", false).is_empty()); + + let found = differing_keys(&keys, 7, &cb, &other, "ref", true); + let rows = found.iter().map(|m| (m.key.as_str(), m.maybe_excluded)); + assert!(rows.eq([("b", true), ("c", false), ("d", false)])); + assert!(found.iter().all(|m| m.slot == 7 && m.settled)); + + let short = differing_keys(&keys[..1], 7, &cb, &other, "ref", false); + assert_eq!((short.len(), short[0].key.as_str()), (1, "response")); + } + + #[test] + fn confirmed_samples_split_into_fork_and_miss() { + assert_eq!(sample_class(1, None), "compared"); + assert_eq!(sample_class(2, Some(false)), "compared"); + assert_eq!(sample_class(0, Some(false)), "fork"); + assert_eq!(sample_class(0, Some(true)), "miss"); + assert_eq!(sample_class(0, None), "unresolved"); + } +} diff --git a/crates/integration_tests/src/main.rs b/crates/integration_tests/src/main.rs index f20f58a..caf9403 100644 --- a/crates/integration_tests/src/main.rs +++ b/crates/integration_tests/src/main.rs @@ -11,6 +11,7 @@ mod compare_accounts_by_mint; mod compare_accounts_by_mint_vs_cluster; mod compare_genesis_hash; mod compare_largest_accounts; +mod compare_processed_accounts; mod compare_program_accounts; mod compare_supply; mod compare_token_largest_accounts; @@ -50,6 +51,8 @@ enum Commands { CompareTokenLargestAccounts(compare_token_largest_accounts::Args), /// Validate getSupply against getMultipleAccounts (independent read path) on the same node CompareSupply(compare_supply::Args), + /// Validate processed account reads against reference RPCs and confirmed data + CompareProcessedAccounts(compare_processed_accounts::Args), /// Compare getGenesisHash between two RPC endpoints (exact equality) CompareGenesisHash(compare_genesis_hash::Args), /// Compare getVersion: cloudbreak's composite string should embed the cluster's solana-core @@ -77,6 +80,7 @@ async fn main() -> anyhow::Result<()> { compare_token_largest_accounts::run(&args).await? } Commands::CompareSupply(args) => compare_supply::run(&args).await?, + Commands::CompareProcessedAccounts(args) => compare_processed_accounts::run(&args).await?, Commands::CompareGenesisHash(args) => compare_genesis_hash::run(&args).await?, Commands::CompareVersion(args) => compare_version::run(&args).await?, Commands::Benchmark(args) => benchmark::run(&args).await?, diff --git a/example.cloudbreak.api.toml b/example.cloudbreak.api.toml index f905774..ddbe439 100644 --- a/example.cloudbreak.api.toml +++ b/example.cloudbreak.api.toml @@ -63,7 +63,9 @@ span-filter = [ "slot_db", "mint_db", "mint_data", - "gpa_cache_finalize_query" + "gpa_cache_finalize_query", + "processed_read", + "account_db" ] # How to handle requests with "processed" commitment level. @@ -88,6 +90,19 @@ span-filter = [ enabled = false interval_ms = 200 +# Serves processed commitment for getAccountInfo, getMultipleAccounts, getBalance, +# getTokenAccountBalance, getTokenSupply and getSlot from a Yellowstone block feed held in +# memory around the Postgres confirmed slot. Other methods keep following +# processed-commitment. Requires [slot-syncronizer] enabled = true. Startup fails +# without it. For these methods it overrides processed-commitment = "reject": +# when no processed blocks can be proven, a processed request +# answers exactly as a confirmed request would. Each API instance carries its own +# full block feed. +# [processed-accounts] +# enabled = false +# endpoint = "https://grpc.example:443" +# x-token = "..." + # Optional module: cache configuration for the GPA queries. [gpa-cache] max-total-bytes = 1_073_741_824 # 1GB