From 1581a0d9c72deb9dd53ee4b9692518828368d395 Mon Sep 17 00:00:00 2001 From: CGodiksen <36046286+CGodiksen@users.noreply.github.com> Date: Tue, 11 Aug 2026 07:10:25 +0200 Subject: [PATCH 01/33] Add proto messages for NodeMetadata and ClusterNodes --- crates/modelardb_types/src/flight/protocol.proto | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/crates/modelardb_types/src/flight/protocol.proto b/crates/modelardb_types/src/flight/protocol.proto index 141816e3..5885d537 100644 --- a/crates/modelardb_types/src/flight/protocol.proto +++ b/crates/modelardb_types/src/flight/protocol.proto @@ -108,3 +108,18 @@ message UpdateConfiguration { // New value for the setting. optional uint64 new_value = 2; } + +// Metadata identifying a single node in a ModelarDB cluster. +message NodeMetadata { + // Apache Arrow Flight URL that uniquely identifies the node. + string url = 1; + + // Mode the node was started in, either "edge" or "cloud". + string mode = 2; +} + +// The nodes that are currently part of a ModelarDB cluster. +message ClusterNodes { + repeated NodeMetadata nodes = 1; +} + From 9370183efc9b6e9850f9ef6b5d924ad4858ed6c1 Mon Sep 17 00:00:00 2001 From: CGodiksen <36046286+CGodiksen@users.noreply.github.com> Date: Tue, 11 Aug 2026 07:18:46 +0200 Subject: [PATCH 02/33] Add NodeMetrics proto message --- .../modelardb_types/src/flight/protocol.proto | 38 +++++++++++++++++++ 1 file changed, 38 insertions(+) diff --git a/crates/modelardb_types/src/flight/protocol.proto b/crates/modelardb_types/src/flight/protocol.proto index 5885d537..8f7dbd64 100644 --- a/crates/modelardb_types/src/flight/protocol.proto +++ b/crates/modelardb_types/src/flight/protocol.proto @@ -123,3 +123,41 @@ message ClusterNodes { repeated NodeMetadata nodes = 1; } +// Resource usage metrics for a single ModelarDB node. +message NodeMetrics { + // Percentage of the CPU that is currently in use across all cores. + double cpu_usage_percentage = 1; + + // Number of logical CPUs available to the node. + uint32 cpu_count = 2; + + // Amount of memory currently in use on the node. + uint64 used_memory_in_bytes = 3; + + // Total amount of memory on the node. + uint64 total_memory_in_bytes = 4; + + // Amount of disk space currently in use on the disk holding the local data folder. + uint64 used_disk_space_in_bytes = 5; + + // Total amount of disk space on the disk holding the local data folder. + uint64 total_disk_space_in_bytes = 6; + + // Amount of memory currently in use for storing ingested time series. + uint64 ingested_used_memory_in_bytes = 7; + + // Total amount of memory reserved for storing ingested time series. + uint64 ingested_reserved_memory_in_bytes = 8; + + // Amount of memory currently in use for storing uncompressed data buffers. + uint64 uncompressed_used_memory_in_bytes = 9; + + // Total amount of memory reserved for storing uncompressed data buffers. + uint64 uncompressed_reserved_memory_in_bytes = 10; + + // Amount of memory currently in use for storing compressed data buffers. + uint64 compressed_used_memory_in_bytes = 11; + + // Total amount of memory reserved for storing compressed data buffers. + uint64 compressed_reserved_memory_in_bytes = 12; +} From 054c01d21e250212693d8743e00180a18bdec49e Mon Sep 17 00:00:00 2001 From: CGodiksen <36046286+CGodiksen@users.noreply.github.com> Date: Tue, 11 Aug 2026 07:39:24 +0200 Subject: [PATCH 03/33] Add node to ClusterMode::SingleNode for self identity --- crates/modelardb_server/src/configuration.rs | 6 ++++-- crates/modelardb_server/src/context.rs | 5 +++-- crates/modelardb_server/src/data_folders.rs | 6 ++++-- crates/modelardb_server/src/main.rs | 4 ++-- crates/modelardb_server/src/remote/mod.rs | 4 ++-- 5 files changed, 15 insertions(+), 10 deletions(-) diff --git a/crates/modelardb_server/src/configuration.rs b/crates/modelardb_server/src/configuration.rs index 39b1ae59..289c6910 100644 --- a/crates/modelardb_server/src/configuration.rs +++ b/crates/modelardb_server/src/configuration.rs @@ -602,9 +602,10 @@ mod tests { .await .unwrap(); + let node = Node::new("edge".to_owned(), ServerMode::Edge); let result = ConfigurationManager::try_new( local_data_folder, - ClusterMode::SingleNode, + ClusterMode::SingleNode(node), &default_args(), ) .await; @@ -625,9 +626,10 @@ mod tests { let path = temp_dir.path().join(CONFIGURATION_FILE_NAME); std::fs::write(path, "invalid_toml").unwrap(); + let node = Node::new("edge".to_owned(), ServerMode::Edge); let result = ConfigurationManager::try_new( local_data_folder, - ClusterMode::SingleNode, + ClusterMode::SingleNode(node), &default_args(), ) .await; diff --git a/crates/modelardb_server/src/context.rs b/crates/modelardb_server/src/context.rs index 9d600959..9048f6bb 100644 --- a/crates/modelardb_server/src/context.rs +++ b/crates/modelardb_server/src/context.rs @@ -504,7 +504,7 @@ mod tests { use clap::Parser; use modelardb_storage::data_folder::DataFolder; use modelardb_test::table::{self, NORMAL_TABLE_NAME, TIME_SERIES_TABLE_NAME}; - use modelardb_types::types::MAX_RETENTION_PERIOD_IN_SECONDS; + use modelardb_types::types::{MAX_RETENTION_PERIOD_IN_SECONDS, Node, ServerMode}; use tempfile::TempDir; // Tests for Context. @@ -1162,11 +1162,12 @@ mod tests { async fn create_context(temp_dir: &TempDir) -> Arc { let temp_dir_url = temp_dir.path().to_str().unwrap(); let local_data_folder = DataFolder::open_local_url(temp_dir_url).await.unwrap(); + let node = Node::new("edge".to_owned(), ServerMode::Edge); Arc::new( Context::try_new( DataFolders::new(local_data_folder.clone(), None, local_data_folder), - ClusterMode::SingleNode, + ClusterMode::SingleNode(node), &ServerArgs::parse_from(["modelardbd", "edge", "data"]), ) .await diff --git a/crates/modelardb_server/src/data_folders.rs b/crates/modelardb_server/src/data_folders.rs index 8c01d2f9..f0098e02 100644 --- a/crates/modelardb_server/src/data_folders.rs +++ b/crates/modelardb_server/src/data_folders.rs @@ -66,9 +66,10 @@ impl DataFolders { .. } => { let local_data_folder = DataFolder::open_local_url(local_data_folder).await?; + let node = Node::new(url_with_port, ServerMode::Edge); Ok(( - ClusterMode::SingleNode, + ClusterMode::SingleNode(node), Self::new(local_data_folder.clone(), None, local_data_folder), )) } @@ -144,7 +145,8 @@ mod tests { .await .unwrap(); - assert!(matches!(cluster_mode, ClusterMode::SingleNode)); + let expected_node = Node::new("grpc://127.0.0.1:9999".to_owned(), ServerMode::Edge); + assert!(matches!(cluster_mode, ClusterMode::SingleNode(node) if node == expected_node)); assert!(data_folders.maybe_remote_data_folder.is_none()); } diff --git a/crates/modelardb_server/src/main.rs b/crates/modelardb_server/src/main.rs index 673e2847..1845518a 100644 --- a/crates/modelardb_server/src/main.rs +++ b/crates/modelardb_server/src/main.rs @@ -26,7 +26,7 @@ mod storage; use std::sync::Arc; use clap::{Parser, Subcommand}; -use modelardb_types::types::CloudCredentials; +use modelardb_types::types::{CloudCredentials, Node}; use tracing_subscriber::{layer::SubscriberExt, util::SubscriberInitExt}; use crate::cluster::Cluster; @@ -41,7 +41,7 @@ static ALLOC: snmalloc_rs::SnMalloc = snmalloc_rs::SnMalloc; /// server is started. #[derive(Clone)] pub(crate) enum ClusterMode { - SingleNode, + SingleNode(Node), MultiNode(Box), } diff --git a/crates/modelardb_server/src/remote/mod.rs b/crates/modelardb_server/src/remote/mod.rs index 8f245963..ac791d0a 100644 --- a/crates/modelardb_server/src/remote/mod.rs +++ b/crates/modelardb_server/src/remote/mod.rs @@ -83,7 +83,7 @@ pub async fn start_apache_arrow_flight_server( let maybe_cluster_key = match context.configuration_manager.read().await.cluster_mode() { ClusterMode::MultiNode(cluster) => Some(cluster.key().clone()), - ClusterMode::SingleNode => None, + ClusterMode::SingleNode(_) => None, }; let auth_layer = AuthLayer::new(maybe_authenticator, maybe_cluster_key); @@ -1060,7 +1060,7 @@ impl FlightService for FlightServiceHandler { let configuration_manager = self.context.configuration_manager.read().await; let node_type = match configuration_manager.cluster_mode() { - ClusterMode::SingleNode => "SingleEdge", + ClusterMode::SingleNode(_) => "SingleEdge", ClusterMode::MultiNode(cluster) => match cluster.node().mode { ServerMode::Edge => "ClusterEdge", ServerMode::Cloud => "ClusterCloud", From 6af255448e5a36ea456eb542dcac3b73639e822b Mon Sep 17 00:00:00 2001 From: CGodiksen <36046286+CGodiksen@users.noreply.github.com> Date: Tue, 11 Aug 2026 09:21:49 +0200 Subject: [PATCH 04/33] Add sysinfo as a dependency to server --- Cargo.lock | 1 + crates/modelardb_server/Cargo.toml | 1 + 2 files changed, 2 insertions(+) diff --git a/Cargo.lock b/Cargo.lock index bae02ae5..4fedc9ef 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3599,6 +3599,7 @@ dependencies = [ "serde", "snmalloc-rs", "sqlparser", + "sysinfo", "tempfile", "tokio", "tokio-stream", diff --git a/crates/modelardb_server/Cargo.toml b/crates/modelardb_server/Cargo.toml index 8d837053..77e82bae 100644 --- a/crates/modelardb_server/Cargo.toml +++ b/crates/modelardb_server/Cargo.toml @@ -46,6 +46,7 @@ rand.workspace = true serde.workspace = true snmalloc-rs = { workspace = true, features = ["build_cc"] } sqlparser.workspace = true +sysinfo.workspace = true tokio = { workspace = true, features = ["rt-multi-thread", "signal"] } tokio-stream.workspace = true toml.workspace = true From d4a965781e2df064f294f9d6ade0b3c45d2d7cbf Mon Sep 17 00:00:00 2001 From: CGodiksen <36046286+CGodiksen@users.noreply.github.com> Date: Tue, 11 Aug 2026 09:22:07 +0200 Subject: [PATCH 05/33] Add getters for the remaining memory in bytes to storage engine --- crates/modelardb_server/src/storage/mod.rs | 15 +++++++++++++++ crates/modelardb_server/src/storage/types.rs | 1 - 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/crates/modelardb_server/src/storage/mod.rs b/crates/modelardb_server/src/storage/mod.rs index 19eb2718..bb914efe 100644 --- a/crates/modelardb_server/src/storage/mod.rs +++ b/crates/modelardb_server/src/storage/mod.rs @@ -357,6 +357,21 @@ impl StorageEngine { .await } + /// Return the amount of memory available for ingested data in bytes. + pub(super) fn remaining_ingested_memory_in_bytes(&self) -> i64 { + self.memory_pool.remaining_ingested_memory_in_bytes() + } + + /// Return the amount of memory available for uncompressed data in bytes. + pub(super) fn remaining_uncompressed_memory_in_bytes(&self) -> i64 { + self.memory_pool.remaining_uncompressed_memory_in_bytes() + } + + /// Return the amount of memory available for compressed data in bytes. + pub(super) fn remaining_compressed_memory_in_bytes(&self) -> i64 { + self.memory_pool.remaining_compressed_memory_in_bytes() + } + /// Mark the table with `table_name` as dropped in the data transfer component. This will prevent /// data related to the table from being transferred to the remote data folder. pub(super) async fn mark_table_as_dropped(&self, table_name: &str) { diff --git a/crates/modelardb_server/src/storage/types.rs b/crates/modelardb_server/src/storage/types.rs index 73d196cc..05c3568c 100644 --- a/crates/modelardb_server/src/storage/types.rs +++ b/crates/modelardb_server/src/storage/types.rs @@ -98,7 +98,6 @@ impl MemoryPool { } /// Return the amount of memory available for ingested data in bytes. - #[cfg(test)] #[must_use] pub(super) fn remaining_ingested_memory_in_bytes(&self) -> i64 { *self From bdea1a23ab61365bcce045b07604ccce8fc17282 Mon Sep 17 00:00:00 2001 From: CGodiksen <36046286+CGodiksen@users.noreply.github.com> Date: Tue, 11 Aug 2026 09:24:01 +0200 Subject: [PATCH 06/33] Add method to get all nodes in the cluster --- crates/modelardb_server/src/cluster.rs | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/crates/modelardb_server/src/cluster.rs b/crates/modelardb_server/src/cluster.rs index 10959a02..9a0b8feb 100644 --- a/crates/modelardb_server/src/cluster.rs +++ b/crates/modelardb_server/src/cluster.rs @@ -357,6 +357,15 @@ impl Cluster { Ok(url.to_owned()) } + /// Return all nodes currently in the cluster. If the nodes could not be retrieved, return + /// [`ModelarDbServerError`]. + pub(crate) async fn nodes(&self) -> Result> { + self.remote_data_folder + .nodes() + .await + .map_err(|error| error.into()) + } + /// Return all nodes in the cluster except the node that was saved when the [`Cluster`] was /// created. If the nodes could not be retrieved, return [`ModelarDbServerError`]. async fn peer_nodes(&self) -> Result> { From 8f3252dc3cd1d4952220f7805ad9db6f10660531 Mon Sep 17 00:00:00 2001 From: CGodiksen <36046286+CGodiksen@users.noreply.github.com> Date: Tue, 11 Aug 2026 09:27:00 +0200 Subject: [PATCH 07/33] Add util method to get local data folder disk space --- crates/modelardb_server/src/remote/mod.rs | 24 +++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/crates/modelardb_server/src/remote/mod.rs b/crates/modelardb_server/src/remote/mod.rs index ac791d0a..c75f201c 100644 --- a/crates/modelardb_server/src/remote/mod.rs +++ b/crates/modelardb_server/src/remote/mod.rs @@ -51,6 +51,7 @@ use modelardb_types::flight::protocol; use modelardb_types::functions; use modelardb_types::types::{ServerMode, Table, TimeSeriesTableMetadata}; use prost::Message; +use sysinfo::Disks; use tokio::sync::mpsc::{self, Sender}; use tokio::task; use tokio_stream::wrappers::ReceiverStream; @@ -291,6 +292,29 @@ fn empty_record_batch_stream() -> SendableRecordBatchStream { Box::pin(EmptyRecordBatchStream::new(Arc::new(Schema::empty()))) } +/// Return the used and total disk space in bytes for the disk holding the local data folder. The +/// disk is identified by finding the mounted disk whose mount point is the longest prefix of the +/// local data folder path. If no disk matches, e.g., because the data folder is in memory, the +/// largest-capacity disk is used instead. If no disks are found, `(0, 0)` is returned. +fn local_data_folder_disk_space(context: &Context) -> (u64, u64) { + let disks = Disks::new_with_refreshed_list(); + let location = context.data_folders.local_data_folder.location(); + + let maybe_disk = disks + .iter() + .filter(|disk| location.starts_with(&*disk.mount_point().to_string_lossy())) + .max_by_key(|disk| disk.mount_point().as_os_str().len()) + .or_else(|| disks.iter().max_by_key(|disk| disk.total_space())); + + if let Some(disk) = maybe_disk { + let total = disk.total_space(); + let used = total.saturating_sub(disk.available_space()); + (used, total) + } else { + (0, 0) + } +} + /// Convert an `error` to a [`Status`] with [`tonic::Code::InvalidArgument`] as the code and `error` /// converted to a [`String`] as the message. pub fn error_to_status_invalid_argument(error: impl Error) -> Status { From 469b6ba16573fc9239823f04333871fb1ed3150a Mon Sep 17 00:00:00 2001 From: CGodiksen <36046286+CGodiksen@users.noreply.github.com> Date: Tue, 11 Aug 2026 09:32:05 +0200 Subject: [PATCH 08/33] Add ListNodes action --- crates/modelardb_server/src/remote/mod.rs | 30 +++++++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/crates/modelardb_server/src/remote/mod.rs b/crates/modelardb_server/src/remote/mod.rs index c75f201c..67faca0f 100644 --- a/crates/modelardb_server/src/remote/mod.rs +++ b/crates/modelardb_server/src/remote/mod.rs @@ -922,6 +922,9 @@ impl FlightService for FlightServiceHandler { /// and the change is persisted in the configuration file. /// * `NodeType`: Get the type of the node. The type is `SingleEdge`, `ClusterEdge`, or /// `ClusterCloud`. The type of the node is returned as a string. + /// * `ListNodes`: Get the nodes that are currently part of the cluster. The nodes are returned + /// in a [`ClusterNodes`](protocol::ClusterNodes) protobuf message. A single node returns only + /// itself. async fn do_action( &self, request: Request, @@ -1098,6 +1101,33 @@ impl FlightService for FlightServiceHandler { Ok(Response::new(Box::pin(stream::once(async { Ok(flight_result) })))) + } else if action.r#type == "ListNodes" { + let configuration_manager = self.context.configuration_manager.read().await; + + let nodes = match configuration_manager.cluster_mode() { + ClusterMode::MultiNode(cluster) => { + cluster.nodes().await.map_err(error_to_status_internal)? + } + ClusterMode::SingleNode(node) => vec![node.clone()], + }; + + let cluster_nodes = protocol::ClusterNodes { + nodes: nodes + .into_iter() + .map(|node| protocol::NodeMetadata { + url: node.url, + mode: node.mode.to_string(), + }) + .collect(), + }; + + let protobuf_bytes = cluster_nodes.encode_to_vec(); + + Ok(Response::new(Box::pin(stream::once(async { + Ok(FlightResult { + body: protobuf_bytes.into(), + }) + })))) } else { Err(Status::unimplemented("Action not implemented.")) } From c6f69167203dbfba1588e51aa72731ac3d61bf74 Mon Sep 17 00:00:00 2001 From: CGodiksen <36046286+CGodiksen@users.noreply.github.com> Date: Tue, 11 Aug 2026 09:44:21 +0200 Subject: [PATCH 09/33] Add NodeMetrics action --- crates/modelardb_server/src/remote/mod.rs | 68 ++++++++++++++++++++++- 1 file changed, 67 insertions(+), 1 deletion(-) diff --git a/crates/modelardb_server/src/remote/mod.rs b/crates/modelardb_server/src/remote/mod.rs index 67faca0f..3dc55264 100644 --- a/crates/modelardb_server/src/remote/mod.rs +++ b/crates/modelardb_server/src/remote/mod.rs @@ -51,7 +51,7 @@ use modelardb_types::flight::protocol; use modelardb_types::functions; use modelardb_types::types::{ServerMode, Table, TimeSeriesTableMetadata}; use prost::Message; -use sysinfo::Disks; +use sysinfo::{Disks, System}; use tokio::sync::mpsc::{self, Sender}; use tokio::task; use tokio_stream::wrappers::ReceiverStream; @@ -925,6 +925,9 @@ impl FlightService for FlightServiceHandler { /// * `ListNodes`: Get the nodes that are currently part of the cluster. The nodes are returned /// in a [`ClusterNodes`](protocol::ClusterNodes) protobuf message. A single node returns only /// itself. + /// * `NodeMetrics`: Get the current resource usage metrics of the node, including CPU, memory, + /// disk, and storage engine memory usage. The metrics are returned in a + /// [`NodeMetrics`](protocol::NodeMetrics) protobuf message. async fn do_action( &self, request: Request, @@ -1123,6 +1126,69 @@ impl FlightService for FlightServiceHandler { let protobuf_bytes = cluster_nodes.encode_to_vec(); + Ok(Response::new(Box::pin(stream::once(async { + Ok(FlightResult { + body: protobuf_bytes.into(), + }) + })))) + } else if action.r#type == "NodeMetrics" { + let mut system = System::new(); + + // Sample the CPU twice, separated by the minimum update interval, since a single + // refresh reads zero. + system.refresh_cpu_usage(); + tokio::time::sleep(sysinfo::MINIMUM_CPU_UPDATE_INTERVAL).await; + system.refresh_cpu_usage(); + + let cpu_usage_percentage = system.global_cpu_usage() as f64; + let cpu_count = system.cpus().len() as u32; + + system.refresh_memory(); + let used_memory_in_bytes = system.used_memory(); + let total_memory_in_bytes = system.total_memory(); + + let (used_disk_space_in_bytes, total_disk_space_in_bytes) = + local_data_folder_disk_space(&self.context); + + // Pair each reserved value from the configuration with the remaining value from the + // storage engine to compute how much of the reservation is currently in use. + let configuration_manager = self.context.configuration_manager.read().await; + let storage_engine = self.context.storage_engine.read().await; + + let ingested_reserved_memory_in_bytes = + configuration_manager.ingested_reserved_memory_in_bytes(); + let uncompressed_reserved_memory_in_bytes = + configuration_manager.uncompressed_reserved_memory_in_bytes(); + let compressed_reserved_memory_in_bytes = + configuration_manager.compressed_reserved_memory_in_bytes(); + + let ingested_used_memory_in_bytes = (ingested_reserved_memory_in_bytes as i64 + - storage_engine.remaining_ingested_memory_in_bytes()) + .max(0) as u64; + let uncompressed_used_memory_in_bytes = (uncompressed_reserved_memory_in_bytes as i64 + - storage_engine.remaining_uncompressed_memory_in_bytes()) + .max(0) as u64; + let compressed_used_memory_in_bytes = (compressed_reserved_memory_in_bytes as i64 + - storage_engine.remaining_compressed_memory_in_bytes()) + .max(0) as u64; + + let node_metrics = protocol::NodeMetrics { + cpu_usage_percentage, + cpu_count, + used_memory_in_bytes, + total_memory_in_bytes, + used_disk_space_in_bytes, + total_disk_space_in_bytes, + ingested_used_memory_in_bytes, + ingested_reserved_memory_in_bytes, + uncompressed_used_memory_in_bytes, + uncompressed_reserved_memory_in_bytes, + compressed_used_memory_in_bytes, + compressed_reserved_memory_in_bytes, + }; + + let protobuf_bytes = node_metrics.encode_to_vec(); + Ok(Response::new(Box::pin(stream::once(async { Ok(FlightResult { body: protobuf_bytes.into(), From 0ce9d4e938b4d48433030cd75ac693897ed15c07 Mon Sep 17 00:00:00 2001 From: CGodiksen <36046286+CGodiksen@users.noreply.github.com> Date: Tue, 11 Aug 2026 10:40:24 +0200 Subject: [PATCH 10/33] Add new actions to list_actions --- crates/modelardb_server/src/remote/mod.rs | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/crates/modelardb_server/src/remote/mod.rs b/crates/modelardb_server/src/remote/mod.rs index 3dc55264..baca266d 100644 --- a/crates/modelardb_server/src/remote/mod.rs +++ b/crates/modelardb_server/src/remote/mod.rs @@ -1246,6 +1246,16 @@ impl FlightService for FlightServiceHandler { description: "Get the type of the node.".to_owned(), }; + let list_nodes_action = ActionType { + r#type: "ListNodes".to_owned(), + description: "Get the nodes that are currently part of the cluster.".to_owned(), + }; + + let node_metrics_action = ActionType { + r#type: "NodeMetrics".to_owned(), + description: "Get the current resource usage metrics of the node.".to_owned(), + }; + let output = stream::iter(vec![ Ok(create_tables_action), Ok(flush_memory_action), @@ -1254,6 +1264,8 @@ impl FlightService for FlightServiceHandler { Ok(get_configuration_action), Ok(update_configuration_action), Ok(node_type_action), + Ok(list_nodes_action), + Ok(node_metrics_action), ]); Ok(Response::new(Box::pin(output))) From b22605e8d82f6ab72f762648f00d1aaec7556a63 Mon Sep 17 00:00:00 2001 From: CGodiksen <36046286+CGodiksen@users.noreply.github.com> Date: Tue, 11 Aug 2026 20:11:51 +0200 Subject: [PATCH 11/33] Use new nodes method in peer_nodes() --- crates/modelardb_server/src/cluster.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/modelardb_server/src/cluster.rs b/crates/modelardb_server/src/cluster.rs index 9a0b8feb..c515f9e7 100644 --- a/crates/modelardb_server/src/cluster.rs +++ b/crates/modelardb_server/src/cluster.rs @@ -369,7 +369,7 @@ impl Cluster { /// Return all nodes in the cluster except the node that was saved when the [`Cluster`] was /// created. If the nodes could not be retrieved, return [`ModelarDbServerError`]. async fn peer_nodes(&self) -> Result> { - let nodes = self.remote_data_folder.nodes().await?; + let nodes = self.nodes().await?; Ok(nodes .into_iter() From 51d55d5fe848a2619fcad4e157fd28898f663929 Mon Sep 17 00:00:00 2001 From: CGodiksen <36046286+CGodiksen@users.noreply.github.com> Date: Tue, 11 Aug 2026 20:19:29 +0200 Subject: [PATCH 12/33] Separate ListNodes implementation into relevant helper functions --- crates/modelardb_server/src/main.rs | 12 ++++++++++++ crates/modelardb_server/src/remote/mod.rs | 24 ++++++----------------- crates/modelardb_types/src/flight/mod.rs | 17 +++++++++++++++- 3 files changed, 34 insertions(+), 19 deletions(-) diff --git a/crates/modelardb_server/src/main.rs b/crates/modelardb_server/src/main.rs index 1845518a..a5e34e03 100644 --- a/crates/modelardb_server/src/main.rs +++ b/crates/modelardb_server/src/main.rs @@ -45,6 +45,18 @@ pub(crate) enum ClusterMode { MultiNode(Box), } +impl ClusterMode { + /// Return all nodes in the deployment. A single node returns only itself, while a node in a + /// cluster returns every node in the cluster. If the nodes could not be retrieved, return + /// [`ModelarDbServerError`](error::ModelarDbServerError). + pub(crate) async fn nodes(&self) -> Result> { + match self { + ClusterMode::SingleNode(node) => Ok(vec![node.clone()]), + ClusterMode::MultiNode(cluster) => cluster.nodes().await, + } + } +} + /// Command line arguments for the ModelarDB server. #[derive(Parser)] #[command( diff --git a/crates/modelardb_server/src/remote/mod.rs b/crates/modelardb_server/src/remote/mod.rs index baca266d..52d31962 100644 --- a/crates/modelardb_server/src/remote/mod.rs +++ b/crates/modelardb_server/src/remote/mod.rs @@ -1106,25 +1106,13 @@ impl FlightService for FlightServiceHandler { })))) } else if action.r#type == "ListNodes" { let configuration_manager = self.context.configuration_manager.read().await; + let nodes = configuration_manager + .cluster_mode() + .nodes() + .await + .map_err(error_to_status_internal)?; - let nodes = match configuration_manager.cluster_mode() { - ClusterMode::MultiNode(cluster) => { - cluster.nodes().await.map_err(error_to_status_internal)? - } - ClusterMode::SingleNode(node) => vec![node.clone()], - }; - - let cluster_nodes = protocol::ClusterNodes { - nodes: nodes - .into_iter() - .map(|node| protocol::NodeMetadata { - url: node.url, - mode: node.mode.to_string(), - }) - .collect(), - }; - - let protobuf_bytes = cluster_nodes.encode_to_vec(); + let protobuf_bytes = modelardb_types::flight::encode_and_serialize_cluster_nodes(nodes); Ok(Response::new(Box::pin(stream::once(async { Ok(FlightResult { diff --git a/crates/modelardb_types/src/flight/mod.rs b/crates/modelardb_types/src/flight/mod.rs index aa077450..a3ad3406 100644 --- a/crates/modelardb_types/src/flight/mod.rs +++ b/crates/modelardb_types/src/flight/mod.rs @@ -28,7 +28,7 @@ use prost::bytes::Bytes; use crate::error::{ModelarDbTypesError, Result}; use crate::functions::{try_convert_bytes_to_schema, try_convert_schema_to_bytes}; -use crate::types::{ErrorBound, GeneratedColumn, Table, TimeSeriesTableMetadata}; +use crate::types::{ErrorBound, GeneratedColumn, Node, Table, TimeSeriesTableMetadata}; pub mod protocol { include!(concat!(env!("OUT_DIR"), "/modelardb.flight.protocol.rs")); @@ -194,6 +194,21 @@ fn decode_error_bounds( Ok(error_bounds) } +/// Encode `nodes` into a [`ClusterNodes`](protocol::ClusterNodes) protobuf message and serialize it. +pub fn encode_and_serialize_cluster_nodes(nodes: Vec) -> Vec { + let cluster_nodes = protocol::ClusterNodes { + nodes: nodes + .into_iter() + .map(|node| protocol::NodeMetadata { + url: node.url, + mode: node.mode.to_string(), + }) + .collect(), + }; + + cluster_nodes.encode_to_vec() +} + /// Decode the generated column expressions from a vector of byte expressions into a vector of /// optional [`GeneratedColumn`]. Return [`ModelarDbTypesError`] if the expression is invalid. fn decode_generated_column_expressions( From 330ef043e8c38356cbb41b38b2012382b7d80f57 Mon Sep 17 00:00:00 2001 From: CGodiksen <36046286+CGodiksen@users.noreply.github.com> Date: Tue, 11 Aug 2026 20:30:29 +0200 Subject: [PATCH 13/33] Move ClusterMode from main.rs into cluster.rs --- crates/modelardb_server/src/cluster.rs | 20 ++++++++++++++++++++ crates/modelardb_server/src/main.rs | 24 ++---------------------- 2 files changed, 22 insertions(+), 22 deletions(-) diff --git a/crates/modelardb_server/src/cluster.rs b/crates/modelardb_server/src/cluster.rs index c515f9e7..1cd79789 100644 --- a/crates/modelardb_server/src/cluster.rs +++ b/crates/modelardb_server/src/cluster.rs @@ -36,6 +36,26 @@ use tonic::transport::Endpoint; use crate::context::Context; use crate::error::{ModelarDbServerError, Result}; +/// The different possible modes that a ModelarDB server can be deployed in, assigned when the +/// server is started. +#[derive(Clone)] +pub(crate) enum ClusterMode { + SingleNode(Node), + MultiNode(Box), +} + +impl ClusterMode { + /// Return all nodes in the deployment. A single node returns only itself, while a node in a + /// cluster returns every node in the cluster. If the nodes could not be retrieved, return + /// [`ModelarDbServerError`]. + pub(crate) async fn nodes(&self) -> Result> { + match self { + ClusterMode::SingleNode(node) => Ok(vec![node.clone()]), + ClusterMode::MultiNode(cluster) => cluster.nodes().await, + } + } +} + /// Stores the node that represents the local system and allows for performing operations that need /// to be applied to every peer node in the cluster. #[derive(Clone)] diff --git a/crates/modelardb_server/src/main.rs b/crates/modelardb_server/src/main.rs index a5e34e03..d4695afd 100644 --- a/crates/modelardb_server/src/main.rs +++ b/crates/modelardb_server/src/main.rs @@ -26,10 +26,10 @@ mod storage; use std::sync::Arc; use clap::{Parser, Subcommand}; -use modelardb_types::types::{CloudCredentials, Node}; +use modelardb_types::types::CloudCredentials; use tracing_subscriber::{layer::SubscriberExt, util::SubscriberInitExt}; -use crate::cluster::Cluster; +use crate::cluster::ClusterMode; use crate::context::Context; use crate::data_folders::DataFolders; use crate::error::Result; @@ -37,26 +37,6 @@ use crate::error::Result; #[global_allocator] static ALLOC: snmalloc_rs::SnMalloc = snmalloc_rs::SnMalloc; -/// The different possible modes that a ModelarDB server can be deployed in, assigned when the -/// server is started. -#[derive(Clone)] -pub(crate) enum ClusterMode { - SingleNode(Node), - MultiNode(Box), -} - -impl ClusterMode { - /// Return all nodes in the deployment. A single node returns only itself, while a node in a - /// cluster returns every node in the cluster. If the nodes could not be retrieved, return - /// [`ModelarDbServerError`](error::ModelarDbServerError). - pub(crate) async fn nodes(&self) -> Result> { - match self { - ClusterMode::SingleNode(node) => Ok(vec![node.clone()]), - ClusterMode::MultiNode(cluster) => cluster.nodes().await, - } - } -} - /// Command line arguments for the ModelarDB server. #[derive(Parser)] #[command( From 1d74d9dda0cd23a9cc2f2dabb90f4d7d0141ed20 Mon Sep 17 00:00:00 2001 From: CGodiksen <36046286+CGodiksen@users.noreply.github.com> Date: Tue, 11 Aug 2026 20:33:33 +0200 Subject: [PATCH 14/33] Update imports for ClusterMode --- crates/modelardb_server/src/cluster.rs | 2 +- crates/modelardb_server/src/context.rs | 3 ++- crates/modelardb_server/src/data_folders.rs | 4 ++-- crates/modelardb_server/src/remote/mod.rs | 2 +- 4 files changed, 6 insertions(+), 5 deletions(-) diff --git a/crates/modelardb_server/src/cluster.rs b/crates/modelardb_server/src/cluster.rs index 1cd79789..047409e6 100644 --- a/crates/modelardb_server/src/cluster.rs +++ b/crates/modelardb_server/src/cluster.rs @@ -514,8 +514,8 @@ mod test { use modelardb_types::types::{ArrowTimestamp, ArrowValue, ErrorBound, ServerMode}; use tempfile::TempDir; + use crate::ServerArgs; use crate::data_folders::DataFolders; - use crate::{ClusterMode, ServerArgs}; // Tests for Cluster. #[tokio::test] diff --git a/crates/modelardb_server/src/context.rs b/crates/modelardb_server/src/context.rs index 9048f6bb..099a7793 100644 --- a/crates/modelardb_server/src/context.rs +++ b/crates/modelardb_server/src/context.rs @@ -24,11 +24,12 @@ use modelardb_types::types::TimeSeriesTableMetadata; use tokio::sync::RwLock; use tracing::{info, warn}; +use crate::cluster::ClusterMode; use crate::configuration::{ConfigurationManager, WalMode}; use crate::error::{ModelarDbServerError, Result}; use crate::storage::StorageEngine; use crate::storage::data_sinks::{NormalTableDataSink, TimeSeriesTableDataSink}; -use crate::{ClusterMode, DataFolders, ServerArgs}; +use crate::{DataFolders, ServerArgs}; /// Provides access to the system's configuration and components. pub struct Context { diff --git a/crates/modelardb_server/src/data_folders.rs b/crates/modelardb_server/src/data_folders.rs index f0098e02..873d3c1a 100644 --- a/crates/modelardb_server/src/data_folders.rs +++ b/crates/modelardb_server/src/data_folders.rs @@ -18,8 +18,8 @@ use modelardb_storage::data_folder::DataFolder; use modelardb_types::types::{Node, ServerMode}; -use crate::cluster::Cluster; -use crate::{ClusterMode, Result, ServerMode as ServerModeArg}; +use crate::cluster::{Cluster, ClusterMode}; +use crate::{Result, ServerMode as ServerModeArg}; /// Folders for storing metadata and data in Apache Parquet files locally and remotely. #[derive(Clone)] diff --git a/crates/modelardb_server/src/remote/mod.rs b/crates/modelardb_server/src/remote/mod.rs index 52d31962..d5a2821b 100644 --- a/crates/modelardb_server/src/remote/mod.rs +++ b/crates/modelardb_server/src/remote/mod.rs @@ -60,7 +60,7 @@ use tonic::transport::{Endpoint, Server}; use tonic::{Request, Response, Status, Streaming}; use tracing::{debug, error, info}; -use crate::ClusterMode; +use crate::cluster::ClusterMode; use crate::context::Context; use crate::error::{ModelarDbServerError, Result}; use crate::remote::auth_layer::AuthLayer; From 6ad47b630d66efaad0af3187fc75d131f2516f01 Mon Sep 17 00:00:00 2001 From: CGodiksen <36046286+CGodiksen@users.noreply.github.com> Date: Tue, 11 Aug 2026 20:39:50 +0200 Subject: [PATCH 15/33] Move collection of metrics out of do_action and into Context --- crates/modelardb_server/src/context.rs | 85 +++++++++++++++++++++++ crates/modelardb_server/src/remote/mod.rs | 81 +-------------------- 2 files changed, 86 insertions(+), 80 deletions(-) diff --git a/crates/modelardb_server/src/context.rs b/crates/modelardb_server/src/context.rs index 099a7793..0da2d369 100644 --- a/crates/modelardb_server/src/context.rs +++ b/crates/modelardb_server/src/context.rs @@ -20,7 +20,9 @@ use std::sync::Arc; use datafusion::arrow::datatypes::Schema; use datafusion::catalog::SchemaProvider; +use modelardb_types::flight::protocol; use modelardb_types::types::TimeSeriesTableMetadata; +use sysinfo::{Disks, System}; use tokio::sync::RwLock; use tracing::{info, warn}; @@ -491,6 +493,89 @@ impl Context { Ok(schema) } + + /// Collect the current resource usage metrics of the node, including CPU, memory, disk, and + /// storage engine memory usage. + pub(crate) async fn node_metrics(&self) -> protocol::NodeMetrics { + let mut system = System::new(); + + // Sample the CPU twice, separated by the minimum update interval, since a single refresh + // reads zero. + system.refresh_cpu_usage(); + tokio::time::sleep(sysinfo::MINIMUM_CPU_UPDATE_INTERVAL).await; + system.refresh_cpu_usage(); + + let cpu_usage_percentage = system.global_cpu_usage() as f64; + let cpu_count = system.cpus().len() as u32; + + system.refresh_memory(); + let used_memory_in_bytes = system.used_memory(); + let total_memory_in_bytes = system.total_memory(); + + let (used_disk_space_in_bytes, total_disk_space_in_bytes) = + self.local_data_folder_disk_space(); + + let configuration_manager = self.configuration_manager.read().await; + let storage_engine = self.storage_engine.read().await; + + let ingested_reserved_memory_in_bytes = + configuration_manager.ingested_reserved_memory_in_bytes(); + let uncompressed_reserved_memory_in_bytes = + configuration_manager.uncompressed_reserved_memory_in_bytes(); + let compressed_reserved_memory_in_bytes = + configuration_manager.compressed_reserved_memory_in_bytes(); + + // The storage engine only tracks how much reserved memory remains, and that value can go + // negative when it is temporarily over budget, so the used memory is clamped to zero. + let ingested_used_memory_in_bytes = (ingested_reserved_memory_in_bytes as i64 + - storage_engine.remaining_ingested_memory_in_bytes()) + .max(0) as u64; + let uncompressed_used_memory_in_bytes = (uncompressed_reserved_memory_in_bytes as i64 + - storage_engine.remaining_uncompressed_memory_in_bytes()) + .max(0) as u64; + let compressed_used_memory_in_bytes = (compressed_reserved_memory_in_bytes as i64 + - storage_engine.remaining_compressed_memory_in_bytes()) + .max(0) as u64; + + protocol::NodeMetrics { + cpu_usage_percentage, + cpu_count, + used_memory_in_bytes, + total_memory_in_bytes, + used_disk_space_in_bytes, + total_disk_space_in_bytes, + ingested_used_memory_in_bytes, + ingested_reserved_memory_in_bytes, + uncompressed_used_memory_in_bytes, + uncompressed_reserved_memory_in_bytes, + compressed_used_memory_in_bytes, + compressed_reserved_memory_in_bytes, + } + } + + /// Return the used and total disk space in bytes for the disk holding the local data folder. + /// The disk is identified by finding the mounted disk whose mount point is the longest prefix + /// of the local data folder path. If no disk matches, e.g., because the data folder is in + /// memory, the largest-capacity disk is used instead. If no disks are found, `(0, 0)` is + /// returned. + fn local_data_folder_disk_space(&self) -> (u64, u64) { + let disks = Disks::new_with_refreshed_list(); + let location = self.data_folders.local_data_folder.location(); + + let maybe_disk = disks + .iter() + .filter(|disk| location.starts_with(&*disk.mount_point().to_string_lossy())) + .max_by_key(|disk| disk.mount_point().as_os_str().len()) + .or_else(|| disks.iter().max_by_key(|disk| disk.total_space())); + + if let Some(disk) = maybe_disk { + let total = disk.total_space(); + let used = total.saturating_sub(disk.available_space()); + (used, total) + } else { + (0, 0) + } + } } /// Return a [`ModelarDbServerError`] indicating that a table with `table_name` does not exist. diff --git a/crates/modelardb_server/src/remote/mod.rs b/crates/modelardb_server/src/remote/mod.rs index d5a2821b..65c776d4 100644 --- a/crates/modelardb_server/src/remote/mod.rs +++ b/crates/modelardb_server/src/remote/mod.rs @@ -51,7 +51,6 @@ use modelardb_types::flight::protocol; use modelardb_types::functions; use modelardb_types::types::{ServerMode, Table, TimeSeriesTableMetadata}; use prost::Message; -use sysinfo::{Disks, System}; use tokio::sync::mpsc::{self, Sender}; use tokio::task; use tokio_stream::wrappers::ReceiverStream; @@ -292,29 +291,6 @@ fn empty_record_batch_stream() -> SendableRecordBatchStream { Box::pin(EmptyRecordBatchStream::new(Arc::new(Schema::empty()))) } -/// Return the used and total disk space in bytes for the disk holding the local data folder. The -/// disk is identified by finding the mounted disk whose mount point is the longest prefix of the -/// local data folder path. If no disk matches, e.g., because the data folder is in memory, the -/// largest-capacity disk is used instead. If no disks are found, `(0, 0)` is returned. -fn local_data_folder_disk_space(context: &Context) -> (u64, u64) { - let disks = Disks::new_with_refreshed_list(); - let location = context.data_folders.local_data_folder.location(); - - let maybe_disk = disks - .iter() - .filter(|disk| location.starts_with(&*disk.mount_point().to_string_lossy())) - .max_by_key(|disk| disk.mount_point().as_os_str().len()) - .or_else(|| disks.iter().max_by_key(|disk| disk.total_space())); - - if let Some(disk) = maybe_disk { - let total = disk.total_space(); - let used = total.saturating_sub(disk.available_space()); - (used, total) - } else { - (0, 0) - } -} - /// Convert an `error` to a [`Status`] with [`tonic::Code::InvalidArgument`] as the code and `error` /// converted to a [`String`] as the message. pub fn error_to_status_invalid_argument(error: impl Error) -> Status { @@ -1120,62 +1096,7 @@ impl FlightService for FlightServiceHandler { }) })))) } else if action.r#type == "NodeMetrics" { - let mut system = System::new(); - - // Sample the CPU twice, separated by the minimum update interval, since a single - // refresh reads zero. - system.refresh_cpu_usage(); - tokio::time::sleep(sysinfo::MINIMUM_CPU_UPDATE_INTERVAL).await; - system.refresh_cpu_usage(); - - let cpu_usage_percentage = system.global_cpu_usage() as f64; - let cpu_count = system.cpus().len() as u32; - - system.refresh_memory(); - let used_memory_in_bytes = system.used_memory(); - let total_memory_in_bytes = system.total_memory(); - - let (used_disk_space_in_bytes, total_disk_space_in_bytes) = - local_data_folder_disk_space(&self.context); - - // Pair each reserved value from the configuration with the remaining value from the - // storage engine to compute how much of the reservation is currently in use. - let configuration_manager = self.context.configuration_manager.read().await; - let storage_engine = self.context.storage_engine.read().await; - - let ingested_reserved_memory_in_bytes = - configuration_manager.ingested_reserved_memory_in_bytes(); - let uncompressed_reserved_memory_in_bytes = - configuration_manager.uncompressed_reserved_memory_in_bytes(); - let compressed_reserved_memory_in_bytes = - configuration_manager.compressed_reserved_memory_in_bytes(); - - let ingested_used_memory_in_bytes = (ingested_reserved_memory_in_bytes as i64 - - storage_engine.remaining_ingested_memory_in_bytes()) - .max(0) as u64; - let uncompressed_used_memory_in_bytes = (uncompressed_reserved_memory_in_bytes as i64 - - storage_engine.remaining_uncompressed_memory_in_bytes()) - .max(0) as u64; - let compressed_used_memory_in_bytes = (compressed_reserved_memory_in_bytes as i64 - - storage_engine.remaining_compressed_memory_in_bytes()) - .max(0) as u64; - - let node_metrics = protocol::NodeMetrics { - cpu_usage_percentage, - cpu_count, - used_memory_in_bytes, - total_memory_in_bytes, - used_disk_space_in_bytes, - total_disk_space_in_bytes, - ingested_used_memory_in_bytes, - ingested_reserved_memory_in_bytes, - uncompressed_used_memory_in_bytes, - uncompressed_reserved_memory_in_bytes, - compressed_used_memory_in_bytes, - compressed_reserved_memory_in_bytes, - }; - - let protobuf_bytes = node_metrics.encode_to_vec(); + let protobuf_bytes = self.context.node_metrics().await.encode_to_vec(); Ok(Response::new(Box::pin(stream::once(async { Ok(FlightResult { From f19e3b9dfd2624dd77d112bd91a65ed65df9bc2c Mon Sep 17 00:00:00 2001 From: CGodiksen <36046286+CGodiksen@users.noreply.github.com> Date: Tue, 11 Aug 2026 20:53:12 +0200 Subject: [PATCH 16/33] Remove misleading comment --- crates/modelardb_server/src/context.rs | 2 -- 1 file changed, 2 deletions(-) diff --git a/crates/modelardb_server/src/context.rs b/crates/modelardb_server/src/context.rs index 0da2d369..31328ad3 100644 --- a/crates/modelardb_server/src/context.rs +++ b/crates/modelardb_server/src/context.rs @@ -525,8 +525,6 @@ impl Context { let compressed_reserved_memory_in_bytes = configuration_manager.compressed_reserved_memory_in_bytes(); - // The storage engine only tracks how much reserved memory remains, and that value can go - // negative when it is temporarily over budget, so the used memory is clamped to zero. let ingested_used_memory_in_bytes = (ingested_reserved_memory_in_bytes as i64 - storage_engine.remaining_ingested_memory_in_bytes()) .max(0) as u64; From 972ebf75b663844792bd7d22d469df8c6bb7234e Mon Sep 17 00:00:00 2001 From: CGodiksen <36046286+CGodiksen@users.noreply.github.com> Date: Tue, 11 Aug 2026 21:02:21 +0200 Subject: [PATCH 17/33] Fix integration test after adding new actions --- crates/modelardb_server/src/configuration.rs | 3 ++- crates/modelardb_server/tests/integration_test.rs | 2 ++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/crates/modelardb_server/src/configuration.rs b/crates/modelardb_server/src/configuration.rs index 289c6910..524e4ff5 100644 --- a/crates/modelardb_server/src/configuration.rs +++ b/crates/modelardb_server/src/configuration.rs @@ -34,9 +34,10 @@ use prost::Message; use serde::{Deserialize, Serialize}; use tokio::sync::RwLock; +use crate::ServerArgs; +use crate::cluster::ClusterMode; use crate::error::{ModelarDbServerError, Result}; use crate::storage::StorageEngine; -use crate::{ClusterMode, ServerArgs}; const CONFIGURATION_FILE_NAME: &str = "modelardbd.toml"; diff --git a/crates/modelardb_server/tests/integration_test.rs b/crates/modelardb_server/tests/integration_test.rs index ea5dad15..fcd8f417 100644 --- a/crates/modelardb_server/tests/integration_test.rs +++ b/crates/modelardb_server/tests/integration_test.rs @@ -988,6 +988,8 @@ async fn test_can_list_actions() { "FlushNode", "GetConfiguration", "KillNode", + "ListNodes", + "NodeMetrics", "NodeType", "UpdateConfiguration", ] From 8861c260f42f4e9a03016946deea55d47bdd4a49 Mon Sep 17 00:00:00 2001 From: CGodiksen <36046286+CGodiksen@users.noreply.github.com> Date: Wed, 12 Aug 2026 06:43:25 +0200 Subject: [PATCH 18/33] Add prost decode error to modelardb_embedded --- Cargo.lock | 1 + crates/modelardb_embedded/Cargo.toml | 1 + crates/modelardb_embedded/src/error.rs | 11 +++++++++++ 3 files changed, 13 insertions(+) diff --git a/Cargo.lock b/Cargo.lock index 4fedc9ef..832c23c9 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3558,6 +3558,7 @@ dependencies = [ "modelardb_compression", "modelardb_storage", "modelardb_types", + "prost", "tempfile", "tokio", "tonic", diff --git a/crates/modelardb_embedded/Cargo.toml b/crates/modelardb_embedded/Cargo.toml index cd33b310..f376fb18 100644 --- a/crates/modelardb_embedded/Cargo.toml +++ b/crates/modelardb_embedded/Cargo.toml @@ -35,6 +35,7 @@ modelardb_auth.workspace = true modelardb_compression.workspace = true modelardb_storage.workspace = true modelardb_types.workspace = true +prost.workspace = true tokio.workspace = true tonic.workspace = true diff --git a/crates/modelardb_embedded/src/error.rs b/crates/modelardb_embedded/src/error.rs index e3d1c78e..9fd0db40 100644 --- a/crates/modelardb_embedded/src/error.rs +++ b/crates/modelardb_embedded/src/error.rs @@ -29,6 +29,7 @@ use deltalake::{DeltaTableError, ObjectStoreError}; use modelardb_compression::error::ModelarDbCompressionError; use modelardb_storage::error::ModelarDbStorageError; use modelardb_types::error::ModelarDbTypesError; +use prost::DecodeError; use tonic::Status as TonicStatusError; use tonic::transport::Error as TonicTransportError; @@ -58,6 +59,8 @@ pub enum ModelarDbEmbeddedError { ObjectStore(ObjectStoreError), /// Error returned by Apache Parquet. Parquet(ParquetError), + /// Error returned by Prost when decoding a message that is not valid. + ProstDecode(DecodeError), /// Status returned by Tonic. TonicStatus(Box), /// Error returned by Tonic. @@ -83,6 +86,7 @@ impl Display for ModelarDbEmbeddedError { Self::ModelarDbTypes(reason) => write!(f, "ModelarDB Types Error: {reason}"), Self::ObjectStore(reason) => write!(f, "Object Store Error: {reason}"), Self::Parquet(reason) => write!(f, "Parquet Error: {reason}"), + Self::ProstDecode(reason) => write!(f, "Prost Decode Error: {reason}"), Self::TonicStatus(reason) => write!(f, "Tonic Status Error: {reason}"), Self::TonicTransport(reason) => write!(f, "Tonic Transport Error: {reason}"), Self::Unimplemented(reason) => write!(f, "Unimplemented Error: {reason}"), @@ -104,6 +108,7 @@ impl Error for ModelarDbEmbeddedError { Self::ModelarDbTypes(reason) => Some(reason), Self::ObjectStore(reason) => Some(reason), Self::Parquet(reason) => Some(reason), + Self::ProstDecode(reason) => Some(reason), Self::TonicStatus(reason) => Some(reason), Self::TonicTransport(reason) => Some(reason), Self::Unimplemented(_reason) => None, @@ -166,6 +171,12 @@ impl From for ModelarDbEmbeddedError { } } +impl From for ModelarDbEmbeddedError { + fn from(error: DecodeError) -> Self { + Self::ProstDecode(error) + } +} + impl From for ModelarDbEmbeddedError { fn from(error: TonicStatusError) -> Self { Self::TonicStatus(Box::new(error)) From 50e227b9b3f724a9f0f7bbf9149f78d9229d15ea Mon Sep 17 00:00:00 2001 From: CGodiksen <36046286+CGodiksen@users.noreply.github.com> Date: Wed, 12 Aug 2026 06:49:49 +0200 Subject: [PATCH 19/33] Add a method to deserialize and extract cluster nodes from protobuf --- crates/modelardb_types/src/flight/mod.rs | 46 +++++++++++++++--------- 1 file changed, 30 insertions(+), 16 deletions(-) diff --git a/crates/modelardb_types/src/flight/mod.rs b/crates/modelardb_types/src/flight/mod.rs index a3ad3406..58a6541d 100644 --- a/crates/modelardb_types/src/flight/mod.rs +++ b/crates/modelardb_types/src/flight/mod.rs @@ -17,6 +17,7 @@ //! defined in `flight/protocol.proto`. The module also provides functions to serialize and //! deserialize encoded messages to and from bytes. +use std::str::FromStr; use std::sync::Arc; use arrow::datatypes::Schema; @@ -28,7 +29,7 @@ use prost::bytes::Bytes; use crate::error::{ModelarDbTypesError, Result}; use crate::functions::{try_convert_bytes_to_schema, try_convert_schema_to_bytes}; -use crate::types::{ErrorBound, GeneratedColumn, Node, Table, TimeSeriesTableMetadata}; +use crate::types::{ErrorBound, GeneratedColumn, Node, ServerMode, Table, TimeSeriesTableMetadata}; pub mod protocol { include!(concat!(env!("OUT_DIR"), "/modelardb.flight.protocol.rs")); @@ -194,21 +195,6 @@ fn decode_error_bounds( Ok(error_bounds) } -/// Encode `nodes` into a [`ClusterNodes`](protocol::ClusterNodes) protobuf message and serialize it. -pub fn encode_and_serialize_cluster_nodes(nodes: Vec) -> Vec { - let cluster_nodes = protocol::ClusterNodes { - nodes: nodes - .into_iter() - .map(|node| protocol::NodeMetadata { - url: node.url, - mode: node.mode.to_string(), - }) - .collect(), - }; - - cluster_nodes.encode_to_vec() -} - /// Decode the generated column expressions from a vector of byte expressions into a vector of /// optional [`GeneratedColumn`]. Return [`ModelarDbTypesError`] if the expression is invalid. fn decode_generated_column_expressions( @@ -230,6 +216,34 @@ fn decode_generated_column_expressions( Ok(expressions) } +/// Encode `nodes` into a [`ClusterNodes`](protocol::ClusterNodes) protobuf message and serialize it. +pub fn encode_and_serialize_cluster_nodes(nodes: Vec) -> Vec { + let cluster_nodes = protocol::ClusterNodes { + nodes: nodes + .into_iter() + .map(|node| protocol::NodeMetadata { + url: node.url, + mode: node.mode.to_string(), + }) + .collect(), + }; + + cluster_nodes.encode_to_vec() +} + +/// Deserialize `bytes` into a [`ClusterNodes`](protocol::ClusterNodes) protobuf message and extract +/// a vector of [`Node`]. If `bytes` cannot be deserialized or a node has an invalid mode, return +/// [`ModelarDbTypesError`]. +pub fn deserialize_and_extract_cluster_nodes(bytes: &[u8]) -> Result> { + let cluster_nodes = protocol::ClusterNodes::decode(bytes)?; + + cluster_nodes + .nodes + .into_iter() + .map(|node| Ok(Node::new(node.url, ServerMode::from_str(&node.mode)?))) + .collect() +} + #[cfg(test)] mod test { use super::*; From 259e0d9e57bab318874a19188f59893cd9e7f4a3 Mon Sep 17 00:00:00 2001 From: CGodiksen <36046286+CGodiksen@users.noreply.github.com> Date: Wed, 12 Aug 2026 06:53:24 +0200 Subject: [PATCH 20/33] Add test for serializing and deserializing nodes --- crates/modelardb_types/src/flight/mod.rs | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/crates/modelardb_types/src/flight/mod.rs b/crates/modelardb_types/src/flight/mod.rs index 58a6541d..96bf4808 100644 --- a/crates/modelardb_types/src/flight/mod.rs +++ b/crates/modelardb_types/src/flight/mod.rs @@ -288,4 +288,18 @@ mod test { _ => panic!("Expected time series table."), } } + + // Test for serializing and deserializing cluster nodes. + #[test] + fn test_serialize_and_deserialize_cluster_nodes() { + let expected_nodes = vec![ + Node::new("grpc://127.0.0.1:9999".to_owned(), ServerMode::Edge), + Node::new("grpc://127.0.0.1:9998".to_owned(), ServerMode::Cloud), + ]; + + let bytes = encode_and_serialize_cluster_nodes(expected_nodes.clone()); + let nodes = deserialize_and_extract_cluster_nodes(&bytes).unwrap(); + + assert_eq!(nodes, expected_nodes); + } } From c4ac14b2c92f51d7870b418952f1a686c5a30503 Mon Sep 17 00:00:00 2001 From: CGodiksen <36046286+CGodiksen@users.noreply.github.com> Date: Wed, 12 Aug 2026 11:27:36 +0200 Subject: [PATCH 21/33] Add helper methods to run actions on Client --- .../src/operations/client.rs | 34 +++++++++++++++++++ 1 file changed, 34 insertions(+) diff --git a/crates/modelardb_embedded/src/operations/client.rs b/crates/modelardb_embedded/src/operations/client.rs index 4456d067..4b07ac53 100644 --- a/crates/modelardb_embedded/src/operations/client.rs +++ b/crates/modelardb_embedded/src/operations/client.rs @@ -35,6 +35,7 @@ use datafusion::execution::RecordBatchStream; use datafusion::physical_plan::stream::RecordBatchStreamAdapter; use futures::{StreamExt, TryStreamExt, stream}; use modelardb_auth::BearerInterceptor; +use prost::bytes::Bytes; use tonic::codegen::InterceptedService; use tonic::transport::{Channel, Endpoint}; use tonic::{Request, Status}; @@ -65,6 +66,39 @@ impl Client { Ok(Client { flight_client }) } + + /// Send the action with the type `action_type` and an empty body to the node. If the action + /// could not be performed, [`ModelarDbEmbeddedError`] is returned. + async fn run_action(&mut self, action_type: &str) -> Result<()> { + let action = Action { + r#type: action_type.to_owned(), + body: vec![].into(), + }; + + self.flight_client.do_action(Request::new(action)).await?; + + Ok(()) + } + + /// Send the action with the type `action_type` and an empty body to the node and return the + /// body of the response. If the action could not be performed, [`ModelarDbEmbeddedError`] is + /// returned. + async fn retrieve_action_bytes(&mut self, action_type: &str) -> Result { + let action = Action { + r#type: action_type.to_owned(), + body: vec![].into(), + }; + + let response = self.flight_client.do_action(Request::new(action)).await?; + + let message = response + .into_inner() + .message() + .await? + .expect("Flight message should exist."); + + Ok(message.body) + } } #[async_trait] From 8d574053b4ab406075e24651d48a20b423bd357e Mon Sep 17 00:00:00 2001 From: CGodiksen <36046286+CGodiksen@users.noreply.github.com> Date: Wed, 12 Aug 2026 11:44:52 +0200 Subject: [PATCH 22/33] Refactor action helper so they can be used in modelardb_type and create --- .../src/operations/client.rs | 51 ++++++------------- 1 file changed, 16 insertions(+), 35 deletions(-) diff --git a/crates/modelardb_embedded/src/operations/client.rs b/crates/modelardb_embedded/src/operations/client.rs index 4b07ac53..a2ee448b 100644 --- a/crates/modelardb_embedded/src/operations/client.rs +++ b/crates/modelardb_embedded/src/operations/client.rs @@ -28,7 +28,7 @@ use arrow::record_batch::RecordBatch; use arrow_flight::decode::FlightRecordBatchStream; use arrow_flight::encode::FlightDataEncoderBuilder; use arrow_flight::flight_service_client::FlightServiceClient; -use arrow_flight::{Action, Criteria, FlightDescriptor, Ticket}; +use arrow_flight::{Action, Criteria, FlightDescriptor, Result as FlightResult, Ticket}; use async_trait::async_trait; use datafusion::error::DataFusionError; use datafusion::execution::RecordBatchStream; @@ -38,7 +38,7 @@ use modelardb_auth::BearerInterceptor; use prost::bytes::Bytes; use tonic::codegen::InterceptedService; use tonic::transport::{Channel, Endpoint}; -use tonic::{Request, Status}; +use tonic::{Request, Status, Streaming}; use crate::error::{ModelarDbEmbeddedError, Result}; use crate::operations::{ @@ -67,32 +67,30 @@ impl Client { Ok(Client { flight_client }) } - /// Send the action with the type `action_type` and an empty body to the node. If the action - /// could not be performed, [`ModelarDbEmbeddedError`] is returned. - async fn run_action(&mut self, action_type: &str) -> Result<()> { + /// Send the action with the type `action_type` and `body` to the node and return the response + /// stream. If the action could not be performed, [`ModelarDbEmbeddedError`] is returned. + async fn send_action( + &mut self, + action_type: &str, + body: Vec, + ) -> Result> { let action = Action { r#type: action_type.to_owned(), - body: vec![].into(), + body: body.into(), }; - self.flight_client.do_action(Request::new(action)).await?; + let response = self.flight_client.do_action(Request::new(action)).await?; - Ok(()) + Ok(response.into_inner()) } /// Send the action with the type `action_type` and an empty body to the node and return the /// body of the response. If the action could not be performed, [`ModelarDbEmbeddedError`] is /// returned. async fn retrieve_action_bytes(&mut self, action_type: &str) -> Result { - let action = Action { - r#type: action_type.to_owned(), - body: vec![].into(), - }; - - let response = self.flight_client.do_action(Request::new(action)).await?; + let mut response = self.send_action(action_type, vec![]).await?; let message = response - .into_inner() .message() .await? .expect("Flight message should exist."); @@ -110,21 +108,9 @@ impl Operations for Client { /// Returns the type of the ModelarDB node that the client is connected to. async fn modelardb_type(&mut self) -> Result { - // Retrieve the node type from the ModelarDB node. - let action = Action { - r#type: "NodeType".to_owned(), - body: vec![].into(), - }; + let bytes = self.retrieve_action_bytes("NodeType").await?; - let response = self.flight_client.do_action(Request::new(action)).await?; - - let message = response - .into_inner() - .message() - .await? - .expect("Flight message should exist."); - - ModelarDBType::from_str(str::from_utf8(&message.body)?) + ModelarDBType::from_str(str::from_utf8(&bytes)?) } /// Creates a table with the name in `table_name` and the information in `table_type`. If the @@ -152,12 +138,7 @@ impl Operations for Client { } }; - let action = Action { - r#type: "CreateTable".to_owned(), - body: protobuf_bytes.into(), - }; - - self.flight_client.do_action(action).await?; + self.send_action("CreateTable", protobuf_bytes).await?; Ok(()) } From c40238360700302b9011132b09847c6a8818371d Mon Sep 17 00:00:00 2001 From: CGodiksen <36046286+CGodiksen@users.noreply.github.com> Date: Wed, 12 Aug 2026 11:52:10 +0200 Subject: [PATCH 23/33] Add basic node operations to Client struct --- .../src/operations/client.rs | 73 +++++++++++++++++++ 1 file changed, 73 insertions(+) diff --git a/crates/modelardb_embedded/src/operations/client.rs b/crates/modelardb_embedded/src/operations/client.rs index a2ee448b..a652f312 100644 --- a/crates/modelardb_embedded/src/operations/client.rs +++ b/crates/modelardb_embedded/src/operations/client.rs @@ -35,6 +35,9 @@ use datafusion::execution::RecordBatchStream; use datafusion::physical_plan::stream::RecordBatchStreamAdapter; use futures::{StreamExt, TryStreamExt, stream}; use modelardb_auth::BearerInterceptor; +use modelardb_types::flight::protocol; +use modelardb_types::types::Node; +use prost::Message; use prost::bytes::Bytes; use tonic::codegen::InterceptedService; use tonic::transport::{Channel, Endpoint}; @@ -67,6 +70,76 @@ impl Client { Ok(Client { flight_client }) } + /// Returns the current configuration of the node. If the configuration could not be retrieved, + /// [`ModelarDbEmbeddedError`] is returned. + pub async fn configuration(&mut self) -> Result { + let bytes = self.retrieve_action_bytes("GetConfiguration").await?; + + Ok(protocol::Configuration::decode(bytes)?) + } + + /// Updates `setting` in the node configuration to `new_value`. If the setting could not be + /// updated, [`ModelarDbEmbeddedError`] is returned. + pub async fn update_configuration( + &mut self, + setting: protocol::update_configuration::Setting, + new_value: Option, + ) -> Result<()> { + let update_configuration = protocol::UpdateConfiguration { + setting: setting as i32, + new_value, + }; + + self.send_action("UpdateConfiguration", update_configuration.encode_to_vec()) + .await?; + + Ok(()) + } + + /// Flushes all data currently in memory to disk. If the data could not be flushed, + /// [`ModelarDbEmbeddedError`] is returned. + pub async fn flush_memory(&mut self) -> Result<()> { + self.send_action("FlushMemory", vec![]).await?; + + Ok(()) + } + + /// Flushes all data in memory to disk and then transfers all compressed data to the remote + /// object store. If the data could not be flushed, [`ModelarDbEmbeddedError`] is returned. + pub async fn flush_node(&mut self) -> Result<()> { + self.send_action("FlushNode", vec![]).await?; + + Ok(()) + } + + /// Flushes all data to disk, transfers it to the remote object store, removes the node from the + /// cluster if necessary, and kills the node process. Since the process is killed, a + /// conventional response cannot be returned, so a dropped connection is not treated as an + /// error. If the data could not be flushed before the node was killed, + /// [`ModelarDbEmbeddedError`] is returned. + pub async fn kill_node(&mut self) -> Result<()> { + // The node exits while handling this action, so the response stream is dropped. + let _ = self.send_action("KillNode", vec![]).await; + + Ok(()) + } + + /// Returns the nodes that are currently part of the cluster. A single node returns only itself. + /// If the nodes could not be retrieved, [`ModelarDbEmbeddedError`] is returned. + pub async fn list_nodes(&mut self) -> Result> { + let bytes = self.retrieve_action_bytes("ListNodes").await?; + + Ok(modelardb_types::flight::deserialize_and_extract_cluster_nodes(&bytes)?) + } + + /// Returns the current resource usage metrics of the node. If the metrics could not be + /// retrieved, [`ModelarDbEmbeddedError`] is returned. + pub async fn node_metrics(&mut self) -> Result { + let bytes = self.retrieve_action_bytes("NodeMetrics").await?; + + Ok(protocol::NodeMetrics::decode(bytes)?) + } + /// Send the action with the type `action_type` and `body` to the node and return the response /// stream. If the action could not be performed, [`ModelarDbEmbeddedError`] is returned. async fn send_action( From 2e173f03f4916eee0c85a4111d2ad119c8091db9 Mon Sep 17 00:00:00 2001 From: CGodiksen <36046286+CGodiksen@users.noreply.github.com> Date: Wed, 12 Aug 2026 11:53:56 +0200 Subject: [PATCH 24/33] Change order of helper methods for consistency --- .../src/operations/client.rs | 28 +++++++++---------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/crates/modelardb_embedded/src/operations/client.rs b/crates/modelardb_embedded/src/operations/client.rs index a652f312..3cd3bf79 100644 --- a/crates/modelardb_embedded/src/operations/client.rs +++ b/crates/modelardb_embedded/src/operations/client.rs @@ -140,6 +140,20 @@ impl Client { Ok(protocol::NodeMetrics::decode(bytes)?) } + /// Send the action with the type `action_type` and an empty body to the node and return the + /// body of the response. If the action could not be performed, [`ModelarDbEmbeddedError`] is + /// returned. + async fn retrieve_action_bytes(&mut self, action_type: &str) -> Result { + let mut response = self.send_action(action_type, vec![]).await?; + + let message = response + .message() + .await? + .expect("Flight message should exist."); + + Ok(message.body) + } + /// Send the action with the type `action_type` and `body` to the node and return the response /// stream. If the action could not be performed, [`ModelarDbEmbeddedError`] is returned. async fn send_action( @@ -156,20 +170,6 @@ impl Client { Ok(response.into_inner()) } - - /// Send the action with the type `action_type` and an empty body to the node and return the - /// body of the response. If the action could not be performed, [`ModelarDbEmbeddedError`] is - /// returned. - async fn retrieve_action_bytes(&mut self, action_type: &str) -> Result { - let mut response = self.send_action(action_type, vec![]).await?; - - let message = response - .message() - .await? - .expect("Flight message should exist."); - - Ok(message.body) - } } #[async_trait] From 1735212cf49bcc8492d5c75b3e542f8db16b4717 Mon Sep 17 00:00:00 2001 From: CGodiksen <36046286+CGodiksen@users.noreply.github.com> Date: Wed, 12 Aug 2026 11:57:15 +0200 Subject: [PATCH 25/33] Add a method that uses get_flight_info to get the cloud query node --- .../src/operations/client.rs | 26 +++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/crates/modelardb_embedded/src/operations/client.rs b/crates/modelardb_embedded/src/operations/client.rs index 3cd3bf79..73234c45 100644 --- a/crates/modelardb_embedded/src/operations/client.rs +++ b/crates/modelardb_embedded/src/operations/client.rs @@ -170,6 +170,32 @@ impl Client { Ok(response.into_inner()) } + + /// Returns the URL of the cloud node that the node assigns to execute the SQL in `sql`. If the + /// node is not running in a cluster, or a cloud node could not be assigned, + /// [`ModelarDbEmbeddedError`] is returned. + pub async fn cloud_query_node(&mut self, sql: &str) -> Result { + let flight_descriptor = FlightDescriptor::new_cmd(sql.to_owned()); + let flight_info = self + .flight_client + .get_flight_info(Request::new(flight_descriptor)) + .await? + .into_inner(); + + let endpoint = flight_info.endpoint.into_iter().next().ok_or_else(|| { + ModelarDbEmbeddedError::InvalidArgument( + "The node did not return an endpoint for the query.".to_owned(), + ) + })?; + + let location = endpoint.location.into_iter().next().ok_or_else(|| { + ModelarDbEmbeddedError::InvalidArgument( + "The endpoint did not return a cloud node location for the query.".to_owned(), + ) + })?; + + Ok(location.uri) + } } #[async_trait] From 7ddb124962caa7fd41ad747bb86f32da089086df Mon Sep 17 00:00:00 2001 From: CGodiksen <36046286+CGodiksen@users.noreply.github.com> Date: Wed, 12 Aug 2026 12:20:18 +0200 Subject: [PATCH 26/33] Add integration test for list nodes action --- .../modelardb_server/tests/integration_test.rs | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/crates/modelardb_server/tests/integration_test.rs b/crates/modelardb_server/tests/integration_test.rs index fcd8f417..5bacb342 100644 --- a/crates/modelardb_server/tests/integration_test.rs +++ b/crates/modelardb_server/tests/integration_test.rs @@ -1614,3 +1614,20 @@ async fn test_can_create_time_series_table_from_metadata() { let retrieved_table_names = test_context.retrieve_all_table_names().await.unwrap(); assert_eq!(retrieved_table_names[0], TIME_SERIES_TABLE_NAME); } + +#[tokio::test] +async fn test_can_list_nodes() { + let mut test_context = TestContext::new().await; + let nodes_bytes = test_context.retrieve_action_bytes("ListNodes").await; + let nodes = + modelardb_types::flight::deserialize_and_extract_cluster_nodes(&nodes_bytes).unwrap(); + + assert_eq!(nodes.len(), 1); + assert_eq!( + nodes[0], + Node::new( + format!("grpc://{HOST}:{}", test_context.port), + ServerMode::Edge + ) + ); +} From b890074d763698df1588a98cadf5c48efbf36327 Mon Sep 17 00:00:00 2001 From: CGodiksen <36046286+CGodiksen@users.noreply.github.com> Date: Wed, 12 Aug 2026 12:20:29 +0200 Subject: [PATCH 27/33] Add integration test for node metrics action --- .../tests/integration_test.rs | 35 ++++++++++++++++++- 1 file changed, 34 insertions(+), 1 deletion(-) diff --git a/crates/modelardb_server/tests/integration_test.rs b/crates/modelardb_server/tests/integration_test.rs index 5bacb342..28bfb3c1 100644 --- a/crates/modelardb_server/tests/integration_test.rs +++ b/crates/modelardb_server/tests/integration_test.rs @@ -40,7 +40,7 @@ use futures::{StreamExt, stream}; use modelardb_test::data_generation; use modelardb_test::table::{self, NORMAL_TABLE_NAME, TIME_SERIES_TABLE_NAME}; use modelardb_types::flight::protocol; -use modelardb_types::types::ErrorBound; +use modelardb_types::types::{ErrorBound, Node, ServerMode}; use prost::Message; use tempfile::TempDir; use tokio::io::{AsyncBufReadExt, BufReader}; @@ -1631,3 +1631,36 @@ async fn test_can_list_nodes() { ) ); } + +#[tokio::test] +async fn test_can_get_node_metrics() { + let mut test_context = TestContext::new().await; + let metrics_bytes = test_context.retrieve_action_bytes("NodeMetrics").await; + let metrics = protocol::NodeMetrics::decode(metrics_bytes).unwrap(); + + // Only stable fields are asserted exactly. CPU usage, used memory, and disk usage vary per run + // and per machine. + assert!(metrics.cpu_usage_percentage > 0.0); + assert!(metrics.cpu_count > 0); + + assert!(metrics.used_memory_in_bytes > 0); + assert!(metrics.total_memory_in_bytes > 0); + + assert!(metrics.used_disk_space_in_bytes > 0); + assert!(metrics.total_disk_space_in_bytes > 0); + + assert_eq!(metrics.ingested_used_memory_in_bytes, 0); + assert_eq!(metrics.ingested_reserved_memory_in_bytes, 512 * 1024 * 1024); + + assert_eq!(metrics.uncompressed_used_memory_in_bytes, 0); + assert_eq!( + metrics.uncompressed_reserved_memory_in_bytes, + 512 * 1024 * 1024 + ); + + assert_eq!(metrics.compressed_used_memory_in_bytes, 0); + assert_eq!( + metrics.compressed_reserved_memory_in_bytes, + 512 * 1024 * 1024 + ); +} From d3a6235d026008c0e61e313603f389d7d1096168 Mon Sep 17 00:00:00 2001 From: CGodiksen <36046286+CGodiksen@users.noreply.github.com> Date: Wed, 12 Aug 2026 13:04:22 +0200 Subject: [PATCH 28/33] Simplify subtraction for used disk space --- crates/modelardb_server/src/context.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/modelardb_server/src/context.rs b/crates/modelardb_server/src/context.rs index 31328ad3..b9ed6117 100644 --- a/crates/modelardb_server/src/context.rs +++ b/crates/modelardb_server/src/context.rs @@ -568,7 +568,7 @@ impl Context { if let Some(disk) = maybe_disk { let total = disk.total_space(); - let used = total.saturating_sub(disk.available_space()); + let used = total - disk.available_space(); (used, total) } else { (0, 0) From 33d1bd8c0ee17dcb5a5affd11945ebc52279776c Mon Sep 17 00:00:00 2001 From: CGodiksen <36046286+CGodiksen@users.noreply.github.com> Date: Thu, 13 Aug 2026 10:32:53 +0200 Subject: [PATCH 29/33] Add better error handling in retrieve_action_bytes() --- crates/modelardb_embedded/src/operations/client.rs | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/crates/modelardb_embedded/src/operations/client.rs b/crates/modelardb_embedded/src/operations/client.rs index 73234c45..413be2bc 100644 --- a/crates/modelardb_embedded/src/operations/client.rs +++ b/crates/modelardb_embedded/src/operations/client.rs @@ -146,10 +146,11 @@ impl Client { async fn retrieve_action_bytes(&mut self, action_type: &str) -> Result { let mut response = self.send_action(action_type, vec![]).await?; - let message = response - .message() - .await? - .expect("Flight message should exist."); + let message = response.message().await?.ok_or_else(|| { + ModelarDbEmbeddedError::from(Status::internal(format!( + "Action '{action_type}' did not return a response message." + ))) + })?; Ok(message.body) } From 20d1376e29326932e92c361e20e27c222f31dda9 Mon Sep 17 00:00:00 2001 From: CGodiksen <36046286+CGodiksen@users.noreply.github.com> Date: Fri, 14 Aug 2026 08:05:35 +0200 Subject: [PATCH 30/33] Simplify local_data_folder_disk_space() based on comments from @skejserjensen --- crates/modelardb_server/src/context.rs | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/crates/modelardb_server/src/context.rs b/crates/modelardb_server/src/context.rs index b9ed6117..8c05df17 100644 --- a/crates/modelardb_server/src/context.rs +++ b/crates/modelardb_server/src/context.rs @@ -16,6 +16,7 @@ //! Implementation of a [`Context`] that provides access to the system's configuration and //! components. +use std::path::Path as StdPath; use std::sync::Arc; use datafusion::arrow::datatypes::Schema; @@ -551,18 +552,20 @@ impl Context { } } - /// Return the used and total disk space in bytes for the disk holding the local data folder. - /// The disk is identified by finding the mounted disk whose mount point is the longest prefix - /// of the local data folder path. If no disk matches, e.g., because the data folder is in - /// memory, the largest-capacity disk is used instead. If no disks are found, `(0, 0)` is - /// returned. + /// Return the used and total disk space in bytes of the disk holding the local data folder. If + /// no disk holds the data folder, the disk with the most capacity is used. If no disks are + /// found, `(0, 0)` is returned. fn local_data_folder_disk_space(&self) -> (u64, u64) { let disks = Disks::new_with_refreshed_list(); - let location = self.data_folders.local_data_folder.location(); + let location = StdPath::new(self.data_folders.local_data_folder.location()); + // A path can sit under multiple mount points when one volume is mounted inside another, so + // pick the disk whose mount point is the longest prefix of the location, as that is the + // most specific match. If the data folder is in memory, no mount point matches, so fall + // back to the disk with the most capacity. let maybe_disk = disks .iter() - .filter(|disk| location.starts_with(&*disk.mount_point().to_string_lossy())) + .filter(|disk| location.starts_with(disk.mount_point())) .max_by_key(|disk| disk.mount_point().as_os_str().len()) .or_else(|| disks.iter().max_by_key(|disk| disk.total_space())); From 0278ae171673589396596d770afd9d3c8d7b591b Mon Sep 17 00:00:00 2001 From: CGodiksen <36046286+CGodiksen@users.noreply.github.com> Date: Fri, 14 Aug 2026 08:10:11 +0200 Subject: [PATCH 31/33] Clamp the remaining memory to 0 instead of the used memory --- crates/modelardb_server/src/context.rs | 24 +++++++++++++----------- 1 file changed, 13 insertions(+), 11 deletions(-) diff --git a/crates/modelardb_server/src/context.rs b/crates/modelardb_server/src/context.rs index 8c05df17..8cb786ce 100644 --- a/crates/modelardb_server/src/context.rs +++ b/crates/modelardb_server/src/context.rs @@ -500,8 +500,8 @@ impl Context { pub(crate) async fn node_metrics(&self) -> protocol::NodeMetrics { let mut system = System::new(); - // Sample the CPU twice, separated by the minimum update interval, since a single refresh - // reads zero. + // Refresh the CPU usage twice, separated by the minimum update interval, since a single + // refresh results in global_cpu_usage() always returning 0. system.refresh_cpu_usage(); tokio::time::sleep(sysinfo::MINIMUM_CPU_UPDATE_INTERVAL).await; system.refresh_cpu_usage(); @@ -526,15 +526,17 @@ impl Context { let compressed_reserved_memory_in_bytes = configuration_manager.compressed_reserved_memory_in_bytes(); - let ingested_used_memory_in_bytes = (ingested_reserved_memory_in_bytes as i64 - - storage_engine.remaining_ingested_memory_in_bytes()) - .max(0) as u64; - let uncompressed_used_memory_in_bytes = (uncompressed_reserved_memory_in_bytes as i64 - - storage_engine.remaining_uncompressed_memory_in_bytes()) - .max(0) as u64; - let compressed_used_memory_in_bytes = (compressed_reserved_memory_in_bytes as i64 - - storage_engine.remaining_compressed_memory_in_bytes()) - .max(0) as u64; + // The used memory is the reserved memory minus the available remaining memory. The + // remaining memory can be negative when the reserved memory is decreased below what is + // currently in use, so treat a negative value as zero. + let ingested_used_memory_in_bytes = ingested_reserved_memory_in_bytes + - storage_engine.remaining_ingested_memory_in_bytes().max(0) as u64; + let uncompressed_used_memory_in_bytes = uncompressed_reserved_memory_in_bytes + - storage_engine + .remaining_uncompressed_memory_in_bytes() + .max(0) as u64; + let compressed_used_memory_in_bytes = compressed_reserved_memory_in_bytes + - storage_engine.remaining_compressed_memory_in_bytes().max(0) as u64; protocol::NodeMetrics { cpu_usage_percentage, From b0d6607c053ddc41b014b02ae89ed5409aa778e8 Mon Sep 17 00:00:00 2001 From: CGodiksen <36046286+CGodiksen@users.noreply.github.com> Date: Fri, 14 Aug 2026 08:13:33 +0200 Subject: [PATCH 32/33] Update based on comments from @skejserjensen --- crates/modelardb_server/src/cluster.rs | 4 ++-- crates/modelardb_server/src/configuration.rs | 2 +- crates/modelardb_server/src/data_folders.rs | 4 ++-- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/crates/modelardb_server/src/cluster.rs b/crates/modelardb_server/src/cluster.rs index 047409e6..5c68a955 100644 --- a/crates/modelardb_server/src/cluster.rs +++ b/crates/modelardb_server/src/cluster.rs @@ -41,7 +41,7 @@ use crate::error::{ModelarDbServerError, Result}; #[derive(Clone)] pub(crate) enum ClusterMode { SingleNode(Node), - MultiNode(Box), + MultiNode(Cluster), } impl ClusterMode { @@ -765,7 +765,7 @@ mod test { Some(cluster.remote_data_folder.clone()), local_data_folder, ), - ClusterMode::MultiNode(Box::new(cluster)), + ClusterMode::MultiNode(cluster), &ServerArgs::parse_from(["modelardbd", "edge", "data", "s3://bucket"]), ) .await diff --git a/crates/modelardb_server/src/configuration.rs b/crates/modelardb_server/src/configuration.rs index 524e4ff5..26e205db 100644 --- a/crates/modelardb_server/src/configuration.rs +++ b/crates/modelardb_server/src/configuration.rs @@ -1023,7 +1023,7 @@ mod tests { let configuration_manager = Arc::new(RwLock::new( ConfigurationManager::try_new( local_data_folder.clone(), - ClusterMode::MultiNode(Box::new(cluster)), + ClusterMode::MultiNode(cluster), &default_args(), ) .await diff --git a/crates/modelardb_server/src/data_folders.rs b/crates/modelardb_server/src/data_folders.rs index 873d3c1a..9c0ad66f 100644 --- a/crates/modelardb_server/src/data_folders.rs +++ b/crates/modelardb_server/src/data_folders.rs @@ -88,7 +88,7 @@ impl DataFolders { let cluster = Cluster::try_new(node, remote_data_folder.clone()).await?; Ok(( - ClusterMode::MultiNode(Box::new(cluster)), + ClusterMode::MultiNode(cluster), Self::new( local_data_folder.clone(), Some(remote_data_folder), @@ -111,7 +111,7 @@ impl DataFolders { let cluster = Cluster::try_new(node, remote_data_folder.clone()).await?; Ok(( - ClusterMode::MultiNode(Box::new(cluster)), + ClusterMode::MultiNode(cluster), Self::new( local_data_folder, Some(remote_data_folder.clone()), From 50f509dba1b89ae1087fb30d89aaeca5dd804c52 Mon Sep 17 00:00:00 2001 From: CGodiksen <36046286+CGodiksen@users.noreply.github.com> Date: Fri, 14 Aug 2026 14:53:39 +0200 Subject: [PATCH 33/33] Update based on comments from @chrthomsen --- crates/modelardb_embedded/src/operations/client.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/crates/modelardb_embedded/src/operations/client.rs b/crates/modelardb_embedded/src/operations/client.rs index 413be2bc..642bafd6 100644 --- a/crates/modelardb_embedded/src/operations/client.rs +++ b/crates/modelardb_embedded/src/operations/client.rs @@ -96,7 +96,7 @@ impl Client { Ok(()) } - /// Flushes all data currently in memory to disk. If the data could not be flushed, + /// Flushes all data in memory to disk. If the data could not be flushed, /// [`ModelarDbEmbeddedError`] is returned. pub async fn flush_memory(&mut self) -> Result<()> { self.send_action("FlushMemory", vec![]).await?; @@ -140,7 +140,7 @@ impl Client { Ok(protocol::NodeMetrics::decode(bytes)?) } - /// Send the action with the type `action_type` and an empty body to the node and return the + /// Sends the action with the type `action_type` and an empty body to the node and returns the /// body of the response. If the action could not be performed, [`ModelarDbEmbeddedError`] is /// returned. async fn retrieve_action_bytes(&mut self, action_type: &str) -> Result { @@ -155,7 +155,7 @@ impl Client { Ok(message.body) } - /// Send the action with the type `action_type` and `body` to the node and return the response + /// Sends the action with the type `action_type` and `body` to the node and returns the response /// stream. If the action could not be performed, [`ModelarDbEmbeddedError`] is returned. async fn send_action( &mut self,