From 91ff19993fd112c1e311e3e99feb44cc78856142 Mon Sep 17 00:00:00 2001 From: CGodiksen <36046286+CGodiksen@users.noreply.github.com> Date: Thu, 9 Jul 2026 06:42:52 +0200 Subject: [PATCH 01/48] Add new data optimizer config to proto file --- crates/modelardb_types/src/flight/protocol.proto | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/crates/modelardb_types/src/flight/protocol.proto b/crates/modelardb_types/src/flight/protocol.proto index 89dbbb9b..6455533b 100644 --- a/crates/modelardb_types/src/flight/protocol.proto +++ b/crates/modelardb_types/src/flight/protocol.proto @@ -79,6 +79,15 @@ message Configuration { // Whether the write-ahead log is enabled. bool wal_enabled = 9; + + // Target size, in bytes, of the files produced when automatically optimizing a table's storage. This is also the + // default value used when an OPTIMIZE query is executed without an explicit target size. + uint64 optimize_target_file_size_in_bytes = 10; + + // Retention period, in seconds, used when automatically vacuuming a table after optimization. This is also the + // default value used when a VACUUM query is executed without an explicit retention period. Note that a very low + // value can delete files an in-flight query is still scanning. + uint64 vacuum_retention_period_in_seconds = 11; } // Request to update the configuration of a ModelarDB node. @@ -89,6 +98,8 @@ message UpdateConfiguration { COMPRESSED_RESERVED_MEMORY_IN_BYTES = 2; TRANSFER_BATCH_SIZE_IN_BYTES = 3; SEGMENT_SIZE_THRESHOLD_IN_BYTES = 4; + OPTIMIZE_TARGET_FILE_SIZE_IN_BYTES = 5; + VACUUM_RETENTION_PERIOD_IN_SECONDS = 6; } // Setting to update in the configuration. From a77ab4d04d0bbd8e33f56b2661d24aba289e004f Mon Sep 17 00:00:00 2001 From: CGodiksen <36046286+CGodiksen@users.noreply.github.com> Date: Thu, 9 Jul 2026 06:46:20 +0200 Subject: [PATCH 02/48] Add data optimizer config to clap arguments --- crates/modelardb_server/src/main.rs | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/crates/modelardb_server/src/main.rs b/crates/modelardb_server/src/main.rs index e117d821..2bff0b5f 100644 --- a/crates/modelardb_server/src/main.rs +++ b/crates/modelardb_server/src/main.rs @@ -88,6 +88,19 @@ pub(crate) struct ServerArgs { #[arg(long, env = "MODELARDBD_WAL_ENABLED")] wal_enabled: Option, + /// Target size, in bytes, of the files produced when automatically optimizing a table's + /// storage. This is also the default value used when an OPTIMIZE query is executed without an + /// explicit target size. + #[arg(long, env = "MODELARDBD_OPTIMIZE_TARGET_FILE_SIZE_IN_BYTES")] + optimize_target_file_size_in_bytes: Option, + + /// Retention period, in seconds, used when automatically vacuuming a table after optimization. + /// This is also the default value used when a VACUUM query is executed without an explicit + /// retention period. Note that a very low value can delete files an in-flight query is still + /// scanning. + #[arg(long, env = "MODELARDBD_VACUUM_RETENTION_PERIOD_IN_SECONDS")] + vacuum_retention_period_in_seconds: Option, + /// Subcommand specifying the mode the server is started in and the required data folders. #[command(subcommand)] mode: ServerMode, From 5847eeccd4763fa51fb8c96148fbc3ddeaaee585 Mon Sep 17 00:00:00 2001 From: CGodiksen <36046286+CGodiksen@users.noreply.github.com> Date: Thu, 9 Jul 2026 07:09:06 +0200 Subject: [PATCH 03/48] Add data optimizer config to Configuration struct --- crates/modelardb_server/src/configuration.rs | 80 +++++++++++++++++++- 1 file changed, 79 insertions(+), 1 deletion(-) diff --git a/crates/modelardb_server/src/configuration.rs b/crates/modelardb_server/src/configuration.rs index c510f566..dfc5ab39 100644 --- a/crates/modelardb_server/src/configuration.rs +++ b/crates/modelardb_server/src/configuration.rs @@ -27,6 +27,7 @@ use std::sync::Arc; use modelardb_storage::data_folder::DataFolder; use modelardb_storage::write_ahead_log::WriteAheadLog; use modelardb_types::flight::protocol; +use modelardb_types::types::MAX_RETENTION_PERIOD_IN_SECONDS; use object_store::path::Path; use object_store::{Error, ObjectStoreExt, PutPayload}; use prost::Message; @@ -63,6 +64,14 @@ struct Configuration { /// The approximate maximum size, in bytes, of a single WAL segment file before it is closed and /// a new one is started. segment_size_threshold_in_bytes: u64, + /// Target size, in bytes, of the files produced when automatically optimizing a table's + /// storage. This is also the default value used when an OPTIMIZE query is executed without an + /// explicit target size. + optimize_target_file_size_in_bytes: u64, + /// Retention period, in seconds, used when automatically vacuuming a table after optimization. + /// This is also the default value used when a VACUUM query is executed without an explicit + /// retention period. + vacuum_retention_period_in_seconds: u64, /// Number of threads to allocate for converting multivariate time series to univariate /// time series. ingestion_threads: u8, @@ -99,6 +108,14 @@ impl Configuration { self.segment_size_threshold_in_bytes = value; } + if let Some(value) = args.optimize_target_file_size_in_bytes { + self.optimize_target_file_size_in_bytes = value; + } + + if let Some(value) = args.vacuum_retention_period_in_seconds { + self.vacuum_retention_period_in_seconds = value; + } + if let Some(value) = args.wal_enabled { self.wal_enabled = value; } @@ -114,10 +131,22 @@ impl Configuration { if self.ingestion_threads != 1 || self.compression_threads != 1 || self.writer_threads != 1 { return Err(ModelarDbServerError::InvalidState( - "Only one thread per component is currently supported.".to_string(), + "Only one thread per component is currently supported.".to_owned(), )); }; + if self.optimize_target_file_size_in_bytes == 0 { + return Err(ModelarDbServerError::InvalidState( + "Optimize target file size must be greater than zero.".to_owned(), + )); + } + + if self.vacuum_retention_period_in_seconds > MAX_RETENTION_PERIOD_IN_SECONDS { + return Err(ModelarDbServerError::InvalidState(format!( + "Vacuum retention period cannot be more than {MAX_RETENTION_PERIOD_IN_SECONDS} seconds." + ))); + } + Ok(()) } @@ -147,6 +176,8 @@ impl Default for Configuration { compressed_reserved_memory_in_bytes: 512 * 1024 * 1024, transfer_batch_size_in_bytes: Some(64 * 1024 * 1024), segment_size_threshold_in_bytes: 64 * 1024 * 1024, + optimize_target_file_size_in_bytes: 64 * 1024 * 1024, + vacuum_retention_period_in_seconds: 60 * 60 * 24 * 7, ingestion_threads: 1, compression_threads: 1, writer_threads: 1, @@ -389,6 +420,14 @@ impl ConfigurationManager { .await } + pub(crate) fn optimize_target_file_size_in_bytes(&self) -> u64 { + self.configuration.optimize_target_file_size_in_bytes + } + + pub(crate) fn vacuum_retention_period_in_seconds(&self) -> u64 { + self.configuration.vacuum_retention_period_in_seconds + } + pub(crate) fn ingestion_threads(&self) -> u8 { self.configuration.ingestion_threads } @@ -420,6 +459,12 @@ impl ConfigurationManager { compression_threads: self.configuration.compression_threads as u32, writer_threads: self.configuration.writer_threads as u32, wal_enabled: self.configuration.wal_enabled, + optimize_target_file_size_in_bytes: self + .configuration + .optimize_target_file_size_in_bytes, + vacuum_retention_period_in_seconds: self + .configuration + .vacuum_retention_period_in_seconds, }; configuration.encode_to_vec() @@ -539,6 +584,39 @@ mod tests { ); } + #[test] + fn test_validate_rejects_zero_optimize_target_file_size() { + let configuration = Configuration { + optimize_target_file_size_in_bytes: 0, + ..Configuration::default() + }; + + let result = configuration.validate(); + + assert_eq!( + result.unwrap_err().to_string(), + "Invalid State Error: Optimize target file size must be greater than zero." + ); + } + + #[test] + fn test_validate_rejects_too_large_vacuum_retention_period() { + let configuration = Configuration { + vacuum_retention_period_in_seconds: MAX_RETENTION_PERIOD_IN_SECONDS + 1, + ..Configuration::default() + }; + + let result = configuration.validate(); + + assert_eq!( + result.unwrap_err().to_string(), + format!( + "Invalid State Error: Vacuum retention period cannot be more than {} seconds.", + MAX_RETENTION_PERIOD_IN_SECONDS + ) + ); + } + #[tokio::test] async fn test_set_multivariate_reserved_memory_in_bytes() { let temp_dir = tempfile::tempdir().unwrap(); From 9550befa255201c2e192d58755c6486a12ca012c Mon Sep 17 00:00:00 2001 From: CGodiksen <36046286+CGodiksen@users.noreply.github.com> Date: Thu, 9 Jul 2026 07:19:36 +0200 Subject: [PATCH 04/48] Add data optimizer config to user docs --- crates/modelardb_server/src/configuration.rs | 2 ++ docs/user/README.md | 22 +++++++++++--------- 2 files changed, 14 insertions(+), 10 deletions(-) diff --git a/crates/modelardb_server/src/configuration.rs b/crates/modelardb_server/src/configuration.rs index dfc5ab39..3068bed1 100644 --- a/crates/modelardb_server/src/configuration.rs +++ b/crates/modelardb_server/src/configuration.rs @@ -511,6 +511,8 @@ mod tests { compressed_reserved_memory_in_bytes: 1, transfer_batch_size_in_bytes: Some(1), segment_size_threshold_in_bytes: 1, + optimize_target_file_size_in_bytes: 1, + vacuum_retention_period_in_seconds: 1, ..Configuration::default() }; diff --git a/docs/user/README.md b/docs/user/README.md index 0dfd82f9..a7004e9b 100644 --- a/docs/user/README.md +++ b/docs/user/README.md @@ -380,16 +380,18 @@ the built-in defaults. Note that the connection settings `--host` and `--port` a file. Variables marked with ✓ in the **Updatable** column can also be updated while the server is running using the `UpdateConfiguration` action without requiring a restart. The update is persisted in the configuration file. -| **CLI Flag** | **Environment Variable** | **Default** | **Updatable** | **Description** | -|-------------------------------------------|----------------------------------------------------|-------------|---------------|--------------------------------------------------------------------------------------------------------------------------------| -| `--host` | `MODELARDBD_HOST` | 127.0.0.1 | | The host address of the `modelardbd` server. | -| `--port` | `MODELARDBD_PORT` | 9999 | | The port of the `modelardbd` server. | -| `--wal-enabled` | `MODELARDBD_WAL_ENABLED` | true | | Whether the write-ahead log is enabled. | -| `--multivariate-reserved-memory-in-bytes` | `MODELARDBD_MULTIVARIATE_RESERVED_MEMORY_IN_BYTES` | 512 MB | ✓ | The amount of memory to reserve for storing multivariate time series. | -| `--uncompressed-reserved-memory-in-bytes` | `MODELARDBD_UNCOMPRESSED_RESERVED_MEMORY_IN_BYTES` | 512 MB | ✓ | The amount of memory to reserve for storing uncompressed data buffers. | -| `--compressed-reserved-memory-in-bytes` | `MODELARDBD_COMPRESSED_RESERVED_MEMORY_IN_BYTES` | 512 MB | ✓ | The amount of memory to reserve for storing compressed data buffers. | -| `--transfer-batch-size-in-bytes` | `MODELARDBD_TRANSFER_BATCH_SIZE_IN_BYTES` | 64 MB | ✓ | The amount of data that must be collected before transferring a batch to the remote object store. | -| `--segment-size-threshold-in-bytes` | `MODELARDBD_SEGMENT_SIZE_THRESHOLD_IN_BYTES` | 64 MB | ✓ | The approximate maximum size of a single WAL segment file before a new one is started. Only updatable when the WAL is enabled. | +| **CLI Flag** | **Environment Variable** | **Default** | **Updatable** | **Description** | +|-------------------------------------------|----------------------------------------------------|-------------|---------------|--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| `--host` | `MODELARDBD_HOST` | 127.0.0.1 | | The host address of the `modelardbd` server. | +| `--port` | `MODELARDBD_PORT` | 9999 | | The port of the `modelardbd` server. | +| `--wal-enabled` | `MODELARDBD_WAL_ENABLED` | true | | Whether the write-ahead log is enabled. | +| `--multivariate-reserved-memory-in-bytes` | `MODELARDBD_MULTIVARIATE_RESERVED_MEMORY_IN_BYTES` | 512 MB | ✓ | The amount of memory to reserve for storing multivariate time series. | +| `--uncompressed-reserved-memory-in-bytes` | `MODELARDBD_UNCOMPRESSED_RESERVED_MEMORY_IN_BYTES` | 512 MB | ✓ | The amount of memory to reserve for storing uncompressed data buffers. | +| `--compressed-reserved-memory-in-bytes` | `MODELARDBD_COMPRESSED_RESERVED_MEMORY_IN_BYTES` | 512 MB | ✓ | The amount of memory to reserve for storing compressed data buffers. | +| `--transfer-batch-size-in-bytes` | `MODELARDBD_TRANSFER_BATCH_SIZE_IN_BYTES` | 64 MB | ✓ | The amount of data that must be collected before transferring a batch to the remote object store. | +| `--segment-size-threshold-in-bytes` | `MODELARDBD_SEGMENT_SIZE_THRESHOLD_IN_BYTES` | 64 MB | ✓ | The approximate maximum size of a single WAL segment file before a new one is started. Only updatable when the WAL is enabled. | +| `--optimize-target-file-size-in-bytes` | `MODELARDBD_OPTIMIZE_TARGET_FILE_SIZE_IN_BYTES` | 64 MB | ✓ | The target size of the files produced when automatically optimizing a table's storage. Also used as the default when `OPTIMIZE` is run without an explicit `TARGET`. | +| `--vacuum-retention-period-in-seconds` | `MODELARDBD_VACUUM_RETENTION_PERIOD_IN_SECONDS` | 7 days | ✓ | The retention period used when automatically vacuuming a table after optimization. Also used as the default when `VACUUM` is run without an explicit `RETAIN`. Note that a very low value can delete files an in-flight query is still scanning. | ## Docker Two different [Docker](https://docs.docker.com/) environments are included to make it easy to experiment with the From ced2819e4db7f74201ceceb9177fcce51aa96271 Mon Sep 17 00:00:00 2001 From: CGodiksen <36046286+CGodiksen@users.noreply.github.com> Date: Thu, 9 Jul 2026 07:37:56 +0200 Subject: [PATCH 05/48] Fix ordering issue and run Rustfmt --- crates/modelardb_auth/src/lib.rs | 4 +++- crates/modelardb_server/src/main.rs | 8 ++++---- crates/modelardb_server/src/remote/mod.rs | 2 +- 3 files changed, 8 insertions(+), 6 deletions(-) diff --git a/crates/modelardb_auth/src/lib.rs b/crates/modelardb_auth/src/lib.rs index 763c383f..cc6ebfe3 100644 --- a/crates/modelardb_auth/src/lib.rs +++ b/crates/modelardb_auth/src/lib.rs @@ -61,7 +61,9 @@ impl BearerInterceptor { .map(|token| { format!("Bearer {token}") .parse::() - .map_err(|error| Status::invalid_argument(format!("Token is not ASCII: {error}."))) + .map_err(|error| { + Status::invalid_argument(format!("Token is not ASCII: {error}.")) + }) }) .transpose()?; diff --git a/crates/modelardb_server/src/main.rs b/crates/modelardb_server/src/main.rs index 2bff0b5f..d0bcf616 100644 --- a/crates/modelardb_server/src/main.rs +++ b/crates/modelardb_server/src/main.rs @@ -84,10 +84,6 @@ pub(crate) struct ServerArgs { #[arg(long, env = "MODELARDBD_SEGMENT_SIZE_THRESHOLD_IN_BYTES")] segment_size_threshold_in_bytes: Option, - /// Whether the write-ahead log is enabled. - #[arg(long, env = "MODELARDBD_WAL_ENABLED")] - wal_enabled: Option, - /// Target size, in bytes, of the files produced when automatically optimizing a table's /// storage. This is also the default value used when an OPTIMIZE query is executed without an /// explicit target size. @@ -101,6 +97,10 @@ pub(crate) struct ServerArgs { #[arg(long, env = "MODELARDBD_VACUUM_RETENTION_PERIOD_IN_SECONDS")] vacuum_retention_period_in_seconds: Option, + /// Whether the write-ahead log is enabled. + #[arg(long, env = "MODELARDBD_WAL_ENABLED")] + wal_enabled: Option, + /// Subcommand specifying the mode the server is started in and the required data folders. #[command(subcommand)] mode: ServerMode, diff --git a/crates/modelardb_server/src/remote/mod.rs b/crates/modelardb_server/src/remote/mod.rs index cf63bc3c..f833d4ce 100644 --- a/crates/modelardb_server/src/remote/mod.rs +++ b/crates/modelardb_server/src/remote/mod.rs @@ -67,7 +67,7 @@ use crate::remote::auth_layer::AuthLayer; /// Start an Apache Arrow Flight server on 0.0.0.0:`port` that passes `context` to the methods that /// process the requests through [`FlightServiceHandler`]. All requests are passed through the /// [`AuthLayer`], which authenticates them using `maybe_authenticator` before they are passed to -/// the [`FlightServiceHandler`]. If `maybe_authenticator` is [`None`], authentication is disabled, +/// the [`FlightServiceHandler`]. If `maybe_authenticator` is [`None`], authentication is disabled, /// and every request that is not an internal cluster request is allowed. pub async fn start_apache_arrow_flight_server( context: Arc, From 32f1167d8227b67a7a286d5341b113037f7fbc90 Mon Sep 17 00:00:00 2001 From: CGodiksen <36046286+CGodiksen@users.noreply.github.com> Date: Wed, 5 Aug 2026 07:24:13 +0200 Subject: [PATCH 06/48] Add a data folder method to return file sizes for a table --- crates/modelardb_storage/src/data_folder/mod.rs | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/crates/modelardb_storage/src/data_folder/mod.rs b/crates/modelardb_storage/src/data_folder/mod.rs index 4fe76801..b5c44d21 100644 --- a/crates/modelardb_storage/src/data_folder/mod.rs +++ b/crates/modelardb_storage/src/data_folder/mod.rs @@ -48,11 +48,11 @@ use modelardb_types::types::{ ArrowValue, CloudCredentials, ErrorBound, GeneratedColumn, MAX_RETENTION_PERIOD_IN_SECONDS, TimeSeriesTableMetadata, }; -use object_store::ObjectStore; use object_store::aws::AmazonS3Builder; use object_store::local::LocalFileSystem; use object_store::memory::InMemory; use object_store::path::Path; +use object_store::{ObjectStore, ObjectStoreExt}; use url::Url; use crate::data_folder::delta_table_writer::DeltaTableWriter; @@ -711,6 +711,21 @@ impl DataFolder { Ok(()) } + /// Return the size in bytes of each Apache Parquet file that makes up the Delta Lake table with + /// `table_name`. If the table does not exist or the size of a file could not be read, a + /// [`ModelarDbStorageError`] is returned. + pub async fn table_file_sizes(&self, table_name: &str) -> Result> { + let delta_table = self.delta_table(table_name).await?; + let object_store = delta_table.object_store(); + + let mut file_sizes_in_bytes = Vec::new(); + for file_path in delta_table.get_files_by_partitions(&[]).await? { + file_sizes_in_bytes.push(object_store.head(&file_path).await?.size); + } + + Ok(file_sizes_in_bytes) + } + /// Return a [`DeltaTableWriter`] for writing to the table with `table_name` in the Delta Lake, /// or a [`ModelarDbStorageError`] if a connection to the Delta Lake cannot be established or /// the table does not exist. From 80000365f4e3b214e9808964e39da81118ce14a6 Mon Sep 17 00:00:00 2001 From: CGodiksen <36046286+CGodiksen@users.noreply.github.com> Date: Wed, 5 Aug 2026 07:43:33 +0200 Subject: [PATCH 07/48] Use data folder method to compute table sizes --- .../src/storage/data_transfer.rs | 30 ++++++++----------- 1 file changed, 12 insertions(+), 18 deletions(-) diff --git a/crates/modelardb_server/src/storage/data_transfer.rs b/crates/modelardb_server/src/storage/data_transfer.rs index 182e896e..885d530c 100644 --- a/crates/modelardb_server/src/storage/data_transfer.rs +++ b/crates/modelardb_server/src/storage/data_transfer.rs @@ -22,7 +22,6 @@ use dashmap::DashMap; use deltalake::arrow::array::RecordBatch; use futures::TryStreamExt; use modelardb_storage::data_folder::DataFolder; -use object_store::ObjectStoreExt; use tracing::debug; use crate::error::Result; @@ -61,15 +60,13 @@ impl DataTransfer { // The size of tables is computed manually as datafusion_table_statistics() is not exact. let table_size_in_bytes = DashMap::with_capacity(table_names.len()); for table_name in table_names { - let delta_table = local_data_folder.delta_table(&table_name).await?; + let size_in_bytes: u64 = local_data_folder + .table_file_sizes(&table_name) + .await? + .into_iter() + .sum(); - let mut table_size_in_bytes = table_size_in_bytes.entry(table_name).or_insert(0); - - let object_store = delta_table.object_store(); - for file_path in delta_table.get_files_by_partitions(&[]).await? { - let object_meta = object_store.head(&file_path).await?; - *table_size_in_bytes += object_meta.size; - } + table_size_in_bytes.insert(table_name, size_in_bytes); } let data_transfer = Self { @@ -460,15 +457,12 @@ mod tests { /// Return the total size of the files in the table with `table_name` in `local_data_folder`. async fn table_files_size(local_data_folder: &DataFolder, table_name: &str) -> u64 { - let delta_table = local_data_folder.delta_table(table_name).await.unwrap(); - - let mut files_size = 0; - for file_path in delta_table.get_files_by_partitions(&[]).await.unwrap() { - let object_meta = delta_table.object_store().head(&file_path).await; - files_size += object_meta.unwrap().size; - } - - files_size + local_data_folder + .table_file_sizes(table_name) + .await + .unwrap() + .into_iter() + .sum() } /// Create a data transfer component with a target object store that is deleted once the test is finished. From 410594d23f26a00cd9b2a97b6721554ff4f38667 Mon Sep 17 00:00:00 2001 From: CGodiksen <36046286+CGodiksen@users.noreply.github.com> Date: Wed, 5 Aug 2026 08:46:38 +0200 Subject: [PATCH 08/48] Add struct for DataStorageOptimizer --- .../src/storage/data_storage_optimizer.rs | 48 +++++++++++++++++++ crates/modelardb_server/src/storage/mod.rs | 1 + 2 files changed, 49 insertions(+) create mode 100644 crates/modelardb_server/src/storage/data_storage_optimizer.rs diff --git a/crates/modelardb_server/src/storage/data_storage_optimizer.rs b/crates/modelardb_server/src/storage/data_storage_optimizer.rs new file mode 100644 index 00000000..0b1ecbf1 --- /dev/null +++ b/crates/modelardb_server/src/storage/data_storage_optimizer.rs @@ -0,0 +1,48 @@ +/* Copyright 2026 The ModelarDB Contributors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +//! Support for automatically optimizing how compressed data is stored on disk. As compressed data +//! is saved, many small Apache Parquet files accumulate for each table. This component compacts +//! those small files into fewer larger files and vacuums the small files left behind to reduce +//! storage use and query time. + +use dashmap::DashMap; +use modelardb_storage::data_folder::DataFolder; +use tracing::debug; + +use crate::error::Result; + +/// Compacts the many small Apache Parquet files that accumulate for a table into fewer larger files +/// and vacuums the files left behind by the compaction. The component accumulates an estimate of +/// how much compactable data each table has and, once the estimate reaches the target file size, +/// optimizes and vacuums that table. The component only operates on the local data folder. +pub(super) struct DataStorageOptimizer { + /// The data folder containing all compressed data managed by the + /// [`StorageEngine`](crate::storage::StorageEngine). + local_data_folder: DataFolder, + /// The target size, in bytes, of the files produced when a table is optimized. Also used as the + /// trigger for when a table is optimized. + optimize_target_file_size_in_bytes: u64, + /// The retention period, in seconds, used when a table is vacuumed after it is optimized. + /// Note that a very low value can let the vacuum physically delete files that an in-flight + /// query is still scanning, causing that query to fail. + vacuum_retention_period_in_seconds: u64, + /// Map from table names to an estimate of how many bytes each table has in files smaller than + /// the target size. The estimate over-approximates the on-disk size since it is increased by + /// the in-memory size of the compressed data, so it ignores the compression applied by Apache + /// Parquet. Note that it is not reduced when a table is truncated by the data transfer + /// component for simplicity. + estimated_compactable_size_in_bytes: DashMap, +} diff --git a/crates/modelardb_server/src/storage/mod.rs b/crates/modelardb_server/src/storage/mod.rs index 8460b51b..eb5da288 100644 --- a/crates/modelardb_server/src/storage/mod.rs +++ b/crates/modelardb_server/src/storage/mod.rs @@ -28,6 +28,7 @@ mod data_transfer; mod types; mod uncompressed_data_buffer; mod uncompressed_data_manager; +mod data_storage_optimizer; use std::sync::Arc; use std::thread::{self, JoinHandle}; From 2c712c27bd2907775025ca12b1abbe2862e8d205 Mon Sep 17 00:00:00 2001 From: CGodiksen <36046286+CGodiksen@users.noreply.github.com> Date: Wed, 5 Aug 2026 08:48:59 +0200 Subject: [PATCH 09/48] Add try_new for DataStorageOptimizer --- .../src/storage/data_storage_optimizer.rs | 34 +++++++++++++++++++ 1 file changed, 34 insertions(+) diff --git a/crates/modelardb_server/src/storage/data_storage_optimizer.rs b/crates/modelardb_server/src/storage/data_storage_optimizer.rs index 0b1ecbf1..951e78fa 100644 --- a/crates/modelardb_server/src/storage/data_storage_optimizer.rs +++ b/crates/modelardb_server/src/storage/data_storage_optimizer.rs @@ -46,3 +46,37 @@ pub(super) struct DataStorageOptimizer { /// component for simplicity. estimated_compactable_size_in_bytes: DashMap, } + +impl DataStorageOptimizer { + /// Create a new [`DataStorageOptimizer`] that optimizes the tables in `local_data_folder`, + /// producing files of approximately `optimize_target_file_size_in_bytes` bytes and vacuuming + /// with a retention period of `vacuum_retention_period_in_seconds` seconds. The estimate for + /// each table is initialized with the combined size of its files smaller than the target size, + /// so small files written before a restart are not forgotten. If the files in `local_data_folder` + /// could not be read, return [`ModelarDbServerError`](crate::error::ModelarDbServerError). + pub(super) async fn try_new( + local_data_folder: DataFolder, + optimize_target_file_size_in_bytes: u64, + vacuum_retention_period_in_seconds: u64, + ) -> Result { + let table_names = local_data_folder.table_names().await?; + + let estimated_compactable_size_in_bytes = DashMap::with_capacity(table_names.len()); + for table_name in table_names { + let compactable_size_in_bytes: u64 = local_data_folder + .table_file_sizes(&table_name) + .await? + .into_iter() + .filter(|size_in_bytes| *size_in_bytes < optimize_target_file_size_in_bytes) + .sum(); + + estimated_compactable_size_in_bytes.insert(table_name, compactable_size_in_bytes); + } + + Ok(Self { + local_data_folder, + optimize_target_file_size_in_bytes, + vacuum_retention_period_in_seconds, + estimated_compactable_size_in_bytes, + }) + } From 0ef64391e727f5267d01c0a400068ce03dcf2595 Mon Sep 17 00:00:00 2001 From: CGodiksen <36046286+CGodiksen@users.noreply.github.com> Date: Wed, 5 Aug 2026 08:54:58 +0200 Subject: [PATCH 10/48] Add method to increase the estimated compactable size --- .../src/storage/data_storage_optimizer.rs | 40 +++++++++++++++++++ 1 file changed, 40 insertions(+) diff --git a/crates/modelardb_server/src/storage/data_storage_optimizer.rs b/crates/modelardb_server/src/storage/data_storage_optimizer.rs index 951e78fa..f06e45ba 100644 --- a/crates/modelardb_server/src/storage/data_storage_optimizer.rs +++ b/crates/modelardb_server/src/storage/data_storage_optimizer.rs @@ -80,3 +80,43 @@ impl DataStorageOptimizer { estimated_compactable_size_in_bytes, }) } + + /// Increase the estimated compactable size of the table with `table_name` by `size_in_bytes`. + /// If the estimate has reached `optimize_target_file_size_in_bytes`, the table's small files + /// are compacted and the files left behind are vacuumed. The trigger assumes each newly written + /// file is smaller than the target size. If the target is set below the size of a typical file, + /// optimization is attempted on nearly every write, but is a harmless no-op. Returns [`Ok`] if + /// the table did not need optimizing or was optimized successfully, otherwise + /// [`ModelarDbServerError`](crate::error::ModelarDbServerError). + pub(super) async fn increase_estimated_compactable_size( + &self, + table_name: &str, + size_in_bytes: u64, + ) -> Result<()> { + // entry() is not used as it would require the allocation of a new String for each lookup as + // it must be given as a K, while get_mut() accepts the key as a &K so one K can be used. + if !self + .estimated_compactable_size_in_bytes + .contains_key(table_name) + { + self.estimated_compactable_size_in_bytes + .insert(table_name.to_owned(), 0); + } + *self + .estimated_compactable_size_in_bytes + .get_mut(table_name) + .unwrap() += size_in_bytes; + + let estimate_reached_target = *self + .estimated_compactable_size_in_bytes + .get(table_name) + .expect("table_name should have been added to estimated_compactable_size_in_bytes.") + .value() + >= self.optimize_target_file_size_in_bytes; + + if estimate_reached_target { + // self.optimize_and_vacuum_table(table_name).await?; + } + + Ok(()) + } From 3cae52c90d8dd2115d2badc1bbbe253e40b0bc39 Mon Sep 17 00:00:00 2001 From: CGodiksen <36046286+CGodiksen@users.noreply.github.com> Date: Wed, 5 Aug 2026 08:58:50 +0200 Subject: [PATCH 11/48] Add method to optimize and vacuum table and use it --- .../src/storage/data_storage_optimizer.rs | 29 ++++++++++++++++++- 1 file changed, 28 insertions(+), 1 deletion(-) diff --git a/crates/modelardb_server/src/storage/data_storage_optimizer.rs b/crates/modelardb_server/src/storage/data_storage_optimizer.rs index f06e45ba..8d509ad4 100644 --- a/crates/modelardb_server/src/storage/data_storage_optimizer.rs +++ b/crates/modelardb_server/src/storage/data_storage_optimizer.rs @@ -115,8 +115,35 @@ impl DataStorageOptimizer { >= self.optimize_target_file_size_in_bytes; if estimate_reached_target { - // self.optimize_and_vacuum_table(table_name).await?; + self.optimize_and_vacuum_table(table_name).await?; } Ok(()) } + + /// Compact the small files of the table with `table_name` into files of approximately + /// `optimize_target_file_size_in_bytes` bytes, vacuum the small files left behind, and reset + /// the table's estimated compactable size. Note that the vacuum can physically delete files + /// that an in-flight query is still scanning if `vacuum_retention_period_in_seconds` is very + /// low. Returns [`Ok`] if the table was optimized successfully, otherwise + /// [`ModelarDbServerError`](crate::error::ModelarDbServerError). + async fn optimize_and_vacuum_table(&self, table_name: &str) -> Result<()> { + debug!("Optimizing the storage of the table '{table_name}'."); + + self.local_data_folder + .optimize_table(table_name, Some(self.optimize_target_file_size_in_bytes)) + .await?; + + self.local_data_folder + .vacuum_table(table_name, Some(self.vacuum_retention_period_in_seconds)) + .await?; + + // Reset the estimate so the next optimization only counts data written from now on. + *self + .estimated_compactable_size_in_bytes + .get_mut(table_name) + .expect("table_name should be in estimated_compactable_size_in_bytes.") = 0; + + Ok(()) + } +} From b01022b28bed673f89378bfdcbbcc407b2e612ca Mon Sep 17 00:00:00 2001 From: CGodiksen <36046286+CGodiksen@users.noreply.github.com> Date: Wed, 5 Aug 2026 09:01:26 +0200 Subject: [PATCH 12/48] Add the actual table name to the expect messages --- .../src/storage/data_storage_optimizer.rs | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/crates/modelardb_server/src/storage/data_storage_optimizer.rs b/crates/modelardb_server/src/storage/data_storage_optimizer.rs index 8d509ad4..26ca87b1 100644 --- a/crates/modelardb_server/src/storage/data_storage_optimizer.rs +++ b/crates/modelardb_server/src/storage/data_storage_optimizer.rs @@ -110,7 +110,9 @@ impl DataStorageOptimizer { let estimate_reached_target = *self .estimated_compactable_size_in_bytes .get(table_name) - .expect("table_name should have been added to estimated_compactable_size_in_bytes.") + .expect(&format!( + "{table_name} should have been added to estimated_compactable_size_in_bytes." + )) .value() >= self.optimize_target_file_size_in_bytes; @@ -142,7 +144,9 @@ impl DataStorageOptimizer { *self .estimated_compactable_size_in_bytes .get_mut(table_name) - .expect("table_name should be in estimated_compactable_size_in_bytes.") = 0; + .expect(&format!( + "{table_name} should be in estimated_compactable_size_in_bytes." + )) = 0; Ok(()) } From f56f4c1796d7a351bc1b44e8a99e77a6912b130e Mon Sep 17 00:00:00 2001 From: CGodiksen <36046286+CGodiksen@users.noreply.github.com> Date: Wed, 5 Aug 2026 09:09:36 +0200 Subject: [PATCH 13/48] Add test util for data storage optimizer tests --- .../src/storage/data_storage_optimizer.rs | 61 +++++++++++++++++++ 1 file changed, 61 insertions(+) diff --git a/crates/modelardb_server/src/storage/data_storage_optimizer.rs b/crates/modelardb_server/src/storage/data_storage_optimizer.rs index 26ca87b1..3780f3b9 100644 --- a/crates/modelardb_server/src/storage/data_storage_optimizer.rs +++ b/crates/modelardb_server/src/storage/data_storage_optimizer.rs @@ -151,3 +151,64 @@ impl DataStorageOptimizer { Ok(()) } } + +#[cfg(test)] +mod tests { + use super::*; + + use modelardb_test::table::{self, TIME_SERIES_TABLE_NAME}; + use tempfile::{self, TempDir}; + + const OPTIMIZE_TARGET_FILE_SIZE_IN_BYTES: u64 = 1024 * 1024; + const VACUUM_RETENTION_PERIOD_IN_SECONDS: u64 = 0; + /// Create a [`DataFolder`] in a local [`TempDir`] containing a single time series table. + async fn create_local_data_folder_with_table() -> (TempDir, DataFolder) { + let temp_dir = tempfile::tempdir().unwrap(); + let temp_dir_url = temp_dir.path().to_str().unwrap(); + let local_data_folder = DataFolder::open_local_url(temp_dir_url).await.unwrap(); + + let time_series_table_metadata = table::time_series_table_metadata(); + local_data_folder + .create_time_series_table(&time_series_table_metadata) + .await + .unwrap(); + + (temp_dir, local_data_folder) + } + + /// Write `batch_count` batches of compressed segments to the time series table in + /// `local_data_folder`, each as a separate file. + async fn write_batches_to_table(local_data_folder: &DataFolder, batch_count: u8) { + for _ in 0..batch_count { + local_data_folder + .write_record_batches( + TIME_SERIES_TABLE_NAME, + vec![table::compressed_segments_record_batch()], + ) + .await + .unwrap(); + } + } + + /// Return the number of active files in the time series table in `local_data_folder`. + async fn active_file_count(local_data_folder: &DataFolder) -> usize { + let mut delta_table = local_data_folder + .delta_table(TIME_SERIES_TABLE_NAME) + .await + .unwrap(); + delta_table.load().await.unwrap(); + + delta_table.get_file_uris().unwrap().count() + } + + /// Create a [`DataStorageOptimizer`] that optimizes the tables in `local_data_folder`. + async fn create_data_storage_optimizer(local_data_folder: DataFolder) -> DataStorageOptimizer { + DataStorageOptimizer::try_new( + local_data_folder, + OPTIMIZE_TARGET_FILE_SIZE_IN_BYTES, + VACUUM_RETENTION_PERIOD_IN_SECONDS, + ) + .await + .unwrap() + } +} From d22d2ccf54092e1285ba9218e2b989f84b9ada75 Mon Sep 17 00:00:00 2001 From: CGodiksen <36046286+CGodiksen@users.noreply.github.com> Date: Wed, 5 Aug 2026 09:12:21 +0200 Subject: [PATCH 14/48] Add test for optimizing when size reaches target file size --- .../src/storage/data_storage_optimizer.rs | 32 +++++++++++++++++++ 1 file changed, 32 insertions(+) diff --git a/crates/modelardb_server/src/storage/data_storage_optimizer.rs b/crates/modelardb_server/src/storage/data_storage_optimizer.rs index 3780f3b9..e6cf6e2d 100644 --- a/crates/modelardb_server/src/storage/data_storage_optimizer.rs +++ b/crates/modelardb_server/src/storage/data_storage_optimizer.rs @@ -161,6 +161,38 @@ mod tests { const OPTIMIZE_TARGET_FILE_SIZE_IN_BYTES: u64 = 1024 * 1024; const VACUUM_RETENTION_PERIOD_IN_SECONDS: u64 = 0; + + // Tests for increase_estimated_compactable_size(). + #[tokio::test] + async fn test_optimize_table_when_estimate_reaches_target() { + let (_temp_dir, local_data_folder) = create_local_data_folder_with_table().await; + let optimizer = create_data_storage_optimizer(local_data_folder.clone()).await; + + write_batches_to_table(&local_data_folder, 3).await; + + let initial_file_count = active_file_count(&local_data_folder).await; + assert_eq!(initial_file_count, 3); + + optimizer + .increase_estimated_compactable_size( + TIME_SERIES_TABLE_NAME, + OPTIMIZE_TARGET_FILE_SIZE_IN_BYTES, + ) + .await + .unwrap(); + + // The small files should have been compacted into a single file. + assert_eq!(active_file_count(&local_data_folder).await, 1); + + // The estimate should have been reset after optimizing. + assert_eq!( + *optimizer + .estimated_compactable_size_in_bytes + .get(TIME_SERIES_TABLE_NAME) + .unwrap(), + 0 + ); + } /// Create a [`DataFolder`] in a local [`TempDir`] containing a single time series table. async fn create_local_data_folder_with_table() -> (TempDir, DataFolder) { let temp_dir = tempfile::tempdir().unwrap(); From 3d9f76b7c300ab661aef6e1ae6b8282b161d3823 Mon Sep 17 00:00:00 2001 From: CGodiksen <36046286+CGodiksen@users.noreply.github.com> Date: Wed, 5 Aug 2026 09:15:27 +0200 Subject: [PATCH 15/48] Add test for not opmizing when the estimate is below the target --- .../src/storage/data_storage_optimizer.rs | 40 +++++++++++++++++-- 1 file changed, 37 insertions(+), 3 deletions(-) diff --git a/crates/modelardb_server/src/storage/data_storage_optimizer.rs b/crates/modelardb_server/src/storage/data_storage_optimizer.rs index e6cf6e2d..0eec6952 100644 --- a/crates/modelardb_server/src/storage/data_storage_optimizer.rs +++ b/crates/modelardb_server/src/storage/data_storage_optimizer.rs @@ -170,7 +170,7 @@ mod tests { write_batches_to_table(&local_data_folder, 3).await; - let initial_file_count = active_file_count(&local_data_folder).await; + let initial_file_count = table_file_count(&local_data_folder).await; assert_eq!(initial_file_count, 3); optimizer @@ -182,7 +182,7 @@ mod tests { .unwrap(); // The small files should have been compacted into a single file. - assert_eq!(active_file_count(&local_data_folder).await, 1); + assert_eq!(table_file_count(&local_data_folder).await, 1); // The estimate should have been reset after optimizing. assert_eq!( @@ -193,6 +193,40 @@ mod tests { 0 ); } + + #[tokio::test] + async fn test_do_not_optimize_table_when_estimate_below_target() { + let (_temp_dir, local_data_folder) = create_local_data_folder_with_table().await; + let optimizer = create_data_storage_optimizer(local_data_folder.clone()).await; + + write_batches_to_table(&local_data_folder, 3).await; + + let initial_file_count = table_file_count(&local_data_folder).await; + assert_eq!(initial_file_count, 3); + + optimizer + .increase_estimated_compactable_size( + TIME_SERIES_TABLE_NAME, + OPTIMIZE_TARGET_FILE_SIZE_IN_BYTES - 1, + ) + .await + .unwrap(); + + // No files should have been compacted since the estimate did not reach the target. + assert_eq!( + table_file_count(&local_data_folder).await, + initial_file_count + ); + + // The estimate should have accumulated without being reset. + assert_eq!( + *optimizer + .estimated_compactable_size_in_bytes + .get(TIME_SERIES_TABLE_NAME) + .unwrap(), + OPTIMIZE_TARGET_FILE_SIZE_IN_BYTES - 1 + ); + } /// Create a [`DataFolder`] in a local [`TempDir`] containing a single time series table. async fn create_local_data_folder_with_table() -> (TempDir, DataFolder) { let temp_dir = tempfile::tempdir().unwrap(); @@ -223,7 +257,7 @@ mod tests { } /// Return the number of active files in the time series table in `local_data_folder`. - async fn active_file_count(local_data_folder: &DataFolder) -> usize { + async fn table_file_count(local_data_folder: &DataFolder) -> usize { let mut delta_table = local_data_folder .delta_table(TIME_SERIES_TABLE_NAME) .await From 2a8f6864dc820f3a72e34ad6a1a66d217dc703ae Mon Sep 17 00:00:00 2001 From: CGodiksen <36046286+CGodiksen@users.noreply.github.com> Date: Wed, 5 Aug 2026 09:17:29 +0200 Subject: [PATCH 16/48] Add test for catching already existing files in try_new --- .../src/storage/data_storage_optimizer.rs | 28 +++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/crates/modelardb_server/src/storage/data_storage_optimizer.rs b/crates/modelardb_server/src/storage/data_storage_optimizer.rs index 0eec6952..c6fea8dc 100644 --- a/crates/modelardb_server/src/storage/data_storage_optimizer.rs +++ b/crates/modelardb_server/src/storage/data_storage_optimizer.rs @@ -162,6 +162,33 @@ mod tests { const OPTIMIZE_TARGET_FILE_SIZE_IN_BYTES: u64 = 1024 * 1024; const VACUUM_RETENTION_PERIOD_IN_SECONDS: u64 = 0; + // Tests for try_new(). + #[tokio::test] + async fn test_initialize_estimate_from_existing_small_files() { + let (_temp_dir, local_data_folder) = create_local_data_folder_with_table().await; + write_batches_to_table(&local_data_folder, 3).await; + + // The optimizer is created after the data is written, so its estimate includes the small + // files already on disk. + let optimizer = create_data_storage_optimizer(local_data_folder.clone()).await; + + let expected_estimate: u64 = local_data_folder + .table_file_sizes(TIME_SERIES_TABLE_NAME) + .await + .unwrap() + .into_iter() + .sum(); + assert!(expected_estimate > 0); + + assert_eq!( + *optimizer + .estimated_compactable_size_in_bytes + .get(TIME_SERIES_TABLE_NAME) + .unwrap(), + expected_estimate + ); + } + // Tests for increase_estimated_compactable_size(). #[tokio::test] async fn test_optimize_table_when_estimate_reaches_target() { @@ -227,6 +254,7 @@ mod tests { OPTIMIZE_TARGET_FILE_SIZE_IN_BYTES - 1 ); } + /// Create a [`DataFolder`] in a local [`TempDir`] containing a single time series table. async fn create_local_data_folder_with_table() -> (TempDir, DataFolder) { let temp_dir = tempfile::tempdir().unwrap(); From df94b12e2bbe22e839ac60400d080289c4a05f2f Mon Sep 17 00:00:00 2001 From: CGodiksen <36046286+CGodiksen@users.noreply.github.com> Date: Wed, 5 Aug 2026 09:18:29 +0200 Subject: [PATCH 17/48] Run Rustfmt --- crates/modelardb_server/src/storage/mod.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/modelardb_server/src/storage/mod.rs b/crates/modelardb_server/src/storage/mod.rs index eb5da288..12266bbc 100644 --- a/crates/modelardb_server/src/storage/mod.rs +++ b/crates/modelardb_server/src/storage/mod.rs @@ -24,11 +24,11 @@ mod compressed_data_buffer; mod compressed_data_manager; pub(super) mod data_sinks; // pub(super) so it can be used in context.rs. +mod data_storage_optimizer; mod data_transfer; mod types; mod uncompressed_data_buffer; mod uncompressed_data_manager; -mod data_storage_optimizer; use std::sync::Arc; use std::thread::{self, JoinHandle}; From 8375abb5d460d02e050dfcd7bb38c50d0fc03bf4 Mon Sep 17 00:00:00 2001 From: CGodiksen <36046286+CGodiksen@users.noreply.github.com> Date: Wed, 5 Aug 2026 10:39:29 +0200 Subject: [PATCH 18/48] Add data storage optimizer to compressed data manager --- .../src/storage/compressed_data_manager.rs | 6 ++++++ crates/modelardb_server/src/storage/mod.rs | 9 +++++++++ 2 files changed, 15 insertions(+) diff --git a/crates/modelardb_server/src/storage/compressed_data_manager.rs b/crates/modelardb_server/src/storage/compressed_data_manager.rs index 0c3e88c0..ba6671df 100644 --- a/crates/modelardb_server/src/storage/compressed_data_manager.rs +++ b/crates/modelardb_server/src/storage/compressed_data_manager.rs @@ -29,6 +29,7 @@ use tracing::{debug, error, info}; use crate::configuration::WalMode; use crate::error::Result; use crate::storage::compressed_data_buffer::{CompressedDataBuffer, CompressedSegmentBatch}; +use crate::storage::data_storage_optimizer::DataStorageOptimizer; use crate::storage::data_transfer::DataTransfer; use crate::storage::types::Message; use crate::storage::types::{Channels, MemoryPool}; @@ -36,6 +37,9 @@ use crate::storage::types::{Channels, MemoryPool}; /// Stores data points compressed as segments containing metadata and models in memory to batch the /// compressed segments before saving them to Apache Parquet files. pub(super) struct CompressedDataManager { + /// Component that compacts the small compressed files that accumulate for a table into fewer + /// larger files and vacuums the files left behind by the compaction. + pub(super) data_storage_optimizer: Arc>, /// Component that transfers saved compressed data to the remote data folder when it is necessary. pub(super) data_transfer: Arc>>, /// Folder containing all compressed data managed by the [`StorageEngine`](crate::storage::StorageEngine). @@ -57,6 +61,7 @@ pub(super) struct CompressedDataManager { impl CompressedDataManager { pub(super) fn new( + data_storage_optimizer: Arc>, data_transfer: Arc>>, local_data_folder: DataFolder, channels: Arc, @@ -64,6 +69,7 @@ impl CompressedDataManager { wal_mode: WalMode, ) -> Self { Self { + data_storage_optimizer, data_transfer, local_data_folder, compressed_data_buffers: DashMap::new(), diff --git a/crates/modelardb_server/src/storage/mod.rs b/crates/modelardb_server/src/storage/mod.rs index 12266bbc..9002b5bf 100644 --- a/crates/modelardb_server/src/storage/mod.rs +++ b/crates/modelardb_server/src/storage/mod.rs @@ -43,6 +43,7 @@ use crate::configuration::{ConfigurationManager, WalMode}; use crate::data_folders::DataFolders; use crate::error::{ModelarDbServerError, Result}; use crate::storage::compressed_data_manager::CompressedDataManager; +use crate::storage::data_storage_optimizer::DataStorageOptimizer; use crate::storage::data_transfer::DataTransfer; use crate::storage::types::{Channels, MemoryPool, Message}; use crate::storage::uncompressed_data_buffer::IngestedDataBuffer; @@ -147,6 +148,13 @@ impl StorageEngine { } // Create the compressed data manager. + let data_storage_optimizer = DataStorageOptimizer::try_new( + data_folders.local_data_folder.clone(), + configuration_manager.optimize_target_file_size_in_bytes(), + configuration_manager.vacuum_retention_period_in_seconds(), + ) + .await?; + let data_transfer = if let Some(remote_data_folder) = data_folders.maybe_remote_data_folder { let data_transfer = DataTransfer::try_new( @@ -162,6 +170,7 @@ impl StorageEngine { }; let compressed_data_manager = Arc::new(CompressedDataManager::new( + Arc::new(RwLock::new(data_storage_optimizer)), Arc::new(RwLock::new(data_transfer)), data_folders.local_data_folder, channels.clone(), From 6a6073b67581783779c366d7d8e7454c2ec71ae1 Mon Sep 17 00:00:00 2001 From: CGodiksen <36046286+CGodiksen@users.noreply.github.com> Date: Wed, 5 Aug 2026 10:43:51 +0200 Subject: [PATCH 19/48] Call data storage optimizer when saving compressed data and fixed test --- .../src/storage/compressed_data_manager.rs | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/crates/modelardb_server/src/storage/compressed_data_manager.rs b/crates/modelardb_server/src/storage/compressed_data_manager.rs index ba6671df..2b621a0a 100644 --- a/crates/modelardb_server/src/storage/compressed_data_manager.rs +++ b/crates/modelardb_server/src/storage/compressed_data_manager.rs @@ -293,6 +293,14 @@ impl CompressedDataManager { self.memory_pool.remaining_compressed_memory_in_bytes() ); + // Optimize how the compressed data for table_name is stored on disk once enough new data + // has been written since the last optimization. + self.data_storage_optimizer + .read() + .await + .increase_estimated_compactable_size(table_name, compressed_data_buffer_size_in_bytes) + .await?; + Ok(()) } @@ -586,9 +594,18 @@ mod tests { .unwrap(), )); + let optimizer = DataStorageOptimizer::try_new( + local_data_folder.clone(), + 64 * 1024 * 1024, + 60 * 60 * 24 * 7, + ) + .await + .unwrap(); + ( temp_dir, CompressedDataManager::new( + Arc::new(RwLock::new(optimizer)), Arc::new(RwLock::new(None)), local_data_folder, channels, From e0060c3bfdc227b741b1c10cc9e654009301fcd7 Mon Sep 17 00:00:00 2001 From: CGodiksen <36046286+CGodiksen@users.noreply.github.com> Date: Wed, 5 Aug 2026 10:53:43 +0200 Subject: [PATCH 20/48] Fix clippy issue --- .../src/storage/data_storage_optimizer.rs | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/crates/modelardb_server/src/storage/data_storage_optimizer.rs b/crates/modelardb_server/src/storage/data_storage_optimizer.rs index c6fea8dc..e356b303 100644 --- a/crates/modelardb_server/src/storage/data_storage_optimizer.rs +++ b/crates/modelardb_server/src/storage/data_storage_optimizer.rs @@ -110,9 +110,7 @@ impl DataStorageOptimizer { let estimate_reached_target = *self .estimated_compactable_size_in_bytes .get(table_name) - .expect(&format!( - "{table_name} should have been added to estimated_compactable_size_in_bytes." - )) + .expect("table_name should have been added to estimated_compactable_size_in_bytes.") .value() >= self.optimize_target_file_size_in_bytes; @@ -144,9 +142,7 @@ impl DataStorageOptimizer { *self .estimated_compactable_size_in_bytes .get_mut(table_name) - .expect(&format!( - "{table_name} should be in estimated_compactable_size_in_bytes." - )) = 0; + .expect("table_name should be in estimated_compactable_size_in_bytes.") = 0; Ok(()) } From e8481057f4987ff09e3612647e0bcb7b9523b9d9 Mon Sep 17 00:00:00 2001 From: CGodiksen <36046286+CGodiksen@users.noreply.github.com> Date: Wed, 5 Aug 2026 20:01:36 +0200 Subject: [PATCH 21/48] Add methods to DataStorarageOptimizer to set configuration --- .../src/storage/data_storage_optimizer.rs | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/crates/modelardb_server/src/storage/data_storage_optimizer.rs b/crates/modelardb_server/src/storage/data_storage_optimizer.rs index e356b303..d49101f6 100644 --- a/crates/modelardb_server/src/storage/data_storage_optimizer.rs +++ b/crates/modelardb_server/src/storage/data_storage_optimizer.rs @@ -146,6 +146,24 @@ impl DataStorageOptimizer { Ok(()) } + + /// Set the target size, in bytes, of the files produced when a table is optimized to + /// `new_optimize_target_file_size_in_bytes`. + pub(super) fn set_optimize_target_file_size_in_bytes( + &mut self, + new_optimize_target_file_size_in_bytes: u64, + ) { + self.optimize_target_file_size_in_bytes = new_optimize_target_file_size_in_bytes; + } + + /// Set the retention period, in seconds, used when a table is vacuumed after it is optimized to + /// `new_vacuum_retention_period_in_seconds`. + pub(super) fn set_vacuum_retention_period_in_seconds( + &mut self, + new_vacuum_retention_period_in_seconds: u64, + ) { + self.vacuum_retention_period_in_seconds = new_vacuum_retention_period_in_seconds; + } } #[cfg(test)] From 5d89ae53f80b9797899f73a9116c38e264b77179 Mon Sep 17 00:00:00 2001 From: CGodiksen <36046286+CGodiksen@users.noreply.github.com> Date: Wed, 5 Aug 2026 20:02:49 +0200 Subject: [PATCH 22/48] Add methods to StorageEngine to set configuration --- crates/modelardb_server/src/storage/mod.rs | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/crates/modelardb_server/src/storage/mod.rs b/crates/modelardb_server/src/storage/mod.rs index 9002b5bf..254177e4 100644 --- a/crates/modelardb_server/src/storage/mod.rs +++ b/crates/modelardb_server/src/storage/mod.rs @@ -398,4 +398,23 @@ impl StorageEngine { )) } } + + /// Set the target file size used when automatically optimizing a table's storage to `new_value`. + pub(super) async fn set_optimize_target_file_size_in_bytes(&self, new_value: u64) { + self.compressed_data_manager + .data_storage_optimizer + .write() + .await + .set_optimize_target_file_size_in_bytes(new_value); + } + + /// Set the retention period used when automatically vacuuming a table after optimization to + /// `new_value`. + pub(super) async fn set_vacuum_retention_period_in_seconds(&self, new_value: u64) { + self.compressed_data_manager + .data_storage_optimizer + .write() + .await + .set_vacuum_retention_period_in_seconds(new_value); + } } From 94a8a4e31692b0c26e6295dca6d46d92dba4866d Mon Sep 17 00:00:00 2001 From: CGodiksen <36046286+CGodiksen@users.noreply.github.com> Date: Wed, 5 Aug 2026 20:06:07 +0200 Subject: [PATCH 23/48] Add method to update optimize_target_file_size_in_bytes --- crates/modelardb_server/src/configuration.rs | 29 ++++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/crates/modelardb_server/src/configuration.rs b/crates/modelardb_server/src/configuration.rs index 3068bed1..1935587a 100644 --- a/crates/modelardb_server/src/configuration.rs +++ b/crates/modelardb_server/src/configuration.rs @@ -424,6 +424,35 @@ impl ConfigurationManager { self.configuration.optimize_target_file_size_in_bytes } + /// Set the new value and update the target file size in the data storage optimizer. If the new + /// value is zero or the new configuration could not be saved to the configuration file, return + /// [`ModelarDbServerError`]. + #[allow(dead_code)] + pub(crate) async fn set_optimize_target_file_size_in_bytes( + &mut self, + new_optimize_target_file_size_in_bytes: u64, + storage_engine: Arc>, + ) -> Result<()> { + if new_optimize_target_file_size_in_bytes == 0 { + return Err(ModelarDbServerError::InvalidArgument( + "Optimize target file size must be greater than zero.".to_owned(), + )); + } + + storage_engine + .write() + .await + .set_optimize_target_file_size_in_bytes(new_optimize_target_file_size_in_bytes) + .await; + + self.configuration.optimize_target_file_size_in_bytes = + new_optimize_target_file_size_in_bytes; + + self.configuration + .save_to_toml(&self.local_data_folder) + .await + } + pub(crate) fn vacuum_retention_period_in_seconds(&self) -> u64 { self.configuration.vacuum_retention_period_in_seconds } From 2153c4439291b3f39f1757d3f11019e9f35c1da9 Mon Sep 17 00:00:00 2001 From: CGodiksen <36046286+CGodiksen@users.noreply.github.com> Date: Wed, 5 Aug 2026 20:07:10 +0200 Subject: [PATCH 24/48] Add method to update vacuum_retention_period_in_seconds --- crates/modelardb_server/src/configuration.rs | 29 +++++++++++++++++++- 1 file changed, 28 insertions(+), 1 deletion(-) diff --git a/crates/modelardb_server/src/configuration.rs b/crates/modelardb_server/src/configuration.rs index 1935587a..3fb0044a 100644 --- a/crates/modelardb_server/src/configuration.rs +++ b/crates/modelardb_server/src/configuration.rs @@ -427,7 +427,6 @@ impl ConfigurationManager { /// Set the new value and update the target file size in the data storage optimizer. If the new /// value is zero or the new configuration could not be saved to the configuration file, return /// [`ModelarDbServerError`]. - #[allow(dead_code)] pub(crate) async fn set_optimize_target_file_size_in_bytes( &mut self, new_optimize_target_file_size_in_bytes: u64, @@ -457,6 +456,34 @@ impl ConfigurationManager { self.configuration.vacuum_retention_period_in_seconds } + /// Set the new value and update the retention period in the data storage optimizer. If the new + /// value is larger than [`MAX_RETENTION_PERIOD_IN_SECONDS`] or the new configuration could not + /// be saved to the configuration file, return [`ModelarDbServerError`]. + pub(crate) async fn set_vacuum_retention_period_in_seconds( + &mut self, + new_vacuum_retention_period_in_seconds: u64, + storage_engine: Arc>, + ) -> Result<()> { + if new_vacuum_retention_period_in_seconds > MAX_RETENTION_PERIOD_IN_SECONDS { + return Err(ModelarDbServerError::InvalidArgument(format!( + "Vacuum retention period cannot be more than {MAX_RETENTION_PERIOD_IN_SECONDS} seconds." + ))); + } + + storage_engine + .write() + .await + .set_vacuum_retention_period_in_seconds(new_vacuum_retention_period_in_seconds) + .await; + + self.configuration.vacuum_retention_period_in_seconds = + new_vacuum_retention_period_in_seconds; + + self.configuration + .save_to_toml(&self.local_data_folder) + .await + } + pub(crate) fn ingestion_threads(&self) -> u8 { self.configuration.ingestion_threads } From 55a63cd18ed2cd374c9b9b4c2b0d5620142f8474 Mon Sep 17 00:00:00 2001 From: CGodiksen <36046286+CGodiksen@users.noreply.github.com> Date: Wed, 5 Aug 2026 20:09:55 +0200 Subject: [PATCH 25/48] Add test for setting optimize_target_file_size_in_bytes --- crates/modelardb_server/src/configuration.rs | 36 ++++++++++++++++++++ 1 file changed, 36 insertions(+) diff --git a/crates/modelardb_server/src/configuration.rs b/crates/modelardb_server/src/configuration.rs index 3fb0044a..680b5707 100644 --- a/crates/modelardb_server/src/configuration.rs +++ b/crates/modelardb_server/src/configuration.rs @@ -873,6 +873,42 @@ mod tests { ); } + #[tokio::test] + async fn test_set_optimize_target_file_size_in_bytes() { + let temp_dir = tempfile::tempdir().unwrap(); + let (storage_engine, configuration_manager) = create_components(&temp_dir).await; + + assert_eq!( + configuration_manager + .read() + .await + .optimize_target_file_size_in_bytes(), + 64 * 1024 * 1024 + ); + + let new_value = 1024; + configuration_manager + .write() + .await + .set_optimize_target_file_size_in_bytes(new_value, storage_engine) + .await + .unwrap(); + + assert_eq!( + configuration_manager + .read() + .await + .optimize_target_file_size_in_bytes(), + new_value + ); + + let configuration_from_file = configuration_from_file(&temp_dir).await; + assert_eq!( + configuration_from_file.optimize_target_file_size_in_bytes, + new_value + ); + } + /// Return the configuration from the configuration file at the root of `temp_dir`. async fn configuration_from_file(temp_dir: &TempDir) -> Configuration { let configuration_file_path = temp_dir.path().join(CONFIGURATION_FILE_NAME); From fd822e780dfe93e872b5ccd2d3c8f90a7c1f80fc Mon Sep 17 00:00:00 2001 From: CGodiksen <36046286+CGodiksen@users.noreply.github.com> Date: Wed, 5 Aug 2026 20:10:24 +0200 Subject: [PATCH 26/48] Add test for rejecting 0 --- crates/modelardb_server/src/configuration.rs | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/crates/modelardb_server/src/configuration.rs b/crates/modelardb_server/src/configuration.rs index 680b5707..1b3db945 100644 --- a/crates/modelardb_server/src/configuration.rs +++ b/crates/modelardb_server/src/configuration.rs @@ -909,6 +909,23 @@ mod tests { ); } + #[tokio::test] + async fn test_set_optimize_target_file_size_in_bytes_rejects_zero() { + let temp_dir = tempfile::tempdir().unwrap(); + let (storage_engine, configuration_manager) = create_components(&temp_dir).await; + + let result = configuration_manager + .write() + .await + .set_optimize_target_file_size_in_bytes(0, storage_engine) + .await; + + assert_eq!( + result.unwrap_err().to_string(), + "Invalid State Error: Optimize target file size must be greater than zero." + ); + } + /// Return the configuration from the configuration file at the root of `temp_dir`. async fn configuration_from_file(temp_dir: &TempDir) -> Configuration { let configuration_file_path = temp_dir.path().join(CONFIGURATION_FILE_NAME); From f54e21a0763bf2e6fc851808b5c11ffefa577013 Mon Sep 17 00:00:00 2001 From: CGodiksen <36046286+CGodiksen@users.noreply.github.com> Date: Wed, 5 Aug 2026 20:11:37 +0200 Subject: [PATCH 27/48] Add test for setting vacuum_retention_period_in_seconds --- crates/modelardb_server/src/configuration.rs | 35 ++++++++++++++++++++ 1 file changed, 35 insertions(+) diff --git a/crates/modelardb_server/src/configuration.rs b/crates/modelardb_server/src/configuration.rs index 1b3db945..d0948c5d 100644 --- a/crates/modelardb_server/src/configuration.rs +++ b/crates/modelardb_server/src/configuration.rs @@ -926,6 +926,41 @@ mod tests { ); } + #[tokio::test] + async fn test_set_vacuum_retention_period_in_seconds() { + let temp_dir = tempfile::tempdir().unwrap(); + let (storage_engine, configuration_manager) = create_components(&temp_dir).await; + + assert_eq!( + configuration_manager + .read() + .await + .vacuum_retention_period_in_seconds(), + 60 * 60 * 24 * 7 + ); + + let new_value = 60; + configuration_manager + .write() + .await + .set_vacuum_retention_period_in_seconds(new_value, storage_engine) + .await + .unwrap(); + + assert_eq!( + configuration_manager + .read() + .await + .vacuum_retention_period_in_seconds(), + new_value + ); + + let configuration_from_file = configuration_from_file(&temp_dir).await; + assert_eq!( + configuration_from_file.vacuum_retention_period_in_seconds, + new_value + ); + } /// Return the configuration from the configuration file at the root of `temp_dir`. async fn configuration_from_file(temp_dir: &TempDir) -> Configuration { let configuration_file_path = temp_dir.path().join(CONFIGURATION_FILE_NAME); From 4870cae406b958ccff6d4e5c7c4a2775e42d731a Mon Sep 17 00:00:00 2001 From: CGodiksen <36046286+CGodiksen@users.noreply.github.com> Date: Wed, 5 Aug 2026 20:12:36 +0200 Subject: [PATCH 28/48] Add test for setting too large vacuum_retention_period_in_seconds --- crates/modelardb_server/src/configuration.rs | 26 +++++++++++++++++++- 1 file changed, 25 insertions(+), 1 deletion(-) diff --git a/crates/modelardb_server/src/configuration.rs b/crates/modelardb_server/src/configuration.rs index d0948c5d..fdeec2df 100644 --- a/crates/modelardb_server/src/configuration.rs +++ b/crates/modelardb_server/src/configuration.rs @@ -922,7 +922,7 @@ mod tests { assert_eq!( result.unwrap_err().to_string(), - "Invalid State Error: Optimize target file size must be greater than zero." + "Invalid Argument Error: Optimize target file size must be greater than zero." ); } @@ -961,6 +961,30 @@ mod tests { new_value ); } + + #[tokio::test] + async fn test_set_vacuum_retention_period_in_seconds_rejects_too_large_value() { + let temp_dir = tempfile::tempdir().unwrap(); + let (storage_engine, configuration_manager) = create_components(&temp_dir).await; + + let result = configuration_manager + .write() + .await + .set_vacuum_retention_period_in_seconds( + MAX_RETENTION_PERIOD_IN_SECONDS + 1, + storage_engine, + ) + .await; + + assert_eq!( + result.unwrap_err().to_string(), + format!( + "Invalid Argument Error: Vacuum retention period cannot be more than {} seconds.", + MAX_RETENTION_PERIOD_IN_SECONDS + ) + ); + } + /// Return the configuration from the configuration file at the root of `temp_dir`. async fn configuration_from_file(temp_dir: &TempDir) -> Configuration { let configuration_file_path = temp_dir.path().join(CONFIGURATION_FILE_NAME); From 3ebdc79043a5f518b6692637c42119f2c6903562 Mon Sep 17 00:00:00 2001 From: CGodiksen <36046286+CGodiksen@users.noreply.github.com> Date: Wed, 5 Aug 2026 20:21:39 +0200 Subject: [PATCH 29/48] Add match arms to update data storage optimizer config --- crates/modelardb_server/src/remote/mod.rs | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/crates/modelardb_server/src/remote/mod.rs b/crates/modelardb_server/src/remote/mod.rs index f833d4ce..2fac431d 100644 --- a/crates/modelardb_server/src/remote/mod.rs +++ b/crates/modelardb_server/src/remote/mod.rs @@ -1021,6 +1021,22 @@ impl FlightService for FlightServiceHandler { .await .map_err(error_to_status_internal) } + Ok(protocol::update_configuration::Setting::OptimizeTargetFileSizeInBytes) => { + let new_value = maybe_new_value.ok_or(invalid_null_error)?; + + configuration_manager + .set_optimize_target_file_size_in_bytes(new_value, storage_engine) + .await + .map_err(error_to_status_internal) + } + Ok(protocol::update_configuration::Setting::VacuumRetentionPeriodInSeconds) => { + let new_value = maybe_new_value.ok_or(invalid_null_error)?; + + configuration_manager + .set_vacuum_retention_period_in_seconds(new_value, storage_engine) + .await + .map_err(error_to_status_internal) + } _ => Err(Status::unimplemented(format!( "{setting} is not an updatable setting in the server configuration." ))), From aba2653a56bf771ecb24770d1ae7411d8c25284f Mon Sep 17 00:00:00 2001 From: CGodiksen <36046286+CGodiksen@users.noreply.github.com> Date: Wed, 5 Aug 2026 20:30:06 +0200 Subject: [PATCH 30/48] Use local defaults if a target size or retention period is not specified --- crates/modelardb_server/src/remote/mod.rs | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/crates/modelardb_server/src/remote/mod.rs b/crates/modelardb_server/src/remote/mod.rs index 2fac431d..a9c1299d 100644 --- a/crates/modelardb_server/src/remote/mod.rs +++ b/crates/modelardb_server/src/remote/mod.rs @@ -471,9 +471,15 @@ impl FlightServiceHandler { } } + // If the retention period is not specified, use the local configuration retention period. + // The local retention period is not passed to the peer nodes above, so they can use their + // own local retention period. + let retention_period_in_seconds = maybe_retention_period_in_seconds + .unwrap_or(configuration_manager.vacuum_retention_period_in_seconds()); + for table_name in table_names { self.context - .vacuum_table(table_name, maybe_retention_period_in_seconds) + .vacuum_table(table_name, Some(retention_period_in_seconds)) .await .map_err(error_to_status_invalid_argument)?; } @@ -503,9 +509,15 @@ impl FlightServiceHandler { } } + // If the target size is not specified, use the local configuration target size. The local + // target size is not passed to the peer nodes above, so they can use their own local target + // size. + let target_size_in_bytes = maybe_target_size_in_bytes + .unwrap_or(configuration_manager.optimize_target_file_size_in_bytes()); + for table_name in table_names { self.context - .optimize_table(table_name, maybe_target_size_in_bytes) + .optimize_table(table_name, Some(target_size_in_bytes)) .await .map_err(error_to_status_invalid_argument)?; } From c00e623d2932b939834d5970ba268fcb16a17747 Mon Sep 17 00:00:00 2001 From: CGodiksen <36046286+CGodiksen@users.noreply.github.com> Date: Wed, 5 Aug 2026 20:33:11 +0200 Subject: [PATCH 31/48] Add update config tests to integration tests --- .../tests/integration_test.rs | 22 +++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/crates/modelardb_server/tests/integration_test.rs b/crates/modelardb_server/tests/integration_test.rs index 164bbb84..f03c854f 100644 --- a/crates/modelardb_server/tests/integration_test.rs +++ b/crates/modelardb_server/tests/integration_test.rs @@ -1481,6 +1481,26 @@ async fn test_can_update_segment_size_threshold_in_bytes() { assert_eq!(updated_configuration.segment_size_threshold_in_bytes, 1); } +#[tokio::test] +async fn test_can_update_optimize_target_file_size_in_bytes() { + let updated_configuration = update_and_get_configuration( + protocol::update_configuration::Setting::OptimizeTargetFileSizeInBytes as i32, + ) + .await; + + assert_eq!(updated_configuration.optimize_target_file_size_in_bytes, 1); +} + +#[tokio::test] +async fn test_can_update_vacuum_retention_period_in_seconds() { + let updated_configuration = update_and_get_configuration( + protocol::update_configuration::Setting::VacuumRetentionPeriodInSeconds as i32, + ) + .await; + + assert_eq!(updated_configuration.vacuum_retention_period_in_seconds, 1); +} + async fn update_and_get_configuration(setting: i32) -> protocol::Configuration { let mut test_context = TestContext::new().await; test_context @@ -1521,6 +1541,8 @@ async fn test_cannot_update_non_nullable_setting_with_null_value() { protocol::update_configuration::Setting::UncompressedReservedMemoryInBytes as i32, protocol::update_configuration::Setting::CompressedReservedMemoryInBytes as i32, protocol::update_configuration::Setting::SegmentSizeThresholdInBytes as i32, + protocol::update_configuration::Setting::OptimizeTargetFileSizeInBytes as i32, + protocol::update_configuration::Setting::VacuumRetentionPeriodInSeconds as i32, ] { update_configuration_and_assert_error( setting, From bab1295ae9b6bd59849650aabe20c848159935eb Mon Sep 17 00:00:00 2001 From: CGodiksen <36046286+CGodiksen@users.noreply.github.com> Date: Wed, 5 Aug 2026 20:47:50 +0200 Subject: [PATCH 32/48] Make it clearer that the config is used as defaults --- crates/modelardb_server/src/configuration.rs | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/crates/modelardb_server/src/configuration.rs b/crates/modelardb_server/src/configuration.rs index fdeec2df..b92fe4ec 100644 --- a/crates/modelardb_server/src/configuration.rs +++ b/crates/modelardb_server/src/configuration.rs @@ -424,9 +424,9 @@ impl ConfigurationManager { self.configuration.optimize_target_file_size_in_bytes } - /// Set the new value and update the target file size in the data storage optimizer. If the new - /// value is zero or the new configuration could not be saved to the configuration file, return - /// [`ModelarDbServerError`]. + /// Set the target file size used by the data storage optimizer and as the default for OPTIMIZE + /// queries without an explicit target size. If the new value is zero or the new configuration + /// could not be saved to the configuration file, return [`ModelarDbServerError`]. pub(crate) async fn set_optimize_target_file_size_in_bytes( &mut self, new_optimize_target_file_size_in_bytes: u64, @@ -456,9 +456,10 @@ impl ConfigurationManager { self.configuration.vacuum_retention_period_in_seconds } - /// Set the new value and update the retention period in the data storage optimizer. If the new - /// value is larger than [`MAX_RETENTION_PERIOD_IN_SECONDS`] or the new configuration could not - /// be saved to the configuration file, return [`ModelarDbServerError`]. + /// Set the retention period used by the data storage optimizer and as the default for VACUUM + /// queries without an explicit retention period. If the new value is larger than + /// [`MAX_RETENTION_PERIOD_IN_SECONDS`] or the new configuration could not be saved to the + /// configuration file, return [`ModelarDbServerError`]. pub(crate) async fn set_vacuum_retention_period_in_seconds( &mut self, new_vacuum_retention_period_in_seconds: u64, From b7ba75f455699676bf88527280adfa3dc8d43653 Mon Sep 17 00:00:00 2001 From: CGodiksen <36046286+CGodiksen@users.noreply.github.com> Date: Wed, 5 Aug 2026 21:00:40 +0200 Subject: [PATCH 33/48] Add test to WAL to ensure that optimize and vacuum does not mess with persisted batch ids --- .../modelardb_storage/src/write_ahead_log.rs | 45 +++++++++++++++++++ 1 file changed, 45 insertions(+) diff --git a/crates/modelardb_storage/src/write_ahead_log.rs b/crates/modelardb_storage/src/write_ahead_log.rs index 34748cd0..8880fc06 100644 --- a/crates/modelardb_storage/src/write_ahead_log.rs +++ b/crates/modelardb_storage/src/write_ahead_log.rs @@ -779,6 +779,51 @@ mod tests { assert_eq!(*persisted, BTreeSet::from([0, 1, 2])); } + #[tokio::test] + async fn test_wal_recovery_survives_optimize_and_vacuum() { + let (temp_dir, data_folder) = create_data_folder_with_time_series_table().await; + + // Write three separate commits, each carrying its own batch ids. + write_compressed_segments_with_batch_ids(&data_folder, HashSet::from([0, 1, 2])).await; + write_compressed_segments_with_batch_ids(&data_folder, HashSet::from([3, 4, 5])).await; + write_compressed_segments_with_batch_ids(&data_folder, HashSet::from([6, 7, 8])).await; + + let column_path = format!( + "{}/tables/{}/field_column=0", + temp_dir.path().to_str().unwrap(), + TIME_SERIES_TABLE_NAME + ); + assert_eq!(std::fs::read_dir(&column_path).unwrap().count(), 3); + + // Optimize compacts the small files into one, and vacuum physically deletes the stale files + // left behind. Vacuum should only remove Parquet data files, never the _delta_log commits. + data_folder + .optimize_table(TIME_SERIES_TABLE_NAME, None) + .await + .unwrap(); + data_folder + .vacuum_table(TIME_SERIES_TABLE_NAME, Some(0)) + .await + .unwrap(); + + // Only the single compacted Parquet file should remain on disk. + assert_eq!(std::fs::read_dir(&column_path).unwrap().count(), 1); + + // Rebuilding the WAL from the same folder must still recover every persisted batch id from + // the Delta commit history. This proves optimize and vacuum did not discard the commits + // that crash recovery relies on to exclude already persisted data from replay. + let wal = WriteAheadLog::try_new(&data_folder, SEGMENT_SIZE_THRESHOLD_IN_BYTES) + .await + .unwrap(); + + let persisted = wal.table_logs[TIME_SERIES_TABLE_NAME] + .persisted_batch_ids + .lock() + .unwrap(); + + assert_eq!(*persisted, BTreeSet::from([0, 1, 2, 3, 4, 5, 6, 7, 8])); + } + #[tokio::test] async fn test_try_new_fails_for_non_local_data_folder() { let data_folder = DataFolder::open_memory().await.unwrap(); From f88688ff3714e9ba293094663983c6558501b363 Mon Sep 17 00:00:00 2001 From: CGodiksen <36046286+CGodiksen@users.noreply.github.com> Date: Wed, 5 Aug 2026 21:28:52 +0200 Subject: [PATCH 34/48] Add new configs to test_can_get_configuration --- crates/modelardb_server/tests/integration_test.rs | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/crates/modelardb_server/tests/integration_test.rs b/crates/modelardb_server/tests/integration_test.rs index f03c854f..d6f46a78 100644 --- a/crates/modelardb_server/tests/integration_test.rs +++ b/crates/modelardb_server/tests/integration_test.rs @@ -1429,6 +1429,14 @@ async fn test_can_get_configuration() { configuration.segment_size_threshold_in_bytes, 64 * 1024 * 1024 ); + assert_eq!( + configuration.optimize_target_file_size_in_bytes, + 64 * 1024 * 1024 + ); + assert_eq!( + configuration.vacuum_retention_period_in_seconds, + 60 * 60 * 24 * 7 + ); assert_eq!(configuration.ingestion_threads, 1); assert_eq!(configuration.compression_threads, 1); assert_eq!(configuration.writer_threads, 1); From bdf6c60fa9242a7b7df4fc255963cc8112318033 Mon Sep 17 00:00:00 2001 From: CGodiksen <36046286+CGodiksen@users.noreply.github.com> Date: Wed, 5 Aug 2026 21:32:52 +0200 Subject: [PATCH 35/48] Add test to ensure big files are not included on startup --- .../src/storage/data_storage_optimizer.rs | 20 +++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/crates/modelardb_server/src/storage/data_storage_optimizer.rs b/crates/modelardb_server/src/storage/data_storage_optimizer.rs index d49101f6..cd2e70bd 100644 --- a/crates/modelardb_server/src/storage/data_storage_optimizer.rs +++ b/crates/modelardb_server/src/storage/data_storage_optimizer.rs @@ -203,6 +203,26 @@ mod tests { ); } + #[tokio::test] + async fn test_initialize_estimate_excludes_files_at_or_above_target() { + let (_temp_dir, local_data_folder) = create_local_data_folder_with_table().await; + write_batches_to_table(&local_data_folder, 3).await; + + // With a one-byte target, every existing file is already at or above the target, so none of + // them count towards the compactable backlog. + let optimizer = DataStorageOptimizer::try_new(local_data_folder.clone(), 1, 0) + .await + .unwrap(); + + assert_eq!( + *optimizer + .estimated_compactable_size_in_bytes + .get(TIME_SERIES_TABLE_NAME) + .unwrap(), + 0 + ); + } + // Tests for increase_estimated_compactable_size(). #[tokio::test] async fn test_optimize_table_when_estimate_reaches_target() { From 087cec962c5b8d252805d80478f69c309ba2efb5 Mon Sep 17 00:00:00 2001 From: CGodiksen <36046286+CGodiksen@users.noreply.github.com> Date: Thu, 6 Aug 2026 07:33:23 +0200 Subject: [PATCH 36/48] Use the physical file count in data storage optimizer tests --- .../src/storage/data_storage_optimizer.rs | 28 +++++++++---------- 1 file changed, 13 insertions(+), 15 deletions(-) diff --git a/crates/modelardb_server/src/storage/data_storage_optimizer.rs b/crates/modelardb_server/src/storage/data_storage_optimizer.rs index cd2e70bd..95897051 100644 --- a/crates/modelardb_server/src/storage/data_storage_optimizer.rs +++ b/crates/modelardb_server/src/storage/data_storage_optimizer.rs @@ -231,7 +231,7 @@ mod tests { write_batches_to_table(&local_data_folder, 3).await; - let initial_file_count = table_file_count(&local_data_folder).await; + let initial_file_count = table_file_count(&local_data_folder); assert_eq!(initial_file_count, 3); optimizer @@ -243,7 +243,7 @@ mod tests { .unwrap(); // The small files should have been compacted into a single file. - assert_eq!(table_file_count(&local_data_folder).await, 1); + assert_eq!(table_file_count(&local_data_folder), 1); // The estimate should have been reset after optimizing. assert_eq!( @@ -262,7 +262,7 @@ mod tests { write_batches_to_table(&local_data_folder, 3).await; - let initial_file_count = table_file_count(&local_data_folder).await; + let initial_file_count = table_file_count(&local_data_folder); assert_eq!(initial_file_count, 3); optimizer @@ -274,10 +274,7 @@ mod tests { .unwrap(); // No files should have been compacted since the estimate did not reach the target. - assert_eq!( - table_file_count(&local_data_folder).await, - initial_file_count - ); + assert_eq!(table_file_count(&local_data_folder), initial_file_count); // The estimate should have accumulated without being reset. assert_eq!( @@ -318,15 +315,16 @@ mod tests { } } - /// Return the number of active files in the time series table in `local_data_folder`. - async fn table_file_count(local_data_folder: &DataFolder) -> usize { - let mut delta_table = local_data_folder - .delta_table(TIME_SERIES_TABLE_NAME) - .await - .unwrap(); - delta_table.load().await.unwrap(); + /// Return the number of physical Apache Parquet files in the time series table in + /// `local_data_folder`. + fn table_file_count(local_data_folder: &DataFolder) -> usize { + let column_path = format!( + "{}/tables/{}/field_column=0", + local_data_folder.location(), + TIME_SERIES_TABLE_NAME + ); - delta_table.get_file_uris().unwrap().count() + std::fs::read_dir(column_path).unwrap().count() } /// Create a [`DataStorageOptimizer`] that optimizes the tables in `local_data_folder`. From e2dfe54c0915e1b2a30b2ffbf8ec7ebcc1135ad9 Mon Sep 17 00:00:00 2001 From: CGodiksen <36046286+CGodiksen@users.noreply.github.com> Date: Sat, 8 Aug 2026 08:03:17 +0200 Subject: [PATCH 37/48] Update based on comments from @skejserjensen --- crates/modelardb_server/src/main.rs | 2 +- .../src/storage/data_storage_optimizer.rs | 32 +++++++++++-------- .../modelardb_types/src/flight/protocol.proto | 2 +- docs/user/README.md | 24 +++++++------- 4 files changed, 33 insertions(+), 27 deletions(-) diff --git a/crates/modelardb_server/src/main.rs b/crates/modelardb_server/src/main.rs index d0bcf616..1d10fce8 100644 --- a/crates/modelardb_server/src/main.rs +++ b/crates/modelardb_server/src/main.rs @@ -92,7 +92,7 @@ pub(crate) struct ServerArgs { /// Retention period, in seconds, used when automatically vacuuming a table after optimization. /// This is also the default value used when a VACUUM query is executed without an explicit - /// retention period. Note that a very low value can delete files an in-flight query is still + /// retention period. Note that a very low value can delete files an in-progress query is still /// scanning. #[arg(long, env = "MODELARDBD_VACUUM_RETENTION_PERIOD_IN_SECONDS")] vacuum_retention_period_in_seconds: Option, diff --git a/crates/modelardb_server/src/storage/data_storage_optimizer.rs b/crates/modelardb_server/src/storage/data_storage_optimizer.rs index 95897051..ecd2b77b 100644 --- a/crates/modelardb_server/src/storage/data_storage_optimizer.rs +++ b/crates/modelardb_server/src/storage/data_storage_optimizer.rs @@ -32,11 +32,12 @@ pub(super) struct DataStorageOptimizer { /// The data folder containing all compressed data managed by the /// [`StorageEngine`](crate::storage::StorageEngine). local_data_folder: DataFolder, - /// The target size, in bytes, of the files produced when a table is optimized. Also used as the - /// trigger for when a table is optimized. + /// The target size, in bytes, of the files produced when a table is optimized. A table is + /// optimized once its `estimated_compactable_size_in_bytes` reaches this size, so the same + /// value decides both when to optimize and how large the resulting files are. optimize_target_file_size_in_bytes: u64, /// The retention period, in seconds, used when a table is vacuumed after it is optimized. - /// Note that a very low value can let the vacuum physically delete files that an in-flight + /// Note that a very low value can let the vacuum physically delete files that an in-progress /// query is still scanning, causing that query to fail. vacuum_retention_period_in_seconds: u64, /// Map from table names to an estimate of how many bytes each table has in files smaller than @@ -124,7 +125,7 @@ impl DataStorageOptimizer { /// Compact the small files of the table with `table_name` into files of approximately /// `optimize_target_file_size_in_bytes` bytes, vacuum the small files left behind, and reset /// the table's estimated compactable size. Note that the vacuum can physically delete files - /// that an in-flight query is still scanning if `vacuum_retention_period_in_seconds` is very + /// that an in-progress query is still scanning if `vacuum_retention_period_in_seconds` is very /// low. Returns [`Ok`] if the table was optimized successfully, otherwise /// [`ModelarDbServerError`](crate::error::ModelarDbServerError). async fn optimize_and_vacuum_table(&self, table_name: &str) -> Result<()> { @@ -148,7 +149,9 @@ impl DataStorageOptimizer { } /// Set the target size, in bytes, of the files produced when a table is optimized to - /// `new_optimize_target_file_size_in_bytes`. + /// `new_optimize_target_file_size_in_bytes`. The new target takes effect the next time each + /// table is written to. Tables are not re-optimized here to keep configuration updates cheap + /// and to avoid having to re-check all files on disk to see if they are compactable. pub(super) fn set_optimize_target_file_size_in_bytes( &mut self, new_optimize_target_file_size_in_bytes: u64, @@ -175,12 +178,13 @@ mod tests { const OPTIMIZE_TARGET_FILE_SIZE_IN_BYTES: u64 = 1024 * 1024; const VACUUM_RETENTION_PERIOD_IN_SECONDS: u64 = 0; + const BATCH_COUNT: u8 = 3; // Tests for try_new(). #[tokio::test] async fn test_initialize_estimate_from_existing_small_files() { let (_temp_dir, local_data_folder) = create_local_data_folder_with_table().await; - write_batches_to_table(&local_data_folder, 3).await; + write_batches_to_table(&local_data_folder, BATCH_COUNT).await; // The optimizer is created after the data is written, so its estimate includes the small // files already on disk. @@ -206,7 +210,7 @@ mod tests { #[tokio::test] async fn test_initialize_estimate_excludes_files_at_or_above_target() { let (_temp_dir, local_data_folder) = create_local_data_folder_with_table().await; - write_batches_to_table(&local_data_folder, 3).await; + write_batches_to_table(&local_data_folder, BATCH_COUNT).await; // With a one-byte target, every existing file is already at or above the target, so none of // them count towards the compactable backlog. @@ -214,6 +218,8 @@ mod tests { .await .unwrap(); + assert_eq!(table_file_count(&local_data_folder), BATCH_COUNT); + assert_eq!( *optimizer .estimated_compactable_size_in_bytes @@ -229,10 +235,10 @@ mod tests { let (_temp_dir, local_data_folder) = create_local_data_folder_with_table().await; let optimizer = create_data_storage_optimizer(local_data_folder.clone()).await; - write_batches_to_table(&local_data_folder, 3).await; + write_batches_to_table(&local_data_folder, BATCH_COUNT).await; let initial_file_count = table_file_count(&local_data_folder); - assert_eq!(initial_file_count, 3); + assert_eq!(initial_file_count, BATCH_COUNT); optimizer .increase_estimated_compactable_size( @@ -260,10 +266,10 @@ mod tests { let (_temp_dir, local_data_folder) = create_local_data_folder_with_table().await; let optimizer = create_data_storage_optimizer(local_data_folder.clone()).await; - write_batches_to_table(&local_data_folder, 3).await; + write_batches_to_table(&local_data_folder, BATCH_COUNT).await; let initial_file_count = table_file_count(&local_data_folder); - assert_eq!(initial_file_count, 3); + assert_eq!(initial_file_count, BATCH_COUNT); optimizer .increase_estimated_compactable_size( @@ -317,14 +323,14 @@ mod tests { /// Return the number of physical Apache Parquet files in the time series table in /// `local_data_folder`. - fn table_file_count(local_data_folder: &DataFolder) -> usize { + fn table_file_count(local_data_folder: &DataFolder) -> u8 { let column_path = format!( "{}/tables/{}/field_column=0", local_data_folder.location(), TIME_SERIES_TABLE_NAME ); - std::fs::read_dir(column_path).unwrap().count() + std::fs::read_dir(column_path).unwrap().count() as u8 } /// Create a [`DataStorageOptimizer`] that optimizes the tables in `local_data_folder`. diff --git a/crates/modelardb_types/src/flight/protocol.proto b/crates/modelardb_types/src/flight/protocol.proto index 6455533b..cfd84af4 100644 --- a/crates/modelardb_types/src/flight/protocol.proto +++ b/crates/modelardb_types/src/flight/protocol.proto @@ -86,7 +86,7 @@ message Configuration { // Retention period, in seconds, used when automatically vacuuming a table after optimization. This is also the // default value used when a VACUUM query is executed without an explicit retention period. Note that a very low - // value can delete files an in-flight query is still scanning. + // value can delete files an in-progress query is still scanning. uint64 vacuum_retention_period_in_seconds = 11; } diff --git a/docs/user/README.md b/docs/user/README.md index a7004e9b..b8415a7e 100644 --- a/docs/user/README.md +++ b/docs/user/README.md @@ -380,18 +380,18 @@ the built-in defaults. Note that the connection settings `--host` and `--port` a file. Variables marked with ✓ in the **Updatable** column can also be updated while the server is running using the `UpdateConfiguration` action without requiring a restart. The update is persisted in the configuration file. -| **CLI Flag** | **Environment Variable** | **Default** | **Updatable** | **Description** | -|-------------------------------------------|----------------------------------------------------|-------------|---------------|--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| -| `--host` | `MODELARDBD_HOST` | 127.0.0.1 | | The host address of the `modelardbd` server. | -| `--port` | `MODELARDBD_PORT` | 9999 | | The port of the `modelardbd` server. | -| `--wal-enabled` | `MODELARDBD_WAL_ENABLED` | true | | Whether the write-ahead log is enabled. | -| `--multivariate-reserved-memory-in-bytes` | `MODELARDBD_MULTIVARIATE_RESERVED_MEMORY_IN_BYTES` | 512 MB | ✓ | The amount of memory to reserve for storing multivariate time series. | -| `--uncompressed-reserved-memory-in-bytes` | `MODELARDBD_UNCOMPRESSED_RESERVED_MEMORY_IN_BYTES` | 512 MB | ✓ | The amount of memory to reserve for storing uncompressed data buffers. | -| `--compressed-reserved-memory-in-bytes` | `MODELARDBD_COMPRESSED_RESERVED_MEMORY_IN_BYTES` | 512 MB | ✓ | The amount of memory to reserve for storing compressed data buffers. | -| `--transfer-batch-size-in-bytes` | `MODELARDBD_TRANSFER_BATCH_SIZE_IN_BYTES` | 64 MB | ✓ | The amount of data that must be collected before transferring a batch to the remote object store. | -| `--segment-size-threshold-in-bytes` | `MODELARDBD_SEGMENT_SIZE_THRESHOLD_IN_BYTES` | 64 MB | ✓ | The approximate maximum size of a single WAL segment file before a new one is started. Only updatable when the WAL is enabled. | -| `--optimize-target-file-size-in-bytes` | `MODELARDBD_OPTIMIZE_TARGET_FILE_SIZE_IN_BYTES` | 64 MB | ✓ | The target size of the files produced when automatically optimizing a table's storage. Also used as the default when `OPTIMIZE` is run without an explicit `TARGET`. | -| `--vacuum-retention-period-in-seconds` | `MODELARDBD_VACUUM_RETENTION_PERIOD_IN_SECONDS` | 7 days | ✓ | The retention period used when automatically vacuuming a table after optimization. Also used as the default when `VACUUM` is run without an explicit `RETAIN`. Note that a very low value can delete files an in-flight query is still scanning. | +| **CLI Flag** | **Environment Variable** | **Default** | **Updatable** | **Description** | +|-------------------------------------------|----------------------------------------------------|-------------|---------------|----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| `--host` | `MODELARDBD_HOST` | 127.0.0.1 | | The host address of the `modelardbd` server. | +| `--port` | `MODELARDBD_PORT` | 9999 | | The port of the `modelardbd` server. | +| `--wal-enabled` | `MODELARDBD_WAL_ENABLED` | true | | Whether the write-ahead log is enabled. | +| `--multivariate-reserved-memory-in-bytes` | `MODELARDBD_MULTIVARIATE_RESERVED_MEMORY_IN_BYTES` | 512 MB | ✓ | The amount of memory to reserve for storing multivariate time series. | +| `--uncompressed-reserved-memory-in-bytes` | `MODELARDBD_UNCOMPRESSED_RESERVED_MEMORY_IN_BYTES` | 512 MB | ✓ | The amount of memory to reserve for storing uncompressed data buffers. | +| `--compressed-reserved-memory-in-bytes` | `MODELARDBD_COMPRESSED_RESERVED_MEMORY_IN_BYTES` | 512 MB | ✓ | The amount of memory to reserve for storing compressed data buffers. | +| `--transfer-batch-size-in-bytes` | `MODELARDBD_TRANSFER_BATCH_SIZE_IN_BYTES` | 64 MB | ✓ | The amount of data that must be collected before transferring a batch to the remote object store. | +| `--segment-size-threshold-in-bytes` | `MODELARDBD_SEGMENT_SIZE_THRESHOLD_IN_BYTES` | 64 MB | ✓ | The approximate maximum size of a single WAL segment file before a new one is started. Only updatable when the WAL is enabled. | +| `--optimize-target-file-size-in-bytes` | `MODELARDBD_OPTIMIZE_TARGET_FILE_SIZE_IN_BYTES` | 64 MB | ✓ | The target size of the files produced when automatically optimizing a table's storage. Also used as the default when `OPTIMIZE` is run without an explicit `TARGET`. | +| `--vacuum-retention-period-in-seconds` | `MODELARDBD_VACUUM_RETENTION_PERIOD_IN_SECONDS` | 7 days | ✓ | The retention period used when automatically vacuuming a table after optimization. Also used as the default when `VACUUM` is run without an explicit `RETAIN`. Note that a very low value can delete files an in-progress query is still scanning. | ## Docker Two different [Docker](https://docs.docker.com/) environments are included to make it easy to experiment with the From 600a2255a5458b1bee2c011030009e72d3131c51 Mon Sep 17 00:00:00 2001 From: CGodiksen <36046286+CGodiksen@users.noreply.github.com> Date: Mon, 10 Aug 2026 13:55:10 +0200 Subject: [PATCH 38/48] Rename DataStorageOptimizer to DataStorageCompactor --- .../src/storage/compressed_data_manager.rs | 2 +- ...optimizer.rs => data_storage_compactor.rs} | 91 +++++++++---------- crates/modelardb_server/src/storage/mod.rs | 4 +- 3 files changed, 48 insertions(+), 49 deletions(-) rename crates/modelardb_server/src/storage/{data_storage_optimizer.rs => data_storage_compactor.rs} (79%) diff --git a/crates/modelardb_server/src/storage/compressed_data_manager.rs b/crates/modelardb_server/src/storage/compressed_data_manager.rs index 2b621a0a..2b6aa9f2 100644 --- a/crates/modelardb_server/src/storage/compressed_data_manager.rs +++ b/crates/modelardb_server/src/storage/compressed_data_manager.rs @@ -29,7 +29,7 @@ use tracing::{debug, error, info}; use crate::configuration::WalMode; use crate::error::Result; use crate::storage::compressed_data_buffer::{CompressedDataBuffer, CompressedSegmentBatch}; -use crate::storage::data_storage_optimizer::DataStorageOptimizer; +use crate::storage::data_storage_compactor::DataStorageOptimizer; use crate::storage::data_transfer::DataTransfer; use crate::storage::types::Message; use crate::storage::types::{Channels, MemoryPool}; diff --git a/crates/modelardb_server/src/storage/data_storage_optimizer.rs b/crates/modelardb_server/src/storage/data_storage_compactor.rs similarity index 79% rename from crates/modelardb_server/src/storage/data_storage_optimizer.rs rename to crates/modelardb_server/src/storage/data_storage_compactor.rs index ecd2b77b..4b502351 100644 --- a/crates/modelardb_server/src/storage/data_storage_optimizer.rs +++ b/crates/modelardb_server/src/storage/data_storage_compactor.rs @@ -13,10 +13,10 @@ * limitations under the License. */ -//! Support for automatically optimizing how compressed data is stored on disk. As compressed data -//! is saved, many small Apache Parquet files accumulate for each table. This component compacts -//! those small files into fewer larger files and vacuums the small files left behind to reduce -//! storage use and query time. +//! Support for automatically compacting the compressed data stored on disk. As compressed data is +//! saved, many small Apache Parquet files accumulate for each table. This component compacts a +//! table by merging those small files into fewer larger ones and vacuuming the files left behind, +//! reducing storage use and query time. use dashmap::DashMap; use modelardb_storage::data_folder::DataFolder; @@ -24,19 +24,19 @@ use tracing::debug; use crate::error::Result; -/// Compacts the many small Apache Parquet files that accumulate for a table into fewer larger files -/// and vacuums the files left behind by the compaction. The component accumulates an estimate of +/// Compacts each table by merging the many small Apache Parquet files that accumulate for it into +/// fewer larger files and vacuuming the files left behind. The component accumulates an estimate of /// how much compactable data each table has and, once the estimate reaches the target file size, -/// optimizes and vacuums that table. The component only operates on the local data folder. -pub(super) struct DataStorageOptimizer { +/// compacts that table. The component only operates on the local data folder. +pub(super) struct DataStorageCompactor { /// The data folder containing all compressed data managed by the /// [`StorageEngine`](crate::storage::StorageEngine). local_data_folder: DataFolder, /// The target size, in bytes, of the files produced when a table is optimized. A table is - /// optimized once its `estimated_compactable_size_in_bytes` reaches this size, so the same - /// value decides both when to optimize and how large the resulting files are. + /// compacted once its `estimated_compactable_size_in_bytes` reaches this size, so the same + /// value decides both when to compact and how large the optimized files are. optimize_target_file_size_in_bytes: u64, - /// The retention period, in seconds, used when a table is vacuumed after it is optimized. + /// The retention period, in seconds, used when a table is vacuumed as part of compaction. /// Note that a very low value can let the vacuum physically delete files that an in-progress /// query is still scanning, causing that query to fail. vacuum_retention_period_in_seconds: u64, @@ -48,11 +48,11 @@ pub(super) struct DataStorageOptimizer { estimated_compactable_size_in_bytes: DashMap, } -impl DataStorageOptimizer { - /// Create a new [`DataStorageOptimizer`] that optimizes the tables in `local_data_folder`, +impl DataStorageCompactor { + /// Create a new [`DataStorageCompactor`] that compacts the tables in `local_data_folder`, /// producing files of approximately `optimize_target_file_size_in_bytes` bytes and vacuuming /// with a retention period of `vacuum_retention_period_in_seconds` seconds. The estimate for - /// each table is initialized with the combined size of its files smaller than the target size, + /// each table is initialized with the combined size of its files smaller than the target size, /// so small files written before a restart are not forgotten. If the files in `local_data_folder` /// could not be read, return [`ModelarDbServerError`](crate::error::ModelarDbServerError). pub(super) async fn try_new( @@ -83,12 +83,11 @@ impl DataStorageOptimizer { } /// Increase the estimated compactable size of the table with `table_name` by `size_in_bytes`. - /// If the estimate has reached `optimize_target_file_size_in_bytes`, the table's small files - /// are compacted and the files left behind are vacuumed. The trigger assumes each newly written - /// file is smaller than the target size. If the target is set below the size of a typical file, - /// optimization is attempted on nearly every write, but is a harmless no-op. Returns [`Ok`] if - /// the table did not need optimizing or was optimized successfully, otherwise - /// [`ModelarDbServerError`](crate::error::ModelarDbServerError). + /// If the estimate has reached `optimize_target_file_size_in_bytes`, the table is compacted. + /// The trigger assumes each newly written file is smaller than the target size. If the target + /// is set below the size of a typical file, compaction is attempted on nearly every write, but + /// is a harmless no-op. Returns [`Ok`] if the table did not need compacting or was compacted + /// successfully, otherwise [`ModelarDbServerError`](crate::error::ModelarDbServerError). pub(super) async fn increase_estimated_compactable_size( &self, table_name: &str, @@ -116,20 +115,20 @@ impl DataStorageOptimizer { >= self.optimize_target_file_size_in_bytes; if estimate_reached_target { - self.optimize_and_vacuum_table(table_name).await?; + self.compact_table(table_name).await?; } Ok(()) } - /// Compact the small files of the table with `table_name` into files of approximately - /// `optimize_target_file_size_in_bytes` bytes, vacuum the small files left behind, and reset + /// Compact the table with `table_name` by merging its small files into files of approximately + /// `optimize_target_file_size_in_bytes` bytes, vacuuming the files left behind, and resetting /// the table's estimated compactable size. Note that the vacuum can physically delete files /// that an in-progress query is still scanning if `vacuum_retention_period_in_seconds` is very - /// low. Returns [`Ok`] if the table was optimized successfully, otherwise + /// low. Returns [`Ok`] if the table was compacted successfully, otherwise /// [`ModelarDbServerError`](crate::error::ModelarDbServerError). - async fn optimize_and_vacuum_table(&self, table_name: &str) -> Result<()> { - debug!("Optimizing the storage of the table '{table_name}'."); + async fn compact_table(&self, table_name: &str) -> Result<()> { + debug!("Compacting the storage of the table '{table_name}'."); self.local_data_folder .optimize_table(table_name, Some(self.optimize_target_file_size_in_bytes)) @@ -139,7 +138,7 @@ impl DataStorageOptimizer { .vacuum_table(table_name, Some(self.vacuum_retention_period_in_seconds)) .await?; - // Reset the estimate so the next optimization only counts data written from now on. + // Reset the estimate so the next compaction only counts data written from now on. *self .estimated_compactable_size_in_bytes .get_mut(table_name) @@ -150,7 +149,7 @@ impl DataStorageOptimizer { /// Set the target size, in bytes, of the files produced when a table is optimized to /// `new_optimize_target_file_size_in_bytes`. The new target takes effect the next time each - /// table is written to. Tables are not re-optimized here to keep configuration updates cheap + /// table is written to. Tables are not re-compacted here to keep configuration updates cheap /// and to avoid having to re-check all files on disk to see if they are compactable. pub(super) fn set_optimize_target_file_size_in_bytes( &mut self, @@ -159,7 +158,7 @@ impl DataStorageOptimizer { self.optimize_target_file_size_in_bytes = new_optimize_target_file_size_in_bytes; } - /// Set the retention period, in seconds, used when a table is vacuumed after it is optimized to + /// Set the retention period, in seconds, used when a table is vacuumed as part of compaction to /// `new_vacuum_retention_period_in_seconds`. pub(super) fn set_vacuum_retention_period_in_seconds( &mut self, @@ -186,9 +185,9 @@ mod tests { let (_temp_dir, local_data_folder) = create_local_data_folder_with_table().await; write_batches_to_table(&local_data_folder, BATCH_COUNT).await; - // The optimizer is created after the data is written, so its estimate includes the small + // The compactor is created after the data is written, so its estimate includes the small // files already on disk. - let optimizer = create_data_storage_optimizer(local_data_folder.clone()).await; + let compactor = create_data_storage_compactor(local_data_folder.clone()).await; let expected_estimate: u64 = local_data_folder .table_file_sizes(TIME_SERIES_TABLE_NAME) @@ -199,7 +198,7 @@ mod tests { assert!(expected_estimate > 0); assert_eq!( - *optimizer + *compactor .estimated_compactable_size_in_bytes .get(TIME_SERIES_TABLE_NAME) .unwrap(), @@ -214,14 +213,14 @@ mod tests { // With a one-byte target, every existing file is already at or above the target, so none of // them count towards the compactable backlog. - let optimizer = DataStorageOptimizer::try_new(local_data_folder.clone(), 1, 0) + let compactor = DataStorageCompactor::try_new(local_data_folder.clone(), 1, 0) .await .unwrap(); assert_eq!(table_file_count(&local_data_folder), BATCH_COUNT); assert_eq!( - *optimizer + *compactor .estimated_compactable_size_in_bytes .get(TIME_SERIES_TABLE_NAME) .unwrap(), @@ -231,16 +230,16 @@ mod tests { // Tests for increase_estimated_compactable_size(). #[tokio::test] - async fn test_optimize_table_when_estimate_reaches_target() { + async fn test_compact_table_when_estimate_reaches_target() { let (_temp_dir, local_data_folder) = create_local_data_folder_with_table().await; - let optimizer = create_data_storage_optimizer(local_data_folder.clone()).await; + let compactor = create_data_storage_compactor(local_data_folder.clone()).await; write_batches_to_table(&local_data_folder, BATCH_COUNT).await; let initial_file_count = table_file_count(&local_data_folder); assert_eq!(initial_file_count, BATCH_COUNT); - optimizer + compactor .increase_estimated_compactable_size( TIME_SERIES_TABLE_NAME, OPTIMIZE_TARGET_FILE_SIZE_IN_BYTES, @@ -251,9 +250,9 @@ mod tests { // The small files should have been compacted into a single file. assert_eq!(table_file_count(&local_data_folder), 1); - // The estimate should have been reset after optimizing. + // The estimate should have been reset after compacting. assert_eq!( - *optimizer + *compactor .estimated_compactable_size_in_bytes .get(TIME_SERIES_TABLE_NAME) .unwrap(), @@ -262,16 +261,16 @@ mod tests { } #[tokio::test] - async fn test_do_not_optimize_table_when_estimate_below_target() { + async fn test_do_not_compact_table_when_estimate_below_target() { let (_temp_dir, local_data_folder) = create_local_data_folder_with_table().await; - let optimizer = create_data_storage_optimizer(local_data_folder.clone()).await; + let compactor = create_data_storage_compactor(local_data_folder.clone()).await; write_batches_to_table(&local_data_folder, BATCH_COUNT).await; let initial_file_count = table_file_count(&local_data_folder); assert_eq!(initial_file_count, BATCH_COUNT); - optimizer + compactor .increase_estimated_compactable_size( TIME_SERIES_TABLE_NAME, OPTIMIZE_TARGET_FILE_SIZE_IN_BYTES - 1, @@ -284,7 +283,7 @@ mod tests { // The estimate should have accumulated without being reset. assert_eq!( - *optimizer + *compactor .estimated_compactable_size_in_bytes .get(TIME_SERIES_TABLE_NAME) .unwrap(), @@ -333,9 +332,9 @@ mod tests { std::fs::read_dir(column_path).unwrap().count() as u8 } - /// Create a [`DataStorageOptimizer`] that optimizes the tables in `local_data_folder`. - async fn create_data_storage_optimizer(local_data_folder: DataFolder) -> DataStorageOptimizer { - DataStorageOptimizer::try_new( + /// Create a [`DataStorageCompactor`] that compacts the tables in `local_data_folder`. + async fn create_data_storage_compactor(local_data_folder: DataFolder) -> DataStorageCompactor { + DataStorageCompactor::try_new( local_data_folder, OPTIMIZE_TARGET_FILE_SIZE_IN_BYTES, VACUUM_RETENTION_PERIOD_IN_SECONDS, diff --git a/crates/modelardb_server/src/storage/mod.rs b/crates/modelardb_server/src/storage/mod.rs index 254177e4..83a7e37a 100644 --- a/crates/modelardb_server/src/storage/mod.rs +++ b/crates/modelardb_server/src/storage/mod.rs @@ -24,7 +24,7 @@ mod compressed_data_buffer; mod compressed_data_manager; pub(super) mod data_sinks; // pub(super) so it can be used in context.rs. -mod data_storage_optimizer; +mod data_storage_compactor; mod data_transfer; mod types; mod uncompressed_data_buffer; @@ -43,7 +43,7 @@ use crate::configuration::{ConfigurationManager, WalMode}; use crate::data_folders::DataFolders; use crate::error::{ModelarDbServerError, Result}; use crate::storage::compressed_data_manager::CompressedDataManager; -use crate::storage::data_storage_optimizer::DataStorageOptimizer; +use crate::storage::data_storage_compactor::DataStorageOptimizer; use crate::storage::data_transfer::DataTransfer; use crate::storage::types::{Channels, MemoryPool, Message}; use crate::storage::uncompressed_data_buffer::IngestedDataBuffer; From 6e7848aa864f0e6938766c5f0fb321d294005dae Mon Sep 17 00:00:00 2001 From: CGodiksen <36046286+CGodiksen@users.noreply.github.com> Date: Mon, 10 Aug 2026 13:58:24 +0200 Subject: [PATCH 39/48] Rename to compact in storage engine module file --- crates/modelardb_server/src/storage/mod.rs | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/crates/modelardb_server/src/storage/mod.rs b/crates/modelardb_server/src/storage/mod.rs index 83a7e37a..7b191624 100644 --- a/crates/modelardb_server/src/storage/mod.rs +++ b/crates/modelardb_server/src/storage/mod.rs @@ -43,7 +43,7 @@ use crate::configuration::{ConfigurationManager, WalMode}; use crate::data_folders::DataFolders; use crate::error::{ModelarDbServerError, Result}; use crate::storage::compressed_data_manager::CompressedDataManager; -use crate::storage::data_storage_compactor::DataStorageOptimizer; +use crate::storage::data_storage_compactor::DataStorageCompactor; use crate::storage::data_transfer::DataTransfer; use crate::storage::types::{Channels, MemoryPool, Message}; use crate::storage::uncompressed_data_buffer::IngestedDataBuffer; @@ -148,7 +148,7 @@ impl StorageEngine { } // Create the compressed data manager. - let data_storage_optimizer = DataStorageOptimizer::try_new( + let data_storage_compactor = DataStorageCompactor::try_new( data_folders.local_data_folder.clone(), configuration_manager.optimize_target_file_size_in_bytes(), configuration_manager.vacuum_retention_period_in_seconds(), @@ -170,7 +170,7 @@ impl StorageEngine { }; let compressed_data_manager = Arc::new(CompressedDataManager::new( - Arc::new(RwLock::new(data_storage_optimizer)), + Arc::new(RwLock::new(data_storage_compactor)), Arc::new(RwLock::new(data_transfer)), data_folders.local_data_folder, channels.clone(), @@ -399,20 +399,20 @@ impl StorageEngine { } } - /// Set the target file size used when automatically optimizing a table's storage to `new_value`. + /// Set the target file size used when automatically compacting a table's storage to `new_value`. pub(super) async fn set_optimize_target_file_size_in_bytes(&self, new_value: u64) { self.compressed_data_manager - .data_storage_optimizer + .data_storage_compactor .write() .await .set_optimize_target_file_size_in_bytes(new_value); } - /// Set the retention period used when automatically vacuuming a table after optimization to + /// Set the retention period used when automatically vacuuming a table during compaction to /// `new_value`. pub(super) async fn set_vacuum_retention_period_in_seconds(&self, new_value: u64) { self.compressed_data_manager - .data_storage_optimizer + .data_storage_compactor .write() .await .set_vacuum_retention_period_in_seconds(new_value); From 1872ff3d6ac19e02ef0eecd04b30081757d7942a Mon Sep 17 00:00:00 2001 From: CGodiksen <36046286+CGodiksen@users.noreply.github.com> Date: Mon, 10 Aug 2026 14:01:53 +0200 Subject: [PATCH 40/48] Rename to compact in compressed data manager --- .../src/storage/compressed_data_manager.rs | 22 +++++++++---------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/crates/modelardb_server/src/storage/compressed_data_manager.rs b/crates/modelardb_server/src/storage/compressed_data_manager.rs index 2b6aa9f2..94cb11e4 100644 --- a/crates/modelardb_server/src/storage/compressed_data_manager.rs +++ b/crates/modelardb_server/src/storage/compressed_data_manager.rs @@ -29,7 +29,7 @@ use tracing::{debug, error, info}; use crate::configuration::WalMode; use crate::error::Result; use crate::storage::compressed_data_buffer::{CompressedDataBuffer, CompressedSegmentBatch}; -use crate::storage::data_storage_compactor::DataStorageOptimizer; +use crate::storage::data_storage_compactor::DataStorageCompactor; use crate::storage::data_transfer::DataTransfer; use crate::storage::types::Message; use crate::storage::types::{Channels, MemoryPool}; @@ -37,9 +37,9 @@ use crate::storage::types::{Channels, MemoryPool}; /// Stores data points compressed as segments containing metadata and models in memory to batch the /// compressed segments before saving them to Apache Parquet files. pub(super) struct CompressedDataManager { - /// Component that compacts the small compressed files that accumulate for a table into fewer - /// larger files and vacuums the files left behind by the compaction. - pub(super) data_storage_optimizer: Arc>, + /// Component that compacts a table by merging the small compressed files that accumulate for it + /// into fewer larger files and vacuuming the files left behind. + pub(super) data_storage_compactor: Arc>, /// Component that transfers saved compressed data to the remote data folder when it is necessary. pub(super) data_transfer: Arc>>, /// Folder containing all compressed data managed by the [`StorageEngine`](crate::storage::StorageEngine). @@ -61,7 +61,7 @@ pub(super) struct CompressedDataManager { impl CompressedDataManager { pub(super) fn new( - data_storage_optimizer: Arc>, + data_storage_compactor: Arc>, data_transfer: Arc>>, local_data_folder: DataFolder, channels: Arc, @@ -69,7 +69,7 @@ impl CompressedDataManager { wal_mode: WalMode, ) -> Self { Self { - data_storage_optimizer, + data_storage_compactor, data_transfer, local_data_folder, compressed_data_buffers: DashMap::new(), @@ -293,9 +293,9 @@ impl CompressedDataManager { self.memory_pool.remaining_compressed_memory_in_bytes() ); - // Optimize how the compressed data for table_name is stored on disk once enough new data - // has been written since the last optimization. - self.data_storage_optimizer + // Compact the compressed data for table_name on disk once enough new data has been written + // since the last compaction. + self.data_storage_compactor .read() .await .increase_estimated_compactable_size(table_name, compressed_data_buffer_size_in_bytes) @@ -594,7 +594,7 @@ mod tests { .unwrap(), )); - let optimizer = DataStorageOptimizer::try_new( + let compactor = DataStorageCompactor::try_new( local_data_folder.clone(), 64 * 1024 * 1024, 60 * 60 * 24 * 7, @@ -605,7 +605,7 @@ mod tests { ( temp_dir, CompressedDataManager::new( - Arc::new(RwLock::new(optimizer)), + Arc::new(RwLock::new(compactor)), Arc::new(RwLock::new(None)), local_data_folder, channels, From 97ad914f4f319b5329d32d69b7b8000d95b4ba64 Mon Sep 17 00:00:00 2001 From: CGodiksen <36046286+CGodiksen@users.noreply.github.com> Date: Mon, 10 Aug 2026 14:04:33 +0200 Subject: [PATCH 41/48] Rename compaction to merge when talking about optimize in DataFolder --- crates/modelardb_storage/src/data_folder/mod.rs | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/crates/modelardb_storage/src/data_folder/mod.rs b/crates/modelardb_storage/src/data_folder/mod.rs index b5c44d21..4d9b4f2b 100644 --- a/crates/modelardb_storage/src/data_folder/mod.rs +++ b/crates/modelardb_storage/src/data_folder/mod.rs @@ -685,12 +685,12 @@ impl DataFolder { Ok(()) } - /// Optimize the Delta Lake table with `table_name` by compacting its many small files into + /// Optimize the Delta Lake table with `table_name` by merging its many small files into /// fewer larger files of approximately `maybe_target_size_in_bytes` bytes. If a target size is - /// not given, a default target size of 64 MiB is used. Compaction only rewrites files smaller + /// not given, a default target size of 64 MiB is used. Optimize only rewrites files smaller /// than the target, so it is safe to call repeatedly. Note that the small files are only marked /// as removed and are not deleted from disk until the table is vacuumed. If the target size is - /// zero, the table does not exist, or the files could not be compacted, a + /// zero, the table does not exist, or the files could not be merged, a /// [`ModelarDbStorageError`] is returned. pub async fn optimize_table( &self, @@ -1619,7 +1619,7 @@ mod tests { .await .unwrap(); - // The small files should be compacted into a single active file with no rows lost or + // The small files should be merged into a single active file with no rows lost or // duplicated. assert_eq!(active_file_count(&data_folder, "normal_table_1").await, 1); assert_eq!(row_count(&data_folder, "normal_table_1").await, rows_before); @@ -1651,7 +1651,7 @@ mod tests { .await .unwrap(); - // The small files should be compacted into a single active file with no rows lost or + // The small files should be merged into a single active file with no rows lost or // duplicated. assert_eq!( active_file_count(&data_folder, TIME_SERIES_TABLE_NAME).await, @@ -1690,7 +1690,7 @@ mod tests { assert_eq!(files_before, 4); // A one-byte target is smaller than every existing file, so none of them are candidates for - // compaction, and the files should be left untouched. + // merging, and the files should be left untouched. data_folder .optimize_table("normal_table_1", Some(1)) .await From 2524978264bf5e43526c897fe911aadee09ce5e8 Mon Sep 17 00:00:00 2001 From: CGodiksen <36046286+CGodiksen@users.noreply.github.com> Date: Mon, 10 Aug 2026 14:08:12 +0200 Subject: [PATCH 42/48] Rename compaction to merge when talking about optimize in context --- crates/modelardb_server/src/context.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/crates/modelardb_server/src/context.rs b/crates/modelardb_server/src/context.rs index c94985b5..9d600959 100644 --- a/crates/modelardb_server/src/context.rs +++ b/crates/modelardb_server/src/context.rs @@ -401,7 +401,7 @@ impl Context { Ok(()) } - /// Optimize the table with `table_name` if it exists by compacting its small files into larger + /// Optimize the table with `table_name` if it exists by merging its small files into larger /// files of approximately `maybe_target_size_in_bytes` bytes. If a target size is not given, a /// default target size of 64 MiB is used. If the target size is zero, the table does not exist, /// or it could not be optimized, [`ModelarDbServerError`] is returned. @@ -922,7 +922,7 @@ mod tests { .await .unwrap(); - // The small files should be compacted into a single active file. + // The small files should be merged into a single active file. assert_eq!(active_file_count(&context, NORMAL_TABLE_NAME).await, 1); } @@ -970,7 +970,7 @@ mod tests { .await .unwrap(); - // The small files should be compacted into a single active file. + // The small files should be merged into a single active file. assert_eq!(active_file_count(&context, TIME_SERIES_TABLE_NAME).await, 1); } From 89f1631fcbc3e0fb43647d19eae2f2b6f35de660 Mon Sep 17 00:00:00 2001 From: CGodiksen <36046286+CGodiksen@users.noreply.github.com> Date: Mon, 10 Aug 2026 14:11:50 +0200 Subject: [PATCH 43/48] Use merge instead of compaction in WAL and integration test --- crates/modelardb_server/tests/integration_test.rs | 6 +++--- crates/modelardb_storage/src/write_ahead_log.rs | 4 ++-- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/crates/modelardb_server/tests/integration_test.rs b/crates/modelardb_server/tests/integration_test.rs index d6f46a78..954bee05 100644 --- a/crates/modelardb_server/tests/integration_test.rs +++ b/crates/modelardb_server/tests/integration_test.rs @@ -831,7 +831,7 @@ async fn test_can_optimize_normal_table() { .await; // ingest_time_series_and_flush_data() writes one file. Ingest and flush three more times so - // there are four small files to compact. + // there are four small files to merge. for _ in 0..3 { let flight_data = TestContext::create_flight_data_from_time_series( NORMAL_TABLE_NAME.to_owned(), @@ -886,7 +886,7 @@ async fn test_can_optimize_time_series_table() { .await; // ingest_time_series_and_flush_data() writes one file per field column partition. ingest and - // flush three more times so each partition has four small files to compact. + // flush three more times so each partition has four small files to merge. for _ in 0..3 { let flight_data = TestContext::create_flight_data_from_time_series( TIME_SERIES_TABLE_NAME.to_owned(), @@ -921,7 +921,7 @@ async fn test_can_optimize_time_series_table() { .await .unwrap(); - // The four files in the partition should be compacted into a single active file. + // The four files in the partition should be merged into a single active file. let files = std::fs::read_dir(&column_path).unwrap(); assert_eq!(files.count(), 1); } diff --git a/crates/modelardb_storage/src/write_ahead_log.rs b/crates/modelardb_storage/src/write_ahead_log.rs index 8880fc06..786fda82 100644 --- a/crates/modelardb_storage/src/write_ahead_log.rs +++ b/crates/modelardb_storage/src/write_ahead_log.rs @@ -795,7 +795,7 @@ mod tests { ); assert_eq!(std::fs::read_dir(&column_path).unwrap().count(), 3); - // Optimize compacts the small files into one, and vacuum physically deletes the stale files + // Optimize merges the small files into one, and vacuum physically deletes the stale files // left behind. Vacuum should only remove Parquet data files, never the _delta_log commits. data_folder .optimize_table(TIME_SERIES_TABLE_NAME, None) @@ -806,7 +806,7 @@ mod tests { .await .unwrap(); - // Only the single compacted Parquet file should remain on disk. + // Only the single merged Parquet file should remain on disk. assert_eq!(std::fs::read_dir(&column_path).unwrap().count(), 1); // Rebuilding the WAL from the same folder must still recover every persisted batch id from From cdf13d84f4475d01fdef8d535c95740b251a6b7f Mon Sep 17 00:00:00 2001 From: CGodiksen <36046286+CGodiksen@users.noreply.github.com> Date: Mon, 10 Aug 2026 14:15:31 +0200 Subject: [PATCH 44/48] Use merge instead of compaction when talking about optimize in embedded --- crates/modelardb_embedded/src/capi.rs | 2 +- crates/modelardb_embedded/src/operations/client.rs | 2 +- crates/modelardb_embedded/src/operations/data_folder.rs | 6 +++--- crates/modelardb_embedded/src/operations/mod.rs | 2 +- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/crates/modelardb_embedded/src/capi.rs b/crates/modelardb_embedded/src/capi.rs index 5447abf6..2f7e1e7f 100644 --- a/crates/modelardb_embedded/src/capi.rs +++ b/crates/modelardb_embedded/src/capi.rs @@ -1007,7 +1007,7 @@ unsafe fn vacuum( } /// Optimizes the table with the name in `table_name_ptr` in the [`DataFolder`] or [`Client`] in -/// `maybe_operations_ptr` by compacting its many small files into fewer larger files of +/// `maybe_operations_ptr` by merging its many small files into fewer larger files of /// approximately `target_size_in_bytes_ptr` bytes. Assumes `maybe_operations_ptr` points to a /// [`DataFolder`] or [`Client`]; `table_name_ptr` points to a valid C string; and /// `target_size_in_bytes_ptr` points to a valid `u64`, or is null to use the default target size. diff --git a/crates/modelardb_embedded/src/operations/client.rs b/crates/modelardb_embedded/src/operations/client.rs index b71d93b4..4456d067 100644 --- a/crates/modelardb_embedded/src/operations/client.rs +++ b/crates/modelardb_embedded/src/operations/client.rs @@ -316,7 +316,7 @@ impl Operations for Client { Ok(()) } - /// Optimize the table with the name in `table_name` by compacting its many small files into + /// Optimize the table with the name in `table_name` by merging its many small files into /// fewer larger files of approximately `maybe_target_size_in_bytes` bytes. If a target size is /// not given, the default target size of 64 MiB is used. If the table does not exist, the table /// could not be optimized, or the target size is zero, [`ModelarDbEmbeddedError`] is returned. diff --git a/crates/modelardb_embedded/src/operations/data_folder.rs b/crates/modelardb_embedded/src/operations/data_folder.rs index bb293dbf..57fe7938 100644 --- a/crates/modelardb_embedded/src/operations/data_folder.rs +++ b/crates/modelardb_embedded/src/operations/data_folder.rs @@ -540,7 +540,7 @@ impl Operations for DataFolder { } } - /// Optimize the table with the name in `table_name` by compacting its many small files into + /// Optimize the table with the name in `table_name` by merging its many small files into /// fewer larger files of approximately `maybe_target_size_in_bytes` bytes. If a target size is /// not given, the default target size of 64 MiB is used. If the table does not exist, the table /// could not be optimized, or the target size is zero, [`ModelarDbEmbeddedError`] is returned. @@ -2314,7 +2314,7 @@ mod tests { data_folder.optimize(NORMAL_TABLE_NAME, None).await.unwrap(); - // The small files should be compacted into a single active file. + // The small files should be merged into a single active file. let delta_table = data_folder.delta_table(NORMAL_TABLE_NAME).await.unwrap(); assert_eq!(delta_table.get_file_uris().unwrap().count(), 1); } @@ -2343,7 +2343,7 @@ mod tests { .await .unwrap(); - // The files in each of the two partitions should be compacted into a single active file. + // The files in each of the two partitions should be merged into a single active file. let delta_table = data_folder .delta_table(TIME_SERIES_TABLE_NAME) .await diff --git a/crates/modelardb_embedded/src/operations/mod.rs b/crates/modelardb_embedded/src/operations/mod.rs index 2ff33b49..24a5793b 100644 --- a/crates/modelardb_embedded/src/operations/mod.rs +++ b/crates/modelardb_embedded/src/operations/mod.rs @@ -146,7 +146,7 @@ pub trait Operations: Sync + Send { maybe_retention_period_in_seconds: Option, ) -> Result<()>; - /// Optimize the table with the name in `table_name` by compacting its many small files into + /// Optimize the table with the name in `table_name` by merging its many small files into /// fewer larger files of approximately `maybe_target_size_in_bytes` bytes. If a target size is /// not given, the default target size of 64 MiB is used. async fn optimize( From 70e55303d549dc6896e8f32df9d85aef8246d607 Mon Sep 17 00:00:00 2001 From: CGodiksen <36046286+CGodiksen@users.noreply.github.com> Date: Mon, 10 Aug 2026 14:18:26 +0200 Subject: [PATCH 45/48] Use merge instead of compaction in bindings --- crates/modelardb_embedded/bindings/c/modelardb_embedded.h | 2 +- .../bindings/python/modelardb/operations.py | 2 +- .../bindings/python/tests/test_operations.py | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/crates/modelardb_embedded/bindings/c/modelardb_embedded.h b/crates/modelardb_embedded/bindings/c/modelardb_embedded.h index 0fe32dcd..34a2e728 100644 --- a/crates/modelardb_embedded/bindings/c/modelardb_embedded.h +++ b/crates/modelardb_embedded/bindings/c/modelardb_embedded.h @@ -199,7 +199,7 @@ int modelardb_embedded_vacuum(void* maybe_operations_ptr, const char* table_name_ptr, const uint64_t* retention_period_in_seconds_ptr); -// Optimize the table by compacting its many small files into fewer larger files. +// Optimize the table by merging its many small files into fewer larger files. int modelardb_embedded_optimize(void* maybe_operations_ptr, bool is_data_folder, const char* table_name_ptr, diff --git a/crates/modelardb_embedded/bindings/python/modelardb/operations.py b/crates/modelardb_embedded/bindings/python/modelardb/operations.py index c975767a..e172f5df 100644 --- a/crates/modelardb_embedded/bindings/python/modelardb/operations.py +++ b/crates/modelardb_embedded/bindings/python/modelardb/operations.py @@ -685,7 +685,7 @@ def optimize(self, table_name: str, target_size_in_bytes: None | int = None): :param table_name: The name of the table to optimize. :type table_name: str - :param target_size_in_bytes: The target file size in bytes. Many small files are compacted + :param target_size_in_bytes: The target file size in bytes. Many small files are merged into fewer larger files of approximately this size. If `None`, the default target size of 64 MiB is used. :type target_size_in_bytes: int, optional diff --git a/crates/modelardb_embedded/bindings/python/tests/test_operations.py b/crates/modelardb_embedded/bindings/python/tests/test_operations.py index 24fea831..0d8fe3ca 100644 --- a/crates/modelardb_embedded/bindings/python/tests/test_operations.py +++ b/crates/modelardb_embedded/bindings/python/tests/test_operations.py @@ -512,10 +512,10 @@ def test_data_folder_optimize(self): data_folder.optimize(TIME_SERIES_TABLE_NAME) - # Vacuum to remove the compacted files. + # Vacuum to remove the stale files left by the merge. data_folder.vacuum(TIME_SERIES_TABLE_NAME, retention_period_in_seconds=0) - # The small files should be compacted into a single active file. + # The small files should be merged into a single active file. file_count = len(os.listdir(folder_path)) self.assertEqual(file_count, 1) From c84d3b17e2b5de449543d6efeeedf8716d0b292f Mon Sep 17 00:00:00 2001 From: CGodiksen <36046286+CGodiksen@users.noreply.github.com> Date: Mon, 10 Aug 2026 14:27:28 +0200 Subject: [PATCH 46/48] Use compaction instead of optimization in user docs --- docs/user/README.md | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/docs/user/README.md b/docs/user/README.md index b8415a7e..e4a097d1 100644 --- a/docs/user/README.md +++ b/docs/user/README.md @@ -380,18 +380,18 @@ the built-in defaults. Note that the connection settings `--host` and `--port` a file. Variables marked with ✓ in the **Updatable** column can also be updated while the server is running using the `UpdateConfiguration` action without requiring a restart. The update is persisted in the configuration file. -| **CLI Flag** | **Environment Variable** | **Default** | **Updatable** | **Description** | -|-------------------------------------------|----------------------------------------------------|-------------|---------------|----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| -| `--host` | `MODELARDBD_HOST` | 127.0.0.1 | | The host address of the `modelardbd` server. | -| `--port` | `MODELARDBD_PORT` | 9999 | | The port of the `modelardbd` server. | -| `--wal-enabled` | `MODELARDBD_WAL_ENABLED` | true | | Whether the write-ahead log is enabled. | -| `--multivariate-reserved-memory-in-bytes` | `MODELARDBD_MULTIVARIATE_RESERVED_MEMORY_IN_BYTES` | 512 MB | ✓ | The amount of memory to reserve for storing multivariate time series. | -| `--uncompressed-reserved-memory-in-bytes` | `MODELARDBD_UNCOMPRESSED_RESERVED_MEMORY_IN_BYTES` | 512 MB | ✓ | The amount of memory to reserve for storing uncompressed data buffers. | -| `--compressed-reserved-memory-in-bytes` | `MODELARDBD_COMPRESSED_RESERVED_MEMORY_IN_BYTES` | 512 MB | ✓ | The amount of memory to reserve for storing compressed data buffers. | -| `--transfer-batch-size-in-bytes` | `MODELARDBD_TRANSFER_BATCH_SIZE_IN_BYTES` | 64 MB | ✓ | The amount of data that must be collected before transferring a batch to the remote object store. | -| `--segment-size-threshold-in-bytes` | `MODELARDBD_SEGMENT_SIZE_THRESHOLD_IN_BYTES` | 64 MB | ✓ | The approximate maximum size of a single WAL segment file before a new one is started. Only updatable when the WAL is enabled. | -| `--optimize-target-file-size-in-bytes` | `MODELARDBD_OPTIMIZE_TARGET_FILE_SIZE_IN_BYTES` | 64 MB | ✓ | The target size of the files produced when automatically optimizing a table's storage. Also used as the default when `OPTIMIZE` is run without an explicit `TARGET`. | -| `--vacuum-retention-period-in-seconds` | `MODELARDBD_VACUUM_RETENTION_PERIOD_IN_SECONDS` | 7 days | ✓ | The retention period used when automatically vacuuming a table after optimization. Also used as the default when `VACUUM` is run without an explicit `RETAIN`. Note that a very low value can delete files an in-progress query is still scanning. | +| **CLI Flag** | **Environment Variable** | **Default** | **Updatable** | **Description** | +|-------------------------------------------|----------------------------------------------------|-------------|---------------|---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| `--host` | `MODELARDBD_HOST` | 127.0.0.1 | | The host address of the `modelardbd` server. | +| `--port` | `MODELARDBD_PORT` | 9999 | | The port of the `modelardbd` server. | +| `--wal-enabled` | `MODELARDBD_WAL_ENABLED` | true | | Whether the write-ahead log is enabled. | +| `--multivariate-reserved-memory-in-bytes` | `MODELARDBD_MULTIVARIATE_RESERVED_MEMORY_IN_BYTES` | 512 MB | ✓ | The amount of memory to reserve for storing multivariate time series. | +| `--uncompressed-reserved-memory-in-bytes` | `MODELARDBD_UNCOMPRESSED_RESERVED_MEMORY_IN_BYTES` | 512 MB | ✓ | The amount of memory to reserve for storing uncompressed data buffers. | +| `--compressed-reserved-memory-in-bytes` | `MODELARDBD_COMPRESSED_RESERVED_MEMORY_IN_BYTES` | 512 MB | ✓ | The amount of memory to reserve for storing compressed data buffers. | +| `--transfer-batch-size-in-bytes` | `MODELARDBD_TRANSFER_BATCH_SIZE_IN_BYTES` | 64 MB | ✓ | The amount of data that must be collected before transferring a batch to the remote object store. | +| `--segment-size-threshold-in-bytes` | `MODELARDBD_SEGMENT_SIZE_THRESHOLD_IN_BYTES` | 64 MB | ✓ | The approximate maximum size of a single WAL segment file before a new one is started. Only updatable when the WAL is enabled. | +| `--optimize-target-file-size-in-bytes` | `MODELARDBD_OPTIMIZE_TARGET_FILE_SIZE_IN_BYTES` | 64 MB | ✓ | The target size of the files produced when automatically compacting a table's storage. Also used as the default when `OPTIMIZE` is run without an explicit `TARGET`. | +| `--vacuum-retention-period-in-seconds` | `MODELARDBD_VACUUM_RETENTION_PERIOD_IN_SECONDS` | 7 days | ✓ | The retention period used when automatically vacuuming a table during compaction. Also used as the default when `VACUUM` is run without an explicit `RETAIN`. Note that a very low value can delete files an in-progress query is still scanning. | ## Docker Two different [Docker](https://docs.docker.com/) environments are included to make it easy to experiment with the From 141262380b457f356ec177eb82927584a99a185d Mon Sep 17 00:00:00 2001 From: CGodiksen <36046286+CGodiksen@users.noreply.github.com> Date: Mon, 10 Aug 2026 14:40:29 +0200 Subject: [PATCH 47/48] Fix table formatting after merge --- docs/user/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/user/README.md b/docs/user/README.md index feae6c00..8a64f20f 100644 --- a/docs/user/README.md +++ b/docs/user/README.md @@ -395,7 +395,7 @@ file. Variables marked with ✓ in the **Updatable** column can also be updated `UpdateConfiguration` action without requiring a restart. The update is persisted in the configuration file. | **CLI Flag** | **Environment Variable** | **Default** | **Updatable** | **Description** | -| ----------------------------------------- | -------------------------------------------------- | ----------- | ------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +|-------------------------------------------|----------------------------------------------------|-------------|---------------|---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| | `--host` | `MODELARDBD_HOST` | 127.0.0.1 | | The host address of the `modelardbd` server. | | `--port` | `MODELARDBD_PORT` | 9999 | | The port of the `modelardbd` server. | | `--wal-enabled` | `MODELARDBD_WAL_ENABLED` | true | | Whether the write-ahead log is enabled. | From f554b478e95fca60db6770d6adc986ec5bf71b2e Mon Sep 17 00:00:00 2001 From: CGodiksen <36046286+CGodiksen@users.noreply.github.com> Date: Mon, 10 Aug 2026 15:06:17 +0200 Subject: [PATCH 48/48] Use compaction when describing what the new config values does --- crates/modelardb_server/src/configuration.rs | 8 ++++---- crates/modelardb_server/src/main.rs | 4 ++-- crates/modelardb_types/src/flight/protocol.proto | 4 ++-- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/crates/modelardb_server/src/configuration.rs b/crates/modelardb_server/src/configuration.rs index 7c3400f5..39b1ae59 100644 --- a/crates/modelardb_server/src/configuration.rs +++ b/crates/modelardb_server/src/configuration.rs @@ -64,11 +64,11 @@ struct Configuration { /// The approximate maximum size, in bytes, of a single WAL segment file before it is closed and /// a new one is started. segment_size_threshold_in_bytes: u64, - /// Target size, in bytes, of the files produced when automatically optimizing a table's + /// Target size, in bytes, of the files produced when automatically compacting a table's /// storage. This is also the default value used when an OPTIMIZE query is executed without an /// explicit target size. optimize_target_file_size_in_bytes: u64, - /// Retention period, in seconds, used when automatically vacuuming a table after optimization. + /// Retention period, in seconds, used when automatically vacuuming a table during compaction. /// This is also the default value used when a VACUUM query is executed without an explicit /// retention period. vacuum_retention_period_in_seconds: u64, @@ -424,7 +424,7 @@ impl ConfigurationManager { self.configuration.optimize_target_file_size_in_bytes } - /// Set the target file size used by the data storage optimizer and as the default for OPTIMIZE + /// Set the target file size used by the data storage compactor and as the default for OPTIMIZE /// queries without an explicit target size. If the new value is zero or the new configuration /// could not be saved to the configuration file, return [`ModelarDbServerError`]. pub(crate) async fn set_optimize_target_file_size_in_bytes( @@ -456,7 +456,7 @@ impl ConfigurationManager { self.configuration.vacuum_retention_period_in_seconds } - /// Set the retention period used by the data storage optimizer and as the default for VACUUM + /// Set the retention period used by the data storage compactor and as the default for VACUUM /// queries without an explicit retention period. If the new value is larger than /// [`MAX_RETENTION_PERIOD_IN_SECONDS`] or the new configuration could not be saved to the /// configuration file, return [`ModelarDbServerError`]. diff --git a/crates/modelardb_server/src/main.rs b/crates/modelardb_server/src/main.rs index eb047797..673e2847 100644 --- a/crates/modelardb_server/src/main.rs +++ b/crates/modelardb_server/src/main.rs @@ -84,13 +84,13 @@ pub(crate) struct ServerArgs { #[arg(long, env = "MODELARDBD_SEGMENT_SIZE_THRESHOLD_IN_BYTES")] segment_size_threshold_in_bytes: Option, - /// Target size, in bytes, of the files produced when automatically optimizing a table's + /// Target size, in bytes, of the files produced when automatically compacting a table's /// storage. This is also the default value used when an OPTIMIZE query is executed without an /// explicit target size. #[arg(long, env = "MODELARDBD_OPTIMIZE_TARGET_FILE_SIZE_IN_BYTES")] optimize_target_file_size_in_bytes: Option, - /// Retention period, in seconds, used when automatically vacuuming a table after optimization. + /// Retention period, in seconds, used when automatically vacuuming a table during compaction. /// This is also the default value used when a VACUUM query is executed without an explicit /// retention period. Note that a very low value can delete files an in-progress query is still /// scanning. diff --git a/crates/modelardb_types/src/flight/protocol.proto b/crates/modelardb_types/src/flight/protocol.proto index af4d0afb..141816e3 100644 --- a/crates/modelardb_types/src/flight/protocol.proto +++ b/crates/modelardb_types/src/flight/protocol.proto @@ -80,11 +80,11 @@ message Configuration { // Whether the write-ahead log is enabled. bool wal_enabled = 9; - // Target size, in bytes, of the files produced when automatically optimizing a table's storage. This is also the + // Target size, in bytes, of the files produced when automatically compacting a table's storage. This is also the // default value used when an OPTIMIZE query is executed without an explicit target size. uint64 optimize_target_file_size_in_bytes = 10; - // Retention period, in seconds, used when automatically vacuuming a table after optimization. This is also the + // Retention period, in seconds, used when automatically vacuuming a table during compaction. This is also the // default value used when a VACUUM query is executed without an explicit retention period. Note that a very low // value can delete files an in-progress query is still scanning. uint64 vacuum_retention_period_in_seconds = 11;