diff --git a/crates/modelardb_auth/src/lib.rs b/crates/modelardb_auth/src/lib.rs index 763c383f..cc6ebfe3 100644 --- a/crates/modelardb_auth/src/lib.rs +++ b/crates/modelardb_auth/src/lib.rs @@ -61,7 +61,9 @@ impl BearerInterceptor { .map(|token| { format!("Bearer {token}") .parse::() - .map_err(|error| Status::invalid_argument(format!("Token is not ASCII: {error}."))) + .map_err(|error| { + Status::invalid_argument(format!("Token is not ASCII: {error}.")) + }) }) .transpose()?; diff --git a/crates/modelardb_server/src/configuration.rs b/crates/modelardb_server/src/configuration.rs index c510f566..3bdf4a14 100644 --- a/crates/modelardb_server/src/configuration.rs +++ b/crates/modelardb_server/src/configuration.rs @@ -51,8 +51,8 @@ pub(crate) enum WalMode { /// only be done through the [`ConfigurationManager`]. #[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] struct Configuration { - /// Amount of memory to reserve for storing multivariate time series. - multivariate_reserved_memory_in_bytes: u64, + /// Amount of memory to reserve for storing ingested time series. + ingested_reserved_memory_in_bytes: u64, /// Amount of memory to reserve for storing uncompressed data buffers. uncompressed_reserved_memory_in_bytes: u64, /// Amount of memory to reserve for storing compressed data buffers. @@ -79,8 +79,8 @@ impl Configuration { /// Update the configuration parameters with the corresponding flags or environment variables /// from the command line if they are set. fn update_from_args(&mut self, args: &ServerArgs) { - if let Some(value) = args.multivariate_reserved_memory_in_bytes { - self.multivariate_reserved_memory_in_bytes = value; + if let Some(value) = args.ingested_reserved_memory_in_bytes { + self.ingested_reserved_memory_in_bytes = value; } if let Some(value) = args.uncompressed_reserved_memory_in_bytes { @@ -142,7 +142,7 @@ impl Configuration { impl Default for Configuration { fn default() -> Self { Self { - multivariate_reserved_memory_in_bytes: 512 * 1024 * 1024, + ingested_reserved_memory_in_bytes: 512 * 1024 * 1024, uncompressed_reserved_memory_in_bytes: 512 * 1024 * 1024, compressed_reserved_memory_in_bytes: 512 * 1024 * 1024, transfer_batch_size_in_bytes: Some(64 * 1024 * 1024), @@ -237,31 +237,31 @@ impl ConfigurationManager { &self.wal_mode } - pub(crate) fn multivariate_reserved_memory_in_bytes(&self) -> u64 { - self.configuration.multivariate_reserved_memory_in_bytes + pub(crate) fn ingested_reserved_memory_in_bytes(&self) -> u64 { + self.configuration.ingested_reserved_memory_in_bytes } - /// Set the new value and update the amount of memory for multivariate data in the storage + /// Set the new value and update the amount of memory for ingested data in the storage /// engine. If the new configuration could not be saved to the configuration file, return /// [`ModelarDbServerError`]. - pub(crate) async fn set_multivariate_reserved_memory_in_bytes( + pub(crate) async fn set_ingested_reserved_memory_in_bytes( &mut self, - new_multivariate_reserved_memory_in_bytes: u64, + new_ingested_reserved_memory_in_bytes: u64, storage_engine: Arc>, ) -> Result<()> { // Since the storage engine only keeps track of the remaining reserved memory, calculate // how much the value should change. - let value_change = new_multivariate_reserved_memory_in_bytes as i64 - - self.configuration.multivariate_reserved_memory_in_bytes as i64; + let value_change = new_ingested_reserved_memory_in_bytes as i64 + - self.configuration.ingested_reserved_memory_in_bytes as i64; storage_engine .write() .await - .adjust_multivariate_remaining_memory_in_bytes(value_change) + .adjust_ingested_remaining_memory_in_bytes(value_change) .await; - self.configuration.multivariate_reserved_memory_in_bytes = - new_multivariate_reserved_memory_in_bytes; + self.configuration.ingested_reserved_memory_in_bytes = + new_ingested_reserved_memory_in_bytes; self.configuration .save_to_toml(&self.local_data_folder) @@ -405,9 +405,7 @@ impl ConfigurationManager { /// protobuf message and serialize it. pub(crate) fn encode_and_serialize(&self) -> Vec { let configuration = protocol::Configuration { - multivariate_reserved_memory_in_bytes: self - .configuration - .multivariate_reserved_memory_in_bytes, + ingested_reserved_memory_in_bytes: self.configuration.ingested_reserved_memory_in_bytes, uncompressed_reserved_memory_in_bytes: self .configuration .uncompressed_reserved_memory_in_bytes, @@ -461,7 +459,7 @@ mod tests { let local_data_folder = DataFolder::open_local_url(local_url).await.unwrap(); let existing_configuration = Configuration { - multivariate_reserved_memory_in_bytes: 1, + ingested_reserved_memory_in_bytes: 1, uncompressed_reserved_memory_in_bytes: 1, compressed_reserved_memory_in_bytes: 1, transfer_batch_size_in_bytes: Some(1), @@ -540,7 +538,7 @@ mod tests { } #[tokio::test] - async fn test_set_multivariate_reserved_memory_in_bytes() { + async fn test_set_ingested_reserved_memory_in_bytes() { let temp_dir = tempfile::tempdir().unwrap(); let (storage_engine, configuration_manager) = create_components(&temp_dir).await; @@ -548,7 +546,7 @@ mod tests { configuration_manager .read() .await - .multivariate_reserved_memory_in_bytes(), + .ingested_reserved_memory_in_bytes(), 512 * 1024 * 1024 ); @@ -556,7 +554,7 @@ mod tests { configuration_manager .write() .await - .set_multivariate_reserved_memory_in_bytes(new_value, storage_engine) + .set_ingested_reserved_memory_in_bytes(new_value, storage_engine) .await .unwrap(); @@ -564,13 +562,13 @@ mod tests { configuration_manager .read() .await - .multivariate_reserved_memory_in_bytes(), + .ingested_reserved_memory_in_bytes(), new_value ); let configuration_from_file = configuration_from_file(&temp_dir).await; assert_eq!( - configuration_from_file.multivariate_reserved_memory_in_bytes, + configuration_from_file.ingested_reserved_memory_in_bytes, new_value ); } diff --git a/crates/modelardb_server/src/main.rs b/crates/modelardb_server/src/main.rs index e117d821..c873db51 100644 --- a/crates/modelardb_server/src/main.rs +++ b/crates/modelardb_server/src/main.rs @@ -61,9 +61,9 @@ pub(crate) struct ServerArgs { #[arg(long, default_value_t = 9999, env = "MODELARDBD_PORT")] port: u16, - /// Amount of memory in bytes to reserve for storing multivariate time series. - #[arg(long, env = "MODELARDBD_MULTIVARIATE_RESERVED_MEMORY_IN_BYTES")] - multivariate_reserved_memory_in_bytes: Option, + /// Amount of memory in bytes to reserve for storing ingested time series. + #[arg(long, env = "MODELARDBD_INGESTED_RESERVED_MEMORY_IN_BYTES")] + ingested_reserved_memory_in_bytes: Option, /// Amount of memory in bytes to reserve for storing uncompressed data buffers. #[arg(long, env = "MODELARDBD_UNCOMPRESSED_RESERVED_MEMORY_IN_BYTES")] diff --git a/crates/modelardb_server/src/remote/mod.rs b/crates/modelardb_server/src/remote/mod.rs index cf63bc3c..720f8521 100644 --- a/crates/modelardb_server/src/remote/mod.rs +++ b/crates/modelardb_server/src/remote/mod.rs @@ -67,7 +67,7 @@ use crate::remote::auth_layer::AuthLayer; /// Start an Apache Arrow Flight server on 0.0.0.0:`port` that passes `context` to the methods that /// process the requests through [`FlightServiceHandler`]. All requests are passed through the /// [`AuthLayer`], which authenticates them using `maybe_authenticator` before they are passed to -/// the [`FlightServiceHandler`]. If `maybe_authenticator` is [`None`], authentication is disabled, +/// the [`FlightServiceHandler`]. If `maybe_authenticator` is [`None`], authentication is disabled, /// and every request that is not an internal cluster request is allowed. pub async fn start_apache_arrow_flight_server( context: Arc, @@ -983,11 +983,11 @@ impl FlightService for FlightServiceHandler { Status::invalid_argument(format!("New value for {setting} cannot be null.")); match protocol::update_configuration::Setting::try_from(setting) { - Ok(protocol::update_configuration::Setting::MultivariateReservedMemoryInBytes) => { + Ok(protocol::update_configuration::Setting::IngestedReservedMemoryInBytes) => { let new_value = maybe_new_value.ok_or(invalid_null_error)?; configuration_manager - .set_multivariate_reserved_memory_in_bytes(new_value, storage_engine) + .set_ingested_reserved_memory_in_bytes(new_value, storage_engine) .await .map_err(error_to_status_internal) } diff --git a/crates/modelardb_server/src/storage/mod.rs b/crates/modelardb_server/src/storage/mod.rs index 8460b51b..0d323396 100644 --- a/crates/modelardb_server/src/storage/mod.rs +++ b/crates/modelardb_server/src/storage/mod.rs @@ -84,10 +84,10 @@ impl StorageEngine { wal_mode: WalMode, configuration_manager: &Arc>, ) -> Result { - // Create shared memory pool. + // Create a shared memory pool. let configuration_manager = configuration_manager.read().await; let memory_pool = Arc::new(MemoryPool::new( - configuration_manager.multivariate_reserved_memory_in_bytes(), + configuration_manager.ingested_reserved_memory_in_bytes(), configuration_manager.uncompressed_reserved_memory_in_bytes(), configuration_manager.compressed_reserved_memory_in_bytes(), )); @@ -319,8 +319,8 @@ impl StorageEngine { Ok(()) } - /// Change the amount of memory for multivariate data in bytes according to `value_change`. - pub(super) async fn adjust_multivariate_remaining_memory_in_bytes(&self, value_change: i64) { + /// Change the amount of memory for ingested data in bytes according to `value_change`. + pub(super) async fn adjust_ingested_remaining_memory_in_bytes(&self, value_change: i64) { self.memory_pool.adjust_ingested_memory(value_change) } diff --git a/crates/modelardb_server/src/storage/types.rs b/crates/modelardb_server/src/storage/types.rs index e650722c..73d196cc 100644 --- a/crates/modelardb_server/src/storage/types.rs +++ b/crates/modelardb_server/src/storage/types.rs @@ -297,7 +297,7 @@ mod tests { // Tests for MemoryPool. #[test] - fn test_adjust_multivariate_memory_increase() { + fn test_adjust_ingested_memory_increase() { let memory_pool = create_memory_pool(); assert_eq!( memory_pool.remaining_uncompressed_memory_in_bytes(), @@ -313,7 +313,7 @@ mod tests { } #[test] - fn test_adjust_multivariate_memory_decrease_above_zero() { + fn test_adjust_ingested_memory_decrease_above_zero() { let memory_pool = create_memory_pool(); assert_eq!( memory_pool.remaining_ingested_memory_in_bytes(), @@ -329,7 +329,7 @@ mod tests { } #[test] - fn test_adjust_multivariate_memory_decrease_below_zero() { + fn test_adjust_ingested_memory_decrease_below_zero() { let memory_pool = create_memory_pool(); assert_eq!( memory_pool.remaining_ingested_memory_in_bytes(), @@ -345,7 +345,7 @@ mod tests { } #[test] - fn test_reserve_available_multivariate_memory() { + fn test_reserve_available_ingested_memory() { let memory_pool = create_memory_pool(); assert_eq!( memory_pool.remaining_ingested_memory_in_bytes(), diff --git a/crates/modelardb_server/tests/integration_test.rs b/crates/modelardb_server/tests/integration_test.rs index 164bbb84..3adf3c7e 100644 --- a/crates/modelardb_server/tests/integration_test.rs +++ b/crates/modelardb_server/tests/integration_test.rs @@ -1410,7 +1410,7 @@ async fn test_can_get_configuration() { let configuration = protocol::Configuration::decode(configuration_bytes).unwrap(); assert_eq!( - configuration.multivariate_reserved_memory_in_bytes, + configuration.ingested_reserved_memory_in_bytes, 512 * 1024 * 1024 ); assert_eq!( @@ -1436,16 +1436,13 @@ async fn test_can_get_configuration() { } #[tokio::test] -async fn test_can_update_multivariate_reserved_memory_in_bytes() { +async fn test_can_update_ingested_reserved_memory_in_bytes() { let updated_configuration = update_and_get_configuration( - protocol::update_configuration::Setting::MultivariateReservedMemoryInBytes as i32, + protocol::update_configuration::Setting::IngestedReservedMemoryInBytes as i32, ) .await; - assert_eq!( - updated_configuration.multivariate_reserved_memory_in_bytes, - 1 - ); + assert_eq!(updated_configuration.ingested_reserved_memory_in_bytes, 1); } #[tokio::test] @@ -1517,7 +1514,7 @@ async fn test_cannot_update_non_updatable_setting() { #[tokio::test] async fn test_cannot_update_non_nullable_setting_with_null_value() { for setting in [ - protocol::update_configuration::Setting::MultivariateReservedMemoryInBytes as i32, + protocol::update_configuration::Setting::IngestedReservedMemoryInBytes as i32, protocol::update_configuration::Setting::UncompressedReservedMemoryInBytes as i32, protocol::update_configuration::Setting::CompressedReservedMemoryInBytes as i32, protocol::update_configuration::Setting::SegmentSizeThresholdInBytes as i32, diff --git a/crates/modelardb_types/src/flight/protocol.proto b/crates/modelardb_types/src/flight/protocol.proto index 89dbbb9b..082f91b0 100644 --- a/crates/modelardb_types/src/flight/protocol.proto +++ b/crates/modelardb_types/src/flight/protocol.proto @@ -52,8 +52,8 @@ message TableMetadata { // Configuration of a ModelarDB node. message Configuration { - // Amount of memory to reserve for storing multivariate time series. - uint64 multivariate_reserved_memory_in_bytes = 1; + // Amount of memory to reserve for storing ingested time series. + uint64 ingested_reserved_memory_in_bytes = 1; // Amount of memory to reserve for storing uncompressed data buffers. uint64 uncompressed_reserved_memory_in_bytes = 2; @@ -84,7 +84,7 @@ message Configuration { // Request to update the configuration of a ModelarDB node. message UpdateConfiguration { enum Setting { - MULTIVARIATE_RESERVED_MEMORY_IN_BYTES = 0; + INGESTED_RESERVED_MEMORY_IN_BYTES = 0; UNCOMPRESSED_RESERVED_MEMORY_IN_BYTES = 1; COMPRESSED_RESERVED_MEMORY_IN_BYTES = 2; TRANSFER_BATCH_SIZE_IN_BYTES = 3; diff --git a/docs/dev/README.md b/docs/dev/README.md index 720d0492..b2a7caaa 100644 --- a/docs/dev/README.md +++ b/docs/dev/README.md @@ -18,6 +18,10 @@ exists, e.g., the bug report if it is a bugfix, and as a new GitHub issue otherw ## Structure The ModelarDB project consists of the following crates and major components: +- [modelardb_auth](/crates/modelardb_auth) - Library providing types to support authentication and authorization in + ModelarDB. + - **Authenticator** - Defines the `Authenticator` trait for validating credentials and authorizing access to + ModelarDB. - [modelardb_bulkloader](/crates/modelardb_bulkloader) - ModelarDB's command-line bulk loader in the form of the binary `modelardbb`. - [modelardb_client](/crates/modelardb_client) - ModelarDB's command-line client in the form of the binary `modelardb`. @@ -37,6 +41,8 @@ data folders from programming languages. - **Error** - Error type used throughout the crate, a single error type is used for simplicity. - **C-API** - A C-API for using modelardb_embedded from other programming languages through a C-FFI. - **ModelarDB** - Module providing functionality for reading from and writing to ModelarDB instances and data folders. +- [modelardb_macros](/crates/modelardb_macros) - Library providing the procedural macros used throughout ModelarDB. + - **Error** - Error type used throughout the crate, a single error type is used for simplicity. - [modelardb_server](/crates/modelardb_server) - ModelarDB's DBMS server in the form of the binary `modelardbd`. - **Storage** - Manages uncompressed data, compresses uncompressed data, manages compressed data, and writes compressed data to Delta Lake. diff --git a/docs/user/README.md b/docs/user/README.md index 0dfd82f9..6e7ca86f 100644 --- a/docs/user/README.md +++ b/docs/user/README.md @@ -385,7 +385,7 @@ file. Variables marked with ✓ in the **Updatable** column can also be updated | `--host` | `MODELARDBD_HOST` | 127.0.0.1 | | The host address of the `modelardbd` server. | | `--port` | `MODELARDBD_PORT` | 9999 | | The port of the `modelardbd` server. | | `--wal-enabled` | `MODELARDBD_WAL_ENABLED` | true | | Whether the write-ahead log is enabled. | -| `--multivariate-reserved-memory-in-bytes` | `MODELARDBD_MULTIVARIATE_RESERVED_MEMORY_IN_BYTES` | 512 MB | ✓ | The amount of memory to reserve for storing multivariate time series. | +| `--ingested-reserved-memory-in-bytes` | `MODELARDBD_INGESTED_RESERVED_MEMORY_IN_BYTES` | 512 MB | ✓ | The amount of memory to reserve for storing ingested time series. | | `--uncompressed-reserved-memory-in-bytes` | `MODELARDBD_UNCOMPRESSED_RESERVED_MEMORY_IN_BYTES` | 512 MB | ✓ | The amount of memory to reserve for storing uncompressed data buffers. | | `--compressed-reserved-memory-in-bytes` | `MODELARDBD_COMPRESSED_RESERVED_MEMORY_IN_BYTES` | 512 MB | ✓ | The amount of memory to reserve for storing compressed data buffers. | | `--transfer-batch-size-in-bytes` | `MODELARDBD_TRANSFER_BATCH_SIZE_IN_BYTES` | 64 MB | ✓ | The amount of data that must be collected before transferring a batch to the remote object store. |