Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 3 additions & 1 deletion crates/modelardb_auth/src/lib.rs
Original file line number Diff line number Diff line change
Expand Up @@ -61,7 +61,9 @@ impl BearerInterceptor {
.map(|token| {
format!("Bearer {token}")
.parse::<AsciiMetadataValue>()
.map_err(|error| Status::invalid_argument(format!("Token is not ASCII: {error}.")))
.map_err(|error| {
Status::invalid_argument(format!("Token is not ASCII: {error}."))
})
})
.transpose()?;

Expand Down
46 changes: 22 additions & 24 deletions crates/modelardb_server/src/configuration.rs
Original file line number Diff line number Diff line change
Expand Up @@ -51,8 +51,8 @@ pub(crate) enum WalMode {
/// only be done through the [`ConfigurationManager`].
#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
struct Configuration {
/// Amount of memory to reserve for storing multivariate time series.
multivariate_reserved_memory_in_bytes: u64,
/// Amount of memory to reserve for storing ingested time series.
ingested_reserved_memory_in_bytes: u64,
/// Amount of memory to reserve for storing uncompressed data buffers.
uncompressed_reserved_memory_in_bytes: u64,
/// Amount of memory to reserve for storing compressed data buffers.
Expand All @@ -79,8 +79,8 @@ impl Configuration {
/// Update the configuration parameters with the corresponding flags or environment variables
/// from the command line if they are set.
fn update_from_args(&mut self, args: &ServerArgs) {
if let Some(value) = args.multivariate_reserved_memory_in_bytes {
self.multivariate_reserved_memory_in_bytes = value;
if let Some(value) = args.ingested_reserved_memory_in_bytes {
self.ingested_reserved_memory_in_bytes = value;
}

if let Some(value) = args.uncompressed_reserved_memory_in_bytes {
Expand Down Expand Up @@ -142,7 +142,7 @@ impl Configuration {
impl Default for Configuration {
fn default() -> Self {
Self {
multivariate_reserved_memory_in_bytes: 512 * 1024 * 1024,
ingested_reserved_memory_in_bytes: 512 * 1024 * 1024,
uncompressed_reserved_memory_in_bytes: 512 * 1024 * 1024,
compressed_reserved_memory_in_bytes: 512 * 1024 * 1024,
transfer_batch_size_in_bytes: Some(64 * 1024 * 1024),
Expand Down Expand Up @@ -237,31 +237,31 @@ impl ConfigurationManager {
&self.wal_mode
}

pub(crate) fn multivariate_reserved_memory_in_bytes(&self) -> u64 {
self.configuration.multivariate_reserved_memory_in_bytes
pub(crate) fn ingested_reserved_memory_in_bytes(&self) -> u64 {
self.configuration.ingested_reserved_memory_in_bytes
}

/// Set the new value and update the amount of memory for multivariate data in the storage
/// Set the new value and update the amount of memory for ingested data in the storage
/// engine. If the new configuration could not be saved to the configuration file, return
/// [`ModelarDbServerError`].
pub(crate) async fn set_multivariate_reserved_memory_in_bytes(
pub(crate) async fn set_ingested_reserved_memory_in_bytes(
&mut self,
new_multivariate_reserved_memory_in_bytes: u64,
new_ingested_reserved_memory_in_bytes: u64,
storage_engine: Arc<RwLock<StorageEngine>>,
) -> Result<()> {
// Since the storage engine only keeps track of the remaining reserved memory, calculate
// how much the value should change.
let value_change = new_multivariate_reserved_memory_in_bytes as i64
- self.configuration.multivariate_reserved_memory_in_bytes as i64;
let value_change = new_ingested_reserved_memory_in_bytes as i64
- self.configuration.ingested_reserved_memory_in_bytes as i64;

storage_engine
.write()
.await
.adjust_multivariate_remaining_memory_in_bytes(value_change)
.adjust_ingested_remaining_memory_in_bytes(value_change)
.await;

self.configuration.multivariate_reserved_memory_in_bytes =
new_multivariate_reserved_memory_in_bytes;
self.configuration.ingested_reserved_memory_in_bytes =
new_ingested_reserved_memory_in_bytes;

self.configuration
.save_to_toml(&self.local_data_folder)
Expand Down Expand Up @@ -405,9 +405,7 @@ impl ConfigurationManager {
/// protobuf message and serialize it.
pub(crate) fn encode_and_serialize(&self) -> Vec<u8> {
let configuration = protocol::Configuration {
multivariate_reserved_memory_in_bytes: self
.configuration
.multivariate_reserved_memory_in_bytes,
ingested_reserved_memory_in_bytes: self.configuration.ingested_reserved_memory_in_bytes,
uncompressed_reserved_memory_in_bytes: self
.configuration
.uncompressed_reserved_memory_in_bytes,
Expand Down Expand Up @@ -461,7 +459,7 @@ mod tests {
let local_data_folder = DataFolder::open_local_url(local_url).await.unwrap();

let existing_configuration = Configuration {
multivariate_reserved_memory_in_bytes: 1,
ingested_reserved_memory_in_bytes: 1,
uncompressed_reserved_memory_in_bytes: 1,
compressed_reserved_memory_in_bytes: 1,
transfer_batch_size_in_bytes: Some(1),
Expand Down Expand Up @@ -540,37 +538,37 @@ mod tests {
}

#[tokio::test]
async fn test_set_multivariate_reserved_memory_in_bytes() {
async fn test_set_ingested_reserved_memory_in_bytes() {
let temp_dir = tempfile::tempdir().unwrap();
let (storage_engine, configuration_manager) = create_components(&temp_dir).await;

assert_eq!(
configuration_manager
.read()
.await
.multivariate_reserved_memory_in_bytes(),
.ingested_reserved_memory_in_bytes(),
512 * 1024 * 1024
);

let new_value = 1024;
configuration_manager
.write()
.await
.set_multivariate_reserved_memory_in_bytes(new_value, storage_engine)
.set_ingested_reserved_memory_in_bytes(new_value, storage_engine)
.await
.unwrap();

assert_eq!(
configuration_manager
.read()
.await
.multivariate_reserved_memory_in_bytes(),
.ingested_reserved_memory_in_bytes(),
new_value
);

let configuration_from_file = configuration_from_file(&temp_dir).await;
assert_eq!(
configuration_from_file.multivariate_reserved_memory_in_bytes,
configuration_from_file.ingested_reserved_memory_in_bytes,
new_value
);
}
Expand Down
6 changes: 3 additions & 3 deletions crates/modelardb_server/src/main.rs
Original file line number Diff line number Diff line change
Expand Up @@ -61,9 +61,9 @@ pub(crate) struct ServerArgs {
#[arg(long, default_value_t = 9999, env = "MODELARDBD_PORT")]
port: u16,

/// Amount of memory in bytes to reserve for storing multivariate time series.
#[arg(long, env = "MODELARDBD_MULTIVARIATE_RESERVED_MEMORY_IN_BYTES")]
multivariate_reserved_memory_in_bytes: Option<u64>,
/// Amount of memory in bytes to reserve for storing ingested time series.
#[arg(long, env = "MODELARDBD_INGESTED_RESERVED_MEMORY_IN_BYTES")]
ingested_reserved_memory_in_bytes: Option<u64>,

/// Amount of memory in bytes to reserve for storing uncompressed data buffers.
#[arg(long, env = "MODELARDBD_UNCOMPRESSED_RESERVED_MEMORY_IN_BYTES")]
Expand Down
6 changes: 3 additions & 3 deletions crates/modelardb_server/src/remote/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -67,7 +67,7 @@ use crate::remote::auth_layer::AuthLayer;
/// Start an Apache Arrow Flight server on 0.0.0.0:`port` that passes `context` to the methods that
/// process the requests through [`FlightServiceHandler`]. All requests are passed through the
/// [`AuthLayer`], which authenticates them using `maybe_authenticator` before they are passed to
/// the [`FlightServiceHandler`]. If `maybe_authenticator` is [`None`], authentication is disabled,
/// the [`FlightServiceHandler`]. If `maybe_authenticator` is [`None`], authentication is disabled,
/// and every request that is not an internal cluster request is allowed.
pub async fn start_apache_arrow_flight_server(
context: Arc<Context>,
Expand Down Expand Up @@ -983,11 +983,11 @@ impl FlightService for FlightServiceHandler {
Status::invalid_argument(format!("New value for {setting} cannot be null."));

match protocol::update_configuration::Setting::try_from(setting) {
Ok(protocol::update_configuration::Setting::MultivariateReservedMemoryInBytes) => {
Ok(protocol::update_configuration::Setting::IngestedReservedMemoryInBytes) => {
let new_value = maybe_new_value.ok_or(invalid_null_error)?;

configuration_manager
.set_multivariate_reserved_memory_in_bytes(new_value, storage_engine)
.set_ingested_reserved_memory_in_bytes(new_value, storage_engine)
.await
.map_err(error_to_status_internal)
}
Expand Down
8 changes: 4 additions & 4 deletions crates/modelardb_server/src/storage/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -84,10 +84,10 @@ impl StorageEngine {
wal_mode: WalMode,
configuration_manager: &Arc<RwLock<ConfigurationManager>>,
) -> Result<Self> {
// Create shared memory pool.
// Create a shared memory pool.
let configuration_manager = configuration_manager.read().await;
let memory_pool = Arc::new(MemoryPool::new(
configuration_manager.multivariate_reserved_memory_in_bytes(),
configuration_manager.ingested_reserved_memory_in_bytes(),
configuration_manager.uncompressed_reserved_memory_in_bytes(),
configuration_manager.compressed_reserved_memory_in_bytes(),
));
Expand Down Expand Up @@ -319,8 +319,8 @@ impl StorageEngine {
Ok(())
}

/// Change the amount of memory for multivariate data in bytes according to `value_change`.
pub(super) async fn adjust_multivariate_remaining_memory_in_bytes(&self, value_change: i64) {
/// Change the amount of memory for ingested data in bytes according to `value_change`.
pub(super) async fn adjust_ingested_remaining_memory_in_bytes(&self, value_change: i64) {
self.memory_pool.adjust_ingested_memory(value_change)
}

Expand Down
8 changes: 4 additions & 4 deletions crates/modelardb_server/src/storage/types.rs
Original file line number Diff line number Diff line change
Expand Up @@ -297,7 +297,7 @@ mod tests {

// Tests for MemoryPool.
#[test]
fn test_adjust_multivariate_memory_increase() {
fn test_adjust_ingested_memory_increase() {
let memory_pool = create_memory_pool();
assert_eq!(
memory_pool.remaining_uncompressed_memory_in_bytes(),
Expand All @@ -313,7 +313,7 @@ mod tests {
}

#[test]
fn test_adjust_multivariate_memory_decrease_above_zero() {
fn test_adjust_ingested_memory_decrease_above_zero() {
let memory_pool = create_memory_pool();
assert_eq!(
memory_pool.remaining_ingested_memory_in_bytes(),
Expand All @@ -329,7 +329,7 @@ mod tests {
}

#[test]
fn test_adjust_multivariate_memory_decrease_below_zero() {
fn test_adjust_ingested_memory_decrease_below_zero() {
let memory_pool = create_memory_pool();
assert_eq!(
memory_pool.remaining_ingested_memory_in_bytes(),
Expand All @@ -345,7 +345,7 @@ mod tests {
}

#[test]
fn test_reserve_available_multivariate_memory() {
fn test_reserve_available_ingested_memory() {
let memory_pool = create_memory_pool();
assert_eq!(
memory_pool.remaining_ingested_memory_in_bytes(),
Expand Down
13 changes: 5 additions & 8 deletions crates/modelardb_server/tests/integration_test.rs
Original file line number Diff line number Diff line change
Expand Up @@ -1410,7 +1410,7 @@ async fn test_can_get_configuration() {
let configuration = protocol::Configuration::decode(configuration_bytes).unwrap();

assert_eq!(
configuration.multivariate_reserved_memory_in_bytes,
configuration.ingested_reserved_memory_in_bytes,
512 * 1024 * 1024
);
assert_eq!(
Expand All @@ -1436,16 +1436,13 @@ async fn test_can_get_configuration() {
}

#[tokio::test]
async fn test_can_update_multivariate_reserved_memory_in_bytes() {
async fn test_can_update_ingested_reserved_memory_in_bytes() {
let updated_configuration = update_and_get_configuration(
protocol::update_configuration::Setting::MultivariateReservedMemoryInBytes as i32,
protocol::update_configuration::Setting::IngestedReservedMemoryInBytes as i32,
)
.await;

assert_eq!(
updated_configuration.multivariate_reserved_memory_in_bytes,
1
);
assert_eq!(updated_configuration.ingested_reserved_memory_in_bytes, 1);
}

#[tokio::test]
Expand Down Expand Up @@ -1517,7 +1514,7 @@ async fn test_cannot_update_non_updatable_setting() {
#[tokio::test]
async fn test_cannot_update_non_nullable_setting_with_null_value() {
for setting in [
protocol::update_configuration::Setting::MultivariateReservedMemoryInBytes as i32,
protocol::update_configuration::Setting::IngestedReservedMemoryInBytes as i32,
protocol::update_configuration::Setting::UncompressedReservedMemoryInBytes as i32,
protocol::update_configuration::Setting::CompressedReservedMemoryInBytes as i32,
protocol::update_configuration::Setting::SegmentSizeThresholdInBytes as i32,
Expand Down
6 changes: 3 additions & 3 deletions crates/modelardb_types/src/flight/protocol.proto
Original file line number Diff line number Diff line change
Expand Up @@ -52,8 +52,8 @@ message TableMetadata {

// Configuration of a ModelarDB node.
message Configuration {
// Amount of memory to reserve for storing multivariate time series.
uint64 multivariate_reserved_memory_in_bytes = 1;
// Amount of memory to reserve for storing ingested time series.
uint64 ingested_reserved_memory_in_bytes = 1;

// Amount of memory to reserve for storing uncompressed data buffers.
uint64 uncompressed_reserved_memory_in_bytes = 2;
Expand Down Expand Up @@ -84,7 +84,7 @@ message Configuration {
// Request to update the configuration of a ModelarDB node.
message UpdateConfiguration {
enum Setting {
MULTIVARIATE_RESERVED_MEMORY_IN_BYTES = 0;
INGESTED_RESERVED_MEMORY_IN_BYTES = 0;
UNCOMPRESSED_RESERVED_MEMORY_IN_BYTES = 1;
COMPRESSED_RESERVED_MEMORY_IN_BYTES = 2;
TRANSFER_BATCH_SIZE_IN_BYTES = 3;
Expand Down
6 changes: 6 additions & 0 deletions docs/dev/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,10 @@ exists, e.g., the bug report if it is a bugfix, and as a new GitHub issue otherw
## Structure
The ModelarDB project consists of the following crates and major components:

- [modelardb_auth](/crates/modelardb_auth) - Library providing types to support authentication and authorization in
ModelarDB.
- **Authenticator** - Defines the `Authenticator` trait for validating credentials and authorizing access to
ModelarDB.
- [modelardb_bulkloader](/crates/modelardb_bulkloader) - ModelarDB's command-line bulk loader in the form of the binary
`modelardbb`.
- [modelardb_client](/crates/modelardb_client) - ModelarDB's command-line client in the form of the binary `modelardb`.
Expand All @@ -37,6 +41,8 @@ data folders from programming languages.
- **Error** - Error type used throughout the crate, a single error type is used for simplicity.
- **C-API** - A C-API for using modelardb_embedded from other programming languages through a C-FFI.
- **ModelarDB** - Module providing functionality for reading from and writing to ModelarDB instances and data folders.
- [modelardb_macros](/crates/modelardb_macros) - Library providing the procedural macros used throughout ModelarDB.
- **Error** - Error type used throughout the crate, a single error type is used for simplicity.
- [modelardb_server](/crates/modelardb_server) - ModelarDB's DBMS server in the form of the binary `modelardbd`.
- **Storage** - Manages uncompressed data, compresses uncompressed data, manages compressed data, and writes
compressed data to Delta Lake.
Expand Down
2 changes: 1 addition & 1 deletion docs/user/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -385,7 +385,7 @@ file. Variables marked with ✓ in the **Updatable** column can also be updated
| `--host` | `MODELARDBD_HOST` | 127.0.0.1 | | The host address of the `modelardbd` server. |
| `--port` | `MODELARDBD_PORT` | 9999 | | The port of the `modelardbd` server. |
| `--wal-enabled` | `MODELARDBD_WAL_ENABLED` | true | | Whether the write-ahead log is enabled. |
| `--multivariate-reserved-memory-in-bytes` | `MODELARDBD_MULTIVARIATE_RESERVED_MEMORY_IN_BYTES` | 512 MB | ✓ | The amount of memory to reserve for storing multivariate time series. |
| `--ingested-reserved-memory-in-bytes` | `MODELARDBD_INGESTED_RESERVED_MEMORY_IN_BYTES` | 512 MB | ✓ | The amount of memory to reserve for storing ingested time series. |
| `--uncompressed-reserved-memory-in-bytes` | `MODELARDBD_UNCOMPRESSED_RESERVED_MEMORY_IN_BYTES` | 512 MB | ✓ | The amount of memory to reserve for storing uncompressed data buffers. |
| `--compressed-reserved-memory-in-bytes` | `MODELARDBD_COMPRESSED_RESERVED_MEMORY_IN_BYTES` | 512 MB | ✓ | The amount of memory to reserve for storing compressed data buffers. |
| `--transfer-batch-size-in-bytes` | `MODELARDBD_TRANSFER_BATCH_SIZE_IN_BYTES` | 64 MB | ✓ | The amount of data that must be collected before transferring a batch to the remote object store. |
Expand Down
Loading