diff --git a/Apache-Arrow-Flight-Tester/common.py b/Apache-Arrow-Flight-Tester/common.py index 759bccd..4fa144d 100644 --- a/Apache-Arrow-Flight-Tester/common.py +++ b/Apache-Arrow-Flight-Tester/common.py @@ -117,6 +117,10 @@ def clean_up_tables(self, tables: list[str], operation: Literal["drop", "truncat else self.truncate_table(table_name) ) + def vacuum(self, table_names: list[str]) -> None: + """Vacuum the given tables in the server or manager.""" + self.do_get(Ticket(f"VACUUM {', '.join(table_names)}")) + def node_type(self) -> str: """Return the type of the node.""" node_type = self.do_action("NodeType", b"") diff --git a/Apache-Arrow-Flight-Tester/protobuf/protocol.proto b/Apache-Arrow-Flight-Tester/protobuf/protocol.proto index bf97d40..77e1524 100644 --- a/Apache-Arrow-Flight-Tester/protobuf/protocol.proto +++ b/Apache-Arrow-Flight-Tester/protobuf/protocol.proto @@ -19,7 +19,7 @@ package modelardb.flight.protocol; // Metadata for the ModelarDB cluster manager, including its unique key and storage configuration. message ManagerMetadata { - // key used to uniquely identify the cluster manager. + // Key used to uniquely identify the cluster manager. string key = 1; // Storage configuration used to connect to an S3 object store. @@ -107,14 +107,17 @@ message Configuration { // The number of seconds between each transfer of data to the remote object store. optional uint64 transfer_time_in_seconds = 5; + // The number of seconds to retain deleted data in storage before it can be removed by vacuum. + uint64 retention_period_in_seconds = 6; + // Number of threads to allocate for converting multivariate time series to univariate time series. - uint32 ingestion_threads = 6; + uint32 ingestion_threads = 7; // Number of threads to allocate for compressing univariate time series to segments. - uint32 compression_threads = 7; + uint32 compression_threads = 8; // Number of threads to allocate for writing segments to a local and/or remote data folder. - uint32 writer_threads = 8; + uint32 writer_threads = 9; } // Request to update the configuration of a ModelarDB node. @@ -125,6 +128,7 @@ message UpdateConfiguration { COMPRESSED_RESERVED_MEMORY_IN_BYTES = 2; TRANSFER_BATCH_SIZE_IN_BYTES = 3; TRANSFER_TIME_IN_SECONDS = 4; + RETENTION_PERIOD_IN_SECONDS = 5; } // Setting to update in the configuration. diff --git a/Apache-Arrow-Flight-Tester/protobuf/protocol_pb2.py b/Apache-Arrow-Flight-Tester/protobuf/protocol_pb2.py index 28cdd3e..0dab8ac 100644 --- a/Apache-Arrow-Flight-Tester/protobuf/protocol_pb2.py +++ b/Apache-Arrow-Flight-Tester/protobuf/protocol_pb2.py @@ -24,7 +24,7 @@ -DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\x0eprotocol.proto\x12\x19modelardb.flight.protocol\"\xb1\x03\n\x0fManagerMetadata\x12\x0b\n\x03key\x18\x01 \x01(\t\x12V\n\x10s3_configuration\x18\x02 \x01(\x0b\x32:.modelardb.flight.protocol.ManagerMetadata.S3ConfigurationH\x00\x12\\\n\x13\x61zure_configuration\x18\x03 \x01(\x0b\x32=.modelardb.flight.protocol.ManagerMetadata.AzureConfigurationH\x00\x1aj\n\x0fS3Configuration\x12\x10\n\x08\x65ndpoint\x18\x01 \x01(\t\x12\x13\n\x0b\x62ucket_name\x18\x02 \x01(\t\x12\x15\n\raccess_key_id\x18\x03 \x01(\t\x12\x19\n\x11secret_access_key\x18\x04 \x01(\t\x1aV\n\x12\x41zureConfiguration\x12\x14\n\x0c\x61\x63\x63ount_name\x18\x01 \x01(\t\x12\x12\n\naccess_key\x18\x02 \x01(\t\x12\x16\n\x0e\x63ontainer_name\x18\x03 \x01(\tB\x17\n\x15storage_configuration\"\x87\x01\n\x0cNodeMetadata\x12\x0b\n\x03url\x18\x01 \x01(\t\x12G\n\x0bserver_mode\x18\x02 \x01(\x0e\x32\x32.modelardb.flight.protocol.NodeMetadata.ServerMode\"!\n\nServerMode\x12\t\n\x05\x43LOUD\x10\x00\x12\x08\n\x04\x45\x44GE\x10\x01\"\xdc\x04\n\rTableMetadata\x12S\n\rnormal_tables\x18\x01 \x03(\x0b\x32<.modelardb.flight.protocol.TableMetadata.NormalTableMetadata\x12\\\n\x12time_series_tables\x18\x02 \x03(\x0b\x32@.modelardb.flight.protocol.TableMetadata.TimeSeriesTableMetadata\x1a\x33\n\x13NormalTableMetadata\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x0e\n\x06schema\x18\x02 \x01(\x0c\x1a\xe2\x02\n\x17TimeSeriesTableMetadata\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x0e\n\x06schema\x18\x02 \x01(\x0c\x12\x61\n\x0c\x65rror_bounds\x18\x03 \x03(\x0b\x32K.modelardb.flight.protocol.TableMetadata.TimeSeriesTableMetadata.ErrorBound\x12$\n\x1cgenerated_column_expressions\x18\x04 \x03(\x0c\x1a\x9f\x01\n\nErrorBound\x12^\n\x04type\x18\x01 \x01(\x0e\x32P.modelardb.flight.protocol.TableMetadata.TimeSeriesTableMetadata.ErrorBound.Type\x12\r\n\x05value\x18\x02 \x01(\x02\"\"\n\x04Type\x12\x0c\n\x08\x41\x42SOLUTE\x10\x00\x12\x0c\n\x08RELATIVE\x10\x01\"\xfa\x02\n\rConfiguration\x12-\n%multivariate_reserved_memory_in_bytes\x18\x01 \x01(\x04\x12-\n%uncompressed_reserved_memory_in_bytes\x18\x02 \x01(\x04\x12+\n#compressed_reserved_memory_in_bytes\x18\x03 \x01(\x04\x12)\n\x1ctransfer_batch_size_in_bytes\x18\x04 \x01(\x04H\x00\x88\x01\x01\x12%\n\x18transfer_time_in_seconds\x18\x05 \x01(\x04H\x01\x88\x01\x01\x12\x19\n\x11ingestion_threads\x18\x06 \x01(\r\x12\x1b\n\x13\x63ompression_threads\x18\x07 \x01(\r\x12\x16\n\x0ewriter_threads\x18\x08 \x01(\rB\x1f\n\x1d_transfer_batch_size_in_bytesB\x1b\n\x19_transfer_time_in_seconds\"\xcf\x02\n\x13UpdateConfiguration\x12G\n\x07setting\x18\x01 \x01(\x0e\x32\x36.modelardb.flight.protocol.UpdateConfiguration.Setting\x12\x16\n\tnew_value\x18\x02 \x01(\x04H\x00\x88\x01\x01\"\xc8\x01\n\x07Setting\x12)\n%MULTIVARIATE_RESERVED_MEMORY_IN_BYTES\x10\x00\x12)\n%UNCOMPRESSED_RESERVED_MEMORY_IN_BYTES\x10\x01\x12\'\n#COMPRESSED_RESERVED_MEMORY_IN_BYTES\x10\x02\x12 \n\x1cTRANSFER_BATCH_SIZE_IN_BYTES\x10\x03\x12\x1c\n\x18TRANSFER_TIME_IN_SECONDS\x10\x04\x42\x0c\n\n_new_value\"\'\n\x10\x44\x61tabaseMetadata\x12\x13\n\x0btable_names\x18\x01 \x03(\tb\x06proto3') +DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\x0eprotocol.proto\x12\x19modelardb.flight.protocol\"\xb1\x03\n\x0fManagerMetadata\x12\x0b\n\x03key\x18\x01 \x01(\t\x12V\n\x10s3_configuration\x18\x02 \x01(\x0b\x32:.modelardb.flight.protocol.ManagerMetadata.S3ConfigurationH\x00\x12\\\n\x13\x61zure_configuration\x18\x03 \x01(\x0b\x32=.modelardb.flight.protocol.ManagerMetadata.AzureConfigurationH\x00\x1aj\n\x0fS3Configuration\x12\x10\n\x08\x65ndpoint\x18\x01 \x01(\t\x12\x13\n\x0b\x62ucket_name\x18\x02 \x01(\t\x12\x15\n\raccess_key_id\x18\x03 \x01(\t\x12\x19\n\x11secret_access_key\x18\x04 \x01(\t\x1aV\n\x12\x41zureConfiguration\x12\x14\n\x0c\x61\x63\x63ount_name\x18\x01 \x01(\t\x12\x12\n\naccess_key\x18\x02 \x01(\t\x12\x16\n\x0e\x63ontainer_name\x18\x03 \x01(\tB\x17\n\x15storage_configuration\"\x87\x01\n\x0cNodeMetadata\x12\x0b\n\x03url\x18\x01 \x01(\t\x12G\n\x0bserver_mode\x18\x02 \x01(\x0e\x32\x32.modelardb.flight.protocol.NodeMetadata.ServerMode\"!\n\nServerMode\x12\t\n\x05\x43LOUD\x10\x00\x12\x08\n\x04\x45\x44GE\x10\x01\"\xdc\x04\n\rTableMetadata\x12S\n\rnormal_tables\x18\x01 \x03(\x0b\x32<.modelardb.flight.protocol.TableMetadata.NormalTableMetadata\x12\\\n\x12time_series_tables\x18\x02 \x03(\x0b\x32@.modelardb.flight.protocol.TableMetadata.TimeSeriesTableMetadata\x1a\x33\n\x13NormalTableMetadata\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x0e\n\x06schema\x18\x02 \x01(\x0c\x1a\xe2\x02\n\x17TimeSeriesTableMetadata\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x0e\n\x06schema\x18\x02 \x01(\x0c\x12\x61\n\x0c\x65rror_bounds\x18\x03 \x03(\x0b\x32K.modelardb.flight.protocol.TableMetadata.TimeSeriesTableMetadata.ErrorBound\x12$\n\x1cgenerated_column_expressions\x18\x04 \x03(\x0c\x1a\x9f\x01\n\nErrorBound\x12^\n\x04type\x18\x01 \x01(\x0e\x32P.modelardb.flight.protocol.TableMetadata.TimeSeriesTableMetadata.ErrorBound.Type\x12\r\n\x05value\x18\x02 \x01(\x02\"\"\n\x04Type\x12\x0c\n\x08\x41\x42SOLUTE\x10\x00\x12\x0c\n\x08RELATIVE\x10\x01\"\x9f\x03\n\rConfiguration\x12-\n%multivariate_reserved_memory_in_bytes\x18\x01 \x01(\x04\x12-\n%uncompressed_reserved_memory_in_bytes\x18\x02 \x01(\x04\x12+\n#compressed_reserved_memory_in_bytes\x18\x03 \x01(\x04\x12)\n\x1ctransfer_batch_size_in_bytes\x18\x04 \x01(\x04H\x00\x88\x01\x01\x12%\n\x18transfer_time_in_seconds\x18\x05 \x01(\x04H\x01\x88\x01\x01\x12#\n\x1bretention_period_in_seconds\x18\x06 \x01(\x04\x12\x19\n\x11ingestion_threads\x18\x07 \x01(\r\x12\x1b\n\x13\x63ompression_threads\x18\x08 \x01(\r\x12\x16\n\x0ewriter_threads\x18\t \x01(\rB\x1f\n\x1d_transfer_batch_size_in_bytesB\x1b\n\x19_transfer_time_in_seconds\"\xf0\x02\n\x13UpdateConfiguration\x12G\n\x07setting\x18\x01 \x01(\x0e\x32\x36.modelardb.flight.protocol.UpdateConfiguration.Setting\x12\x16\n\tnew_value\x18\x02 \x01(\x04H\x00\x88\x01\x01\"\xe9\x01\n\x07Setting\x12)\n%MULTIVARIATE_RESERVED_MEMORY_IN_BYTES\x10\x00\x12)\n%UNCOMPRESSED_RESERVED_MEMORY_IN_BYTES\x10\x01\x12\'\n#COMPRESSED_RESERVED_MEMORY_IN_BYTES\x10\x02\x12 \n\x1cTRANSFER_BATCH_SIZE_IN_BYTES\x10\x03\x12\x1c\n\x18TRANSFER_TIME_IN_SECONDS\x10\x04\x12\x1f\n\x1bRETENTION_PERIOD_IN_SECONDS\x10\x05\x42\x0c\n\n_new_value\"\'\n\x10\x44\x61tabaseMetadata\x12\x13\n\x0btable_names\x18\x01 \x03(\tb\x06proto3') _globals = globals() _builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, _globals) @@ -52,11 +52,11 @@ _globals['_TABLEMETADATA_TIMESERIESTABLEMETADATA_ERRORBOUND_TYPE']._serialized_start=1190 _globals['_TABLEMETADATA_TIMESERIESTABLEMETADATA_ERRORBOUND_TYPE']._serialized_end=1224 _globals['_CONFIGURATION']._serialized_start=1227 - _globals['_CONFIGURATION']._serialized_end=1605 - _globals['_UPDATECONFIGURATION']._serialized_start=1608 - _globals['_UPDATECONFIGURATION']._serialized_end=1943 - _globals['_UPDATECONFIGURATION_SETTING']._serialized_start=1729 - _globals['_UPDATECONFIGURATION_SETTING']._serialized_end=1929 - _globals['_DATABASEMETADATA']._serialized_start=1945 - _globals['_DATABASEMETADATA']._serialized_end=1984 + _globals['_CONFIGURATION']._serialized_end=1642 + _globals['_UPDATECONFIGURATION']._serialized_start=1645 + _globals['_UPDATECONFIGURATION']._serialized_end=2013 + _globals['_UPDATECONFIGURATION_SETTING']._serialized_start=1766 + _globals['_UPDATECONFIGURATION_SETTING']._serialized_end=1999 + _globals['_DATABASEMETADATA']._serialized_start=2015 + _globals['_DATABASEMETADATA']._serialized_end=2054 # @@protoc_insertion_point(module_scope) diff --git a/Apache-Arrow-Flight-Tester/protobuf/protocol_pb2.pyi b/Apache-Arrow-Flight-Tester/protobuf/protocol_pb2.pyi index 5148c93..63a75b5 100644 --- a/Apache-Arrow-Flight-Tester/protobuf/protocol_pb2.pyi +++ b/Apache-Arrow-Flight-Tester/protobuf/protocol_pb2.pyi @@ -91,12 +91,13 @@ class TableMetadata(_message.Message): def __init__(self, normal_tables: _Optional[_Iterable[_Union[TableMetadata.NormalTableMetadata, _Mapping]]] = ..., time_series_tables: _Optional[_Iterable[_Union[TableMetadata.TimeSeriesTableMetadata, _Mapping]]] = ...) -> None: ... class Configuration(_message.Message): - __slots__ = ("multivariate_reserved_memory_in_bytes", "uncompressed_reserved_memory_in_bytes", "compressed_reserved_memory_in_bytes", "transfer_batch_size_in_bytes", "transfer_time_in_seconds", "ingestion_threads", "compression_threads", "writer_threads") + __slots__ = ("multivariate_reserved_memory_in_bytes", "uncompressed_reserved_memory_in_bytes", "compressed_reserved_memory_in_bytes", "transfer_batch_size_in_bytes", "transfer_time_in_seconds", "retention_period_in_seconds", "ingestion_threads", "compression_threads", "writer_threads") MULTIVARIATE_RESERVED_MEMORY_IN_BYTES_FIELD_NUMBER: _ClassVar[int] UNCOMPRESSED_RESERVED_MEMORY_IN_BYTES_FIELD_NUMBER: _ClassVar[int] COMPRESSED_RESERVED_MEMORY_IN_BYTES_FIELD_NUMBER: _ClassVar[int] TRANSFER_BATCH_SIZE_IN_BYTES_FIELD_NUMBER: _ClassVar[int] TRANSFER_TIME_IN_SECONDS_FIELD_NUMBER: _ClassVar[int] + RETENTION_PERIOD_IN_SECONDS_FIELD_NUMBER: _ClassVar[int] INGESTION_THREADS_FIELD_NUMBER: _ClassVar[int] COMPRESSION_THREADS_FIELD_NUMBER: _ClassVar[int] WRITER_THREADS_FIELD_NUMBER: _ClassVar[int] @@ -105,10 +106,11 @@ class Configuration(_message.Message): compressed_reserved_memory_in_bytes: int transfer_batch_size_in_bytes: int transfer_time_in_seconds: int + retention_period_in_seconds: int ingestion_threads: int compression_threads: int writer_threads: int - def __init__(self, multivariate_reserved_memory_in_bytes: _Optional[int] = ..., uncompressed_reserved_memory_in_bytes: _Optional[int] = ..., compressed_reserved_memory_in_bytes: _Optional[int] = ..., transfer_batch_size_in_bytes: _Optional[int] = ..., transfer_time_in_seconds: _Optional[int] = ..., ingestion_threads: _Optional[int] = ..., compression_threads: _Optional[int] = ..., writer_threads: _Optional[int] = ...) -> None: ... + def __init__(self, multivariate_reserved_memory_in_bytes: _Optional[int] = ..., uncompressed_reserved_memory_in_bytes: _Optional[int] = ..., compressed_reserved_memory_in_bytes: _Optional[int] = ..., transfer_batch_size_in_bytes: _Optional[int] = ..., transfer_time_in_seconds: _Optional[int] = ..., retention_period_in_seconds: _Optional[int] = ..., ingestion_threads: _Optional[int] = ..., compression_threads: _Optional[int] = ..., writer_threads: _Optional[int] = ...) -> None: ... class UpdateConfiguration(_message.Message): __slots__ = ("setting", "new_value") @@ -119,11 +121,13 @@ class UpdateConfiguration(_message.Message): COMPRESSED_RESERVED_MEMORY_IN_BYTES: _ClassVar[UpdateConfiguration.Setting] TRANSFER_BATCH_SIZE_IN_BYTES: _ClassVar[UpdateConfiguration.Setting] TRANSFER_TIME_IN_SECONDS: _ClassVar[UpdateConfiguration.Setting] + RETENTION_PERIOD_IN_SECONDS: _ClassVar[UpdateConfiguration.Setting] MULTIVARIATE_RESERVED_MEMORY_IN_BYTES: UpdateConfiguration.Setting UNCOMPRESSED_RESERVED_MEMORY_IN_BYTES: UpdateConfiguration.Setting COMPRESSED_RESERVED_MEMORY_IN_BYTES: UpdateConfiguration.Setting TRANSFER_BATCH_SIZE_IN_BYTES: UpdateConfiguration.Setting TRANSFER_TIME_IN_SECONDS: UpdateConfiguration.Setting + RETENTION_PERIOD_IN_SECONDS: UpdateConfiguration.Setting SETTING_FIELD_NUMBER: _ClassVar[int] NEW_VALUE_FIELD_NUMBER: _ClassVar[int] setting: UpdateConfiguration.Setting