diff --git a/Apache-Arrow-Flight-Tester/common.py b/Apache-Arrow-Flight-Tester/common.py index 32bbf08..3608dbc 100644 --- a/Apache-Arrow-Flight-Tester/common.py +++ b/Apache-Arrow-Flight-Tester/common.py @@ -137,8 +137,8 @@ def create_test_tables_from_metadata(self): time_series_table_schema = get_time_series_table_schema() - absolute = protocol_pb2.TableMetadata.TimeSeriesTableMetadata.ErrorBound.Type.ABSOLUTE - error_bounds = [protocol_pb2.TableMetadata.TimeSeriesTableMetadata.ErrorBound(value=0, type=absolute) + lossless = protocol_pb2.TableMetadata.TimeSeriesTableMetadata.ErrorBound.Type.LOSSLESS + error_bounds = [protocol_pb2.TableMetadata.TimeSeriesTableMetadata.ErrorBound(value=0, type=lossless) for _ in range(len(time_series_table_schema))] generated_column_expressions = [b'' for _ in range(len(time_series_table_schema))] diff --git a/Apache-Arrow-Flight-Tester/protobuf/protocol.proto b/Apache-Arrow-Flight-Tester/protobuf/protocol.proto index 5ddec35..85242ac 100644 --- a/Apache-Arrow-Flight-Tester/protobuf/protocol.proto +++ b/Apache-Arrow-Flight-Tester/protobuf/protocol.proto @@ -72,6 +72,7 @@ message TableMetadata { enum Type { ABSOLUTE = 0; RELATIVE = 1; + LOSSLESS = 2; } Type type = 1; float value = 2; @@ -107,17 +108,14 @@ message Configuration { // The number of seconds between each transfer of data to the remote object store. optional uint64 transfer_time_in_seconds = 5; - // The number of seconds to retain deleted data in storage before it can be removed by vacuum. - uint64 retention_period_in_seconds = 6; - // Number of threads to allocate for converting multivariate time series to univariate time series. - uint32 ingestion_threads = 7; + uint32 ingestion_threads = 6; // Number of threads to allocate for compressing univariate time series to segments. - uint32 compression_threads = 8; + uint32 compression_threads = 7; // Number of threads to allocate for writing segments to a local and/or remote data folder. - uint32 writer_threads = 9; + uint32 writer_threads = 8; } // Request to update the configuration of a ModelarDB node. @@ -128,7 +126,6 @@ message UpdateConfiguration { COMPRESSED_RESERVED_MEMORY_IN_BYTES = 2; TRANSFER_BATCH_SIZE_IN_BYTES = 3; TRANSFER_TIME_IN_SECONDS = 4; - RETENTION_PERIOD_IN_SECONDS = 5; } // Setting to update in the configuration. diff --git a/Apache-Arrow-Flight-Tester/protobuf/protocol_pb2.py b/Apache-Arrow-Flight-Tester/protobuf/protocol_pb2.py index 88c1a37..9d868a0 100644 --- a/Apache-Arrow-Flight-Tester/protobuf/protocol_pb2.py +++ b/Apache-Arrow-Flight-Tester/protobuf/protocol_pb2.py @@ -24,7 +24,7 @@ -DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\x0eprotocol.proto\x12\x19modelardb.flight.protocol\"\xb1\x03\n\x0fManagerMetadata\x12\x0b\n\x03key\x18\x01 \x01(\t\x12V\n\x10s3_configuration\x18\x02 \x01(\x0b\x32:.modelardb.flight.protocol.ManagerMetadata.S3ConfigurationH\x00\x12\\\n\x13\x61zure_configuration\x18\x03 \x01(\x0b\x32=.modelardb.flight.protocol.ManagerMetadata.AzureConfigurationH\x00\x1aj\n\x0fS3Configuration\x12\x10\n\x08\x65ndpoint\x18\x01 \x01(\t\x12\x13\n\x0b\x62ucket_name\x18\x02 \x01(\t\x12\x15\n\raccess_key_id\x18\x03 \x01(\t\x12\x19\n\x11secret_access_key\x18\x04 \x01(\t\x1aV\n\x12\x41zureConfiguration\x12\x14\n\x0c\x61\x63\x63ount_name\x18\x01 \x01(\t\x12\x12\n\naccess_key\x18\x02 \x01(\t\x12\x16\n\x0e\x63ontainer_name\x18\x03 \x01(\tB\x17\n\x15storage_configuration\"\x87\x01\n\x0cNodeMetadata\x12\x0b\n\x03url\x18\x01 \x01(\t\x12G\n\x0bserver_mode\x18\x02 \x01(\x0e\x32\x32.modelardb.flight.protocol.NodeMetadata.ServerMode\"!\n\nServerMode\x12\t\n\x05\x43LOUD\x10\x00\x12\x08\n\x04\x45\x44GE\x10\x01\"\xf0\x04\n\rTableMetadata\x12T\n\x0cnormal_table\x18\x01 \x01(\x0b\x32<.modelardb.flight.protocol.TableMetadata.NormalTableMetadataH\x00\x12]\n\x11time_series_table\x18\x02 \x01(\x0b\x32@.modelardb.flight.protocol.TableMetadata.TimeSeriesTableMetadataH\x00\x1a\x33\n\x13NormalTableMetadata\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x0e\n\x06schema\x18\x02 \x01(\x0c\x1a\xe2\x02\n\x17TimeSeriesTableMetadata\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x0e\n\x06schema\x18\x02 \x01(\x0c\x12\x61\n\x0c\x65rror_bounds\x18\x03 \x03(\x0b\x32K.modelardb.flight.protocol.TableMetadata.TimeSeriesTableMetadata.ErrorBound\x12$\n\x1cgenerated_column_expressions\x18\x04 \x03(\x0c\x1a\x9f\x01\n\nErrorBound\x12^\n\x04type\x18\x01 \x01(\x0e\x32P.modelardb.flight.protocol.TableMetadata.TimeSeriesTableMetadata.ErrorBound.Type\x12\r\n\x05value\x18\x02 \x01(\x02\"\"\n\x04Type\x12\x0c\n\x08\x41\x42SOLUTE\x10\x00\x12\x0c\n\x08RELATIVE\x10\x01\x42\x10\n\x0etable_metadata\"\x9f\x03\n\rConfiguration\x12-\n%multivariate_reserved_memory_in_bytes\x18\x01 \x01(\x04\x12-\n%uncompressed_reserved_memory_in_bytes\x18\x02 \x01(\x04\x12+\n#compressed_reserved_memory_in_bytes\x18\x03 \x01(\x04\x12)\n\x1ctransfer_batch_size_in_bytes\x18\x04 \x01(\x04H\x00\x88\x01\x01\x12%\n\x18transfer_time_in_seconds\x18\x05 \x01(\x04H\x01\x88\x01\x01\x12#\n\x1bretention_period_in_seconds\x18\x06 \x01(\x04\x12\x19\n\x11ingestion_threads\x18\x07 \x01(\r\x12\x1b\n\x13\x63ompression_threads\x18\x08 \x01(\r\x12\x16\n\x0ewriter_threads\x18\t \x01(\rB\x1f\n\x1d_transfer_batch_size_in_bytesB\x1b\n\x19_transfer_time_in_seconds\"\xf0\x02\n\x13UpdateConfiguration\x12G\n\x07setting\x18\x01 \x01(\x0e\x32\x36.modelardb.flight.protocol.UpdateConfiguration.Setting\x12\x16\n\tnew_value\x18\x02 \x01(\x04H\x00\x88\x01\x01\"\xe9\x01\n\x07Setting\x12)\n%MULTIVARIATE_RESERVED_MEMORY_IN_BYTES\x10\x00\x12)\n%UNCOMPRESSED_RESERVED_MEMORY_IN_BYTES\x10\x01\x12\'\n#COMPRESSED_RESERVED_MEMORY_IN_BYTES\x10\x02\x12 \n\x1cTRANSFER_BATCH_SIZE_IN_BYTES\x10\x03\x12\x1c\n\x18TRANSFER_TIME_IN_SECONDS\x10\x04\x12\x1f\n\x1bRETENTION_PERIOD_IN_SECONDS\x10\x05\x42\x0c\n\n_new_valueb\x06proto3') +DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\x0eprotocol.proto\x12\x19modelardb.flight.protocol\"\xb1\x03\n\x0fManagerMetadata\x12\x0b\n\x03key\x18\x01 \x01(\t\x12V\n\x10s3_configuration\x18\x02 \x01(\x0b\x32:.modelardb.flight.protocol.ManagerMetadata.S3ConfigurationH\x00\x12\\\n\x13\x61zure_configuration\x18\x03 \x01(\x0b\x32=.modelardb.flight.protocol.ManagerMetadata.AzureConfigurationH\x00\x1aj\n\x0fS3Configuration\x12\x10\n\x08\x65ndpoint\x18\x01 \x01(\t\x12\x13\n\x0b\x62ucket_name\x18\x02 \x01(\t\x12\x15\n\raccess_key_id\x18\x03 \x01(\t\x12\x19\n\x11secret_access_key\x18\x04 \x01(\t\x1aV\n\x12\x41zureConfiguration\x12\x14\n\x0c\x61\x63\x63ount_name\x18\x01 \x01(\t\x12\x12\n\naccess_key\x18\x02 \x01(\t\x12\x16\n\x0e\x63ontainer_name\x18\x03 \x01(\tB\x17\n\x15storage_configuration\"\x87\x01\n\x0cNodeMetadata\x12\x0b\n\x03url\x18\x01 \x01(\t\x12G\n\x0bserver_mode\x18\x02 \x01(\x0e\x32\x32.modelardb.flight.protocol.NodeMetadata.ServerMode\"!\n\nServerMode\x12\t\n\x05\x43LOUD\x10\x00\x12\x08\n\x04\x45\x44GE\x10\x01\"\xfe\x04\n\rTableMetadata\x12T\n\x0cnormal_table\x18\x01 \x01(\x0b\x32<.modelardb.flight.protocol.TableMetadata.NormalTableMetadataH\x00\x12]\n\x11time_series_table\x18\x02 \x01(\x0b\x32@.modelardb.flight.protocol.TableMetadata.TimeSeriesTableMetadataH\x00\x1a\x33\n\x13NormalTableMetadata\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x0e\n\x06schema\x18\x02 \x01(\x0c\x1a\xf0\x02\n\x17TimeSeriesTableMetadata\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x0e\n\x06schema\x18\x02 \x01(\x0c\x12\x61\n\x0c\x65rror_bounds\x18\x03 \x03(\x0b\x32K.modelardb.flight.protocol.TableMetadata.TimeSeriesTableMetadata.ErrorBound\x12$\n\x1cgenerated_column_expressions\x18\x04 \x03(\x0c\x1a\xad\x01\n\nErrorBound\x12^\n\x04type\x18\x01 \x01(\x0e\x32P.modelardb.flight.protocol.TableMetadata.TimeSeriesTableMetadata.ErrorBound.Type\x12\r\n\x05value\x18\x02 \x01(\x02\"0\n\x04Type\x12\x0c\n\x08\x41\x42SOLUTE\x10\x00\x12\x0c\n\x08RELATIVE\x10\x01\x12\x0c\n\x08LOSSLESS\x10\x02\x42\x10\n\x0etable_metadata\"\xfa\x02\n\rConfiguration\x12-\n%multivariate_reserved_memory_in_bytes\x18\x01 \x01(\x04\x12-\n%uncompressed_reserved_memory_in_bytes\x18\x02 \x01(\x04\x12+\n#compressed_reserved_memory_in_bytes\x18\x03 \x01(\x04\x12)\n\x1ctransfer_batch_size_in_bytes\x18\x04 \x01(\x04H\x00\x88\x01\x01\x12%\n\x18transfer_time_in_seconds\x18\x05 \x01(\x04H\x01\x88\x01\x01\x12\x19\n\x11ingestion_threads\x18\x06 \x01(\r\x12\x1b\n\x13\x63ompression_threads\x18\x07 \x01(\r\x12\x16\n\x0ewriter_threads\x18\x08 \x01(\rB\x1f\n\x1d_transfer_batch_size_in_bytesB\x1b\n\x19_transfer_time_in_seconds\"\xcf\x02\n\x13UpdateConfiguration\x12G\n\x07setting\x18\x01 \x01(\x0e\x32\x36.modelardb.flight.protocol.UpdateConfiguration.Setting\x12\x16\n\tnew_value\x18\x02 \x01(\x04H\x00\x88\x01\x01\"\xc8\x01\n\x07Setting\x12)\n%MULTIVARIATE_RESERVED_MEMORY_IN_BYTES\x10\x00\x12)\n%UNCOMPRESSED_RESERVED_MEMORY_IN_BYTES\x10\x01\x12\'\n#COMPRESSED_RESERVED_MEMORY_IN_BYTES\x10\x02\x12 \n\x1cTRANSFER_BATCH_SIZE_IN_BYTES\x10\x03\x12\x1c\n\x18TRANSFER_TIME_IN_SECONDS\x10\x04\x42\x0c\n\n_new_valueb\x06proto3') _globals = globals() _builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, _globals) @@ -42,19 +42,19 @@ _globals['_NODEMETADATA_SERVERMODE']._serialized_start=584 _globals['_NODEMETADATA_SERVERMODE']._serialized_end=617 _globals['_TABLEMETADATA']._serialized_start=620 - _globals['_TABLEMETADATA']._serialized_end=1244 + _globals['_TABLEMETADATA']._serialized_end=1258 _globals['_TABLEMETADATA_NORMALTABLEMETADATA']._serialized_start=818 _globals['_TABLEMETADATA_NORMALTABLEMETADATA']._serialized_end=869 _globals['_TABLEMETADATA_TIMESERIESTABLEMETADATA']._serialized_start=872 - _globals['_TABLEMETADATA_TIMESERIESTABLEMETADATA']._serialized_end=1226 + _globals['_TABLEMETADATA_TIMESERIESTABLEMETADATA']._serialized_end=1240 _globals['_TABLEMETADATA_TIMESERIESTABLEMETADATA_ERRORBOUND']._serialized_start=1067 - _globals['_TABLEMETADATA_TIMESERIESTABLEMETADATA_ERRORBOUND']._serialized_end=1226 + _globals['_TABLEMETADATA_TIMESERIESTABLEMETADATA_ERRORBOUND']._serialized_end=1240 _globals['_TABLEMETADATA_TIMESERIESTABLEMETADATA_ERRORBOUND_TYPE']._serialized_start=1192 - _globals['_TABLEMETADATA_TIMESERIESTABLEMETADATA_ERRORBOUND_TYPE']._serialized_end=1226 - _globals['_CONFIGURATION']._serialized_start=1247 - _globals['_CONFIGURATION']._serialized_end=1662 - _globals['_UPDATECONFIGURATION']._serialized_start=1665 - _globals['_UPDATECONFIGURATION']._serialized_end=2033 - _globals['_UPDATECONFIGURATION_SETTING']._serialized_start=1786 - _globals['_UPDATECONFIGURATION_SETTING']._serialized_end=2019 + _globals['_TABLEMETADATA_TIMESERIESTABLEMETADATA_ERRORBOUND_TYPE']._serialized_end=1240 + _globals['_CONFIGURATION']._serialized_start=1261 + _globals['_CONFIGURATION']._serialized_end=1639 + _globals['_UPDATECONFIGURATION']._serialized_start=1642 + _globals['_UPDATECONFIGURATION']._serialized_end=1977 + _globals['_UPDATECONFIGURATION_SETTING']._serialized_start=1763 + _globals['_UPDATECONFIGURATION_SETTING']._serialized_end=1963 # @@protoc_insertion_point(module_scope) diff --git a/Apache-Arrow-Flight-Tester/protobuf/protocol_pb2.pyi b/Apache-Arrow-Flight-Tester/protobuf/protocol_pb2.pyi index 6d756f5..e5b4de4 100644 --- a/Apache-Arrow-Flight-Tester/protobuf/protocol_pb2.pyi +++ b/Apache-Arrow-Flight-Tester/protobuf/protocol_pb2.pyi @@ -68,8 +68,10 @@ class TableMetadata(_message.Message): __slots__ = () ABSOLUTE: _ClassVar[TableMetadata.TimeSeriesTableMetadata.ErrorBound.Type] RELATIVE: _ClassVar[TableMetadata.TimeSeriesTableMetadata.ErrorBound.Type] + LOSSLESS: _ClassVar[TableMetadata.TimeSeriesTableMetadata.ErrorBound.Type] ABSOLUTE: TableMetadata.TimeSeriesTableMetadata.ErrorBound.Type RELATIVE: TableMetadata.TimeSeriesTableMetadata.ErrorBound.Type + LOSSLESS: TableMetadata.TimeSeriesTableMetadata.ErrorBound.Type TYPE_FIELD_NUMBER: _ClassVar[int] VALUE_FIELD_NUMBER: _ClassVar[int] type: TableMetadata.TimeSeriesTableMetadata.ErrorBound.Type @@ -91,13 +93,12 @@ class TableMetadata(_message.Message): def __init__(self, normal_table: _Optional[_Union[TableMetadata.NormalTableMetadata, _Mapping]] = ..., time_series_table: _Optional[_Union[TableMetadata.TimeSeriesTableMetadata, _Mapping]] = ...) -> None: ... class Configuration(_message.Message): - __slots__ = ("multivariate_reserved_memory_in_bytes", "uncompressed_reserved_memory_in_bytes", "compressed_reserved_memory_in_bytes", "transfer_batch_size_in_bytes", "transfer_time_in_seconds", "retention_period_in_seconds", "ingestion_threads", "compression_threads", "writer_threads") + __slots__ = ("multivariate_reserved_memory_in_bytes", "uncompressed_reserved_memory_in_bytes", "compressed_reserved_memory_in_bytes", "transfer_batch_size_in_bytes", "transfer_time_in_seconds", "ingestion_threads", "compression_threads", "writer_threads") MULTIVARIATE_RESERVED_MEMORY_IN_BYTES_FIELD_NUMBER: _ClassVar[int] UNCOMPRESSED_RESERVED_MEMORY_IN_BYTES_FIELD_NUMBER: _ClassVar[int] COMPRESSED_RESERVED_MEMORY_IN_BYTES_FIELD_NUMBER: _ClassVar[int] TRANSFER_BATCH_SIZE_IN_BYTES_FIELD_NUMBER: _ClassVar[int] TRANSFER_TIME_IN_SECONDS_FIELD_NUMBER: _ClassVar[int] - RETENTION_PERIOD_IN_SECONDS_FIELD_NUMBER: _ClassVar[int] INGESTION_THREADS_FIELD_NUMBER: _ClassVar[int] COMPRESSION_THREADS_FIELD_NUMBER: _ClassVar[int] WRITER_THREADS_FIELD_NUMBER: _ClassVar[int] @@ -106,11 +107,10 @@ class Configuration(_message.Message): compressed_reserved_memory_in_bytes: int transfer_batch_size_in_bytes: int transfer_time_in_seconds: int - retention_period_in_seconds: int ingestion_threads: int compression_threads: int writer_threads: int - def __init__(self, multivariate_reserved_memory_in_bytes: _Optional[int] = ..., uncompressed_reserved_memory_in_bytes: _Optional[int] = ..., compressed_reserved_memory_in_bytes: _Optional[int] = ..., transfer_batch_size_in_bytes: _Optional[int] = ..., transfer_time_in_seconds: _Optional[int] = ..., retention_period_in_seconds: _Optional[int] = ..., ingestion_threads: _Optional[int] = ..., compression_threads: _Optional[int] = ..., writer_threads: _Optional[int] = ...) -> None: ... + def __init__(self, multivariate_reserved_memory_in_bytes: _Optional[int] = ..., uncompressed_reserved_memory_in_bytes: _Optional[int] = ..., compressed_reserved_memory_in_bytes: _Optional[int] = ..., transfer_batch_size_in_bytes: _Optional[int] = ..., transfer_time_in_seconds: _Optional[int] = ..., ingestion_threads: _Optional[int] = ..., compression_threads: _Optional[int] = ..., writer_threads: _Optional[int] = ...) -> None: ... class UpdateConfiguration(_message.Message): __slots__ = ("setting", "new_value") @@ -121,13 +121,11 @@ class UpdateConfiguration(_message.Message): COMPRESSED_RESERVED_MEMORY_IN_BYTES: _ClassVar[UpdateConfiguration.Setting] TRANSFER_BATCH_SIZE_IN_BYTES: _ClassVar[UpdateConfiguration.Setting] TRANSFER_TIME_IN_SECONDS: _ClassVar[UpdateConfiguration.Setting] - RETENTION_PERIOD_IN_SECONDS: _ClassVar[UpdateConfiguration.Setting] MULTIVARIATE_RESERVED_MEMORY_IN_BYTES: UpdateConfiguration.Setting UNCOMPRESSED_RESERVED_MEMORY_IN_BYTES: UpdateConfiguration.Setting COMPRESSED_RESERVED_MEMORY_IN_BYTES: UpdateConfiguration.Setting TRANSFER_BATCH_SIZE_IN_BYTES: UpdateConfiguration.Setting TRANSFER_TIME_IN_SECONDS: UpdateConfiguration.Setting - RETENTION_PERIOD_IN_SECONDS: UpdateConfiguration.Setting SETTING_FIELD_NUMBER: _ClassVar[int] NEW_VALUE_FIELD_NUMBER: _ClassVar[int] setting: UpdateConfiguration.Setting