Add support for testing vacuum in Apache Arrow Flight Tester

Apache-Arrow-Flight-Tester/common.py

@@ -117,6 +117,10 @@ def clean_up_tables(self, tables: list[str], operation: Literal["drop", "truncat
                 else self.truncate_table(table_name)
             )
 
+    def vacuum(self, table_names: list[str]) -> None:
+        """Vacuum the given tables in the server or manager."""
+        self.do_get(Ticket(f"VACUUM {', '.join(table_names)}"))
+
     def node_type(self) -> str:
         """Return the type of the node."""
         node_type = self.do_action("NodeType", b"")

Apache-Arrow-Flight-Tester/protobuf/protocol.proto

@@ -19,7 +19,7 @@ package modelardb.flight.protocol;
 
 // Metadata for the ModelarDB cluster manager, including its unique key and storage configuration.
 message ManagerMetadata {
-  // key used to uniquely identify the cluster manager.
+  // Key used to uniquely identify the cluster manager.
   string key = 1;
 
   // Storage configuration used to connect to an S3 object store.
@@ -107,14 +107,17 @@ message Configuration {
   // The number of seconds between each transfer of data to the remote object store.
   optional uint64 transfer_time_in_seconds = 5;
 
+  // The number of seconds to retain deleted data in storage before it can be removed by vacuum.
+  uint64 retention_period_in_seconds = 6;
+
   // Number of threads to allocate for converting multivariate time series to univariate time series.
-  uint32 ingestion_threads = 6;
+  uint32 ingestion_threads = 7;
 
   // Number of threads to allocate for compressing univariate time series to segments.
-  uint32 compression_threads = 7;
+  uint32 compression_threads = 8;
 
   // Number of threads to allocate for writing segments to a local and/or remote data folder.
-  uint32 writer_threads = 8;
+  uint32 writer_threads = 9;
 }
 
 // Request to update the configuration of a ModelarDB node.
@@ -125,6 +128,7 @@ message UpdateConfiguration {
     COMPRESSED_RESERVED_MEMORY_IN_BYTES = 2;
     TRANSFER_BATCH_SIZE_IN_BYTES = 3;
     TRANSFER_TIME_IN_SECONDS = 4;
+    RETENTION_PERIOD_IN_SECONDS = 5;
   }
 
   // Setting to update in the configuration.

Apache-Arrow-Flight-Tester/protobuf/protocol_pb2.pyi

@@ -91,12 +91,13 @@ class TableMetadata(_message.Message):
     def __init__(self, normal_tables: _Optional[_Iterable[_Union[TableMetadata.NormalTableMetadata, _Mapping]]] = ..., time_series_tables: _Optional[_Iterable[_Union[TableMetadata.TimeSeriesTableMetadata, _Mapping]]] = ...) -> None: ...
 
 class Configuration(_message.Message):
-    __slots__ = ("multivariate_reserved_memory_in_bytes", "uncompressed_reserved_memory_in_bytes", "compressed_reserved_memory_in_bytes", "transfer_batch_size_in_bytes", "transfer_time_in_seconds", "ingestion_threads", "compression_threads", "writer_threads")
+    __slots__ = ("multivariate_reserved_memory_in_bytes", "uncompressed_reserved_memory_in_bytes", "compressed_reserved_memory_in_bytes", "transfer_batch_size_in_bytes", "transfer_time_in_seconds", "retention_period_in_seconds", "ingestion_threads", "compression_threads", "writer_threads")
     MULTIVARIATE_RESERVED_MEMORY_IN_BYTES_FIELD_NUMBER: _ClassVar[int]
     UNCOMPRESSED_RESERVED_MEMORY_IN_BYTES_FIELD_NUMBER: _ClassVar[int]
     COMPRESSED_RESERVED_MEMORY_IN_BYTES_FIELD_NUMBER: _ClassVar[int]
     TRANSFER_BATCH_SIZE_IN_BYTES_FIELD_NUMBER: _ClassVar[int]
     TRANSFER_TIME_IN_SECONDS_FIELD_NUMBER: _ClassVar[int]
+    RETENTION_PERIOD_IN_SECONDS_FIELD_NUMBER: _ClassVar[int]
     INGESTION_THREADS_FIELD_NUMBER: _ClassVar[int]
     COMPRESSION_THREADS_FIELD_NUMBER: _ClassVar[int]
     WRITER_THREADS_FIELD_NUMBER: _ClassVar[int]
@@ -105,10 +106,11 @@ class Configuration(_message.Message):
     compressed_reserved_memory_in_bytes: int
     transfer_batch_size_in_bytes: int
     transfer_time_in_seconds: int
+    retention_period_in_seconds: int
     ingestion_threads: int
     compression_threads: int
     writer_threads: int
-    def __init__(self, multivariate_reserved_memory_in_bytes: _Optional[int] = ..., uncompressed_reserved_memory_in_bytes: _Optional[int] = ..., compressed_reserved_memory_in_bytes: _Optional[int] = ..., transfer_batch_size_in_bytes: _Optional[int] = ..., transfer_time_in_seconds: _Optional[int] = ..., ingestion_threads: _Optional[int] = ..., compression_threads: _Optional[int] = ..., writer_threads: _Optional[int] = ...) -> None: ...
+    def __init__(self, multivariate_reserved_memory_in_bytes: _Optional[int] = ..., uncompressed_reserved_memory_in_bytes: _Optional[int] = ..., compressed_reserved_memory_in_bytes: _Optional[int] = ..., transfer_batch_size_in_bytes: _Optional[int] = ..., transfer_time_in_seconds: _Optional[int] = ..., retention_period_in_seconds: _Optional[int] = ..., ingestion_threads: _Optional[int] = ..., compression_threads: _Optional[int] = ..., writer_threads: _Optional[int] = ...) -> None: ...
 
 class UpdateConfiguration(_message.Message):
     __slots__ = ("setting", "new_value")
@@ -119,11 +121,13 @@ class UpdateConfiguration(_message.Message):
         COMPRESSED_RESERVED_MEMORY_IN_BYTES: _ClassVar[UpdateConfiguration.Setting]
         TRANSFER_BATCH_SIZE_IN_BYTES: _ClassVar[UpdateConfiguration.Setting]
         TRANSFER_TIME_IN_SECONDS: _ClassVar[UpdateConfiguration.Setting]
+        RETENTION_PERIOD_IN_SECONDS: _ClassVar[UpdateConfiguration.Setting]
     MULTIVARIATE_RESERVED_MEMORY_IN_BYTES: UpdateConfiguration.Setting
     UNCOMPRESSED_RESERVED_MEMORY_IN_BYTES: UpdateConfiguration.Setting
     COMPRESSED_RESERVED_MEMORY_IN_BYTES: UpdateConfiguration.Setting
     TRANSFER_BATCH_SIZE_IN_BYTES: UpdateConfiguration.Setting
     TRANSFER_TIME_IN_SECONDS: UpdateConfiguration.Setting
+    RETENTION_PERIOD_IN_SECONDS: UpdateConfiguration.Setting
     SETTING_FIELD_NUMBER: _ClassVar[int]
     NEW_VALUE_FIELD_NUMBER: _ClassVar[int]
     setting: UpdateConfiguration.Setting