ci: unbreak docker-up (backport #3273 RustFS swap and #3174 HMS jar ADD)

Makefile

@@ -107,7 +107,7 @@ site: install-mdbook
 
 # Docker targets for integration tests
 docker-up:
-	docker compose -f dev/docker-compose.yaml up -d --build --wait
+	docker compose -f dev/docker-compose.yaml up -d --build --wait --remove-orphans
 
 docker-down:
 	docker compose -f dev/docker-compose.yaml down -v --remove-orphans --timeout 0

crates/catalog/hms/tests/hms_catalog_test.rs

@@ -33,7 +33,7 @@ use iceberg_catalog_hms::{
     HmsCatalog, HmsCatalogBuilder, THRIFT_TRANSPORT_BUFFERED,
 };
 use iceberg_storage_opendal::OpenDalStorageFactory;
-use iceberg_test_utils::{get_hms_endpoint, get_minio_endpoint, set_up};
+use iceberg_test_utils::{get_hms_endpoint, get_object_store_endpoint, set_up};
 use tokio::time::sleep;
 use tracing::info;
 
@@ -43,7 +43,7 @@ async fn get_catalog() -> HmsCatalog {
     set_up();
 
     let hms_endpoint = get_hms_endpoint();
-    let minio_endpoint = get_minio_endpoint();
+    let object_store_endpoint = get_object_store_endpoint();
 
     let props = HashMap::from([
         (HMS_CATALOG_PROP_URI.to_string(), hms_endpoint),
@@ -55,7 +55,7 @@ async fn get_catalog() -> HmsCatalog {
             HMS_CATALOG_PROP_WAREHOUSE.to_string(),
             "s3a://warehouse/hive".to_string(),
         ),
-        (S3_ENDPOINT.to_string(), minio_endpoint),
+        (S3_ENDPOINT.to_string(), object_store_endpoint),
         (S3_ACCESS_KEY_ID.to_string(), "admin".to_string()),
         (S3_SECRET_ACCESS_KEY.to_string(), "password".to_string()),
         (S3_REGION.to_string(), "us-east-1".to_string()),

crates/catalog/loader/tests/common/mod.rs

@@ -49,7 +49,8 @@ use iceberg_catalog_sql::{
 };
 use iceberg_storage_opendal::OpenDalStorageFactory;
 use iceberg_test_utils::{
-    get_glue_endpoint, get_hms_endpoint, get_minio_endpoint, get_rest_catalog_endpoint, set_up,
+    get_glue_endpoint, get_hms_endpoint, get_object_store_endpoint, get_rest_catalog_endpoint,
+    set_up,
 };
 use sqlx::migrate::MigrateDatabase;
 use tempfile::TempDir;
@@ -216,7 +217,7 @@ async fn rest_catalog() -> RestCatalog {
 
 async fn glue_catalog() -> GlueCatalog {
     let glue_endpoint = get_glue_endpoint();
-    let minio_endpoint = get_minio_endpoint();
+    let object_store_endpoint = get_object_store_endpoint();
 
     let props = HashMap::from([
         (AWS_ACCESS_KEY_ID.to_string(), "my_access_id".to_string()),
@@ -225,7 +226,7 @@ async fn glue_catalog() -> GlueCatalog {
             "my_secret_key".to_string(),
         ),
         (AWS_REGION_NAME.to_string(), "us-east-1".to_string()),
-        (S3_ENDPOINT.to_string(), minio_endpoint),
+        (S3_ENDPOINT.to_string(), object_store_endpoint),
         (S3_ACCESS_KEY_ID.to_string(), "admin".to_string()),
         (S3_SECRET_ACCESS_KEY.to_string(), "password".to_string()),
         (S3_REGION.to_string(), "us-east-1".to_string()),
@@ -264,7 +265,7 @@ async fn glue_catalog() -> GlueCatalog {
 
 async fn hms_catalog() -> HmsCatalog {
     let hms_endpoint = get_hms_endpoint();
-    let minio_endpoint = get_minio_endpoint();
+    let object_store_endpoint = get_object_store_endpoint();
 
     let props = HashMap::from([
         (HMS_CATALOG_PROP_URI.to_string(), hms_endpoint),
@@ -276,7 +277,7 @@ async fn hms_catalog() -> HmsCatalog {
             HMS_CATALOG_PROP_WAREHOUSE.to_string(),
             "s3a://warehouse/hive".to_string(),
         ),
-        (S3_ENDPOINT.to_string(), minio_endpoint),
+        (S3_ENDPOINT.to_string(), object_store_endpoint),
         (S3_ACCESS_KEY_ID.to_string(), "admin".to_string()),
         (S3_SECRET_ACCESS_KEY.to_string(), "password".to_string()),
         (S3_REGION.to_string(), "us-east-1".to_string()),

crates/integration_tests/src/lib.rs

@@ -22,7 +22,7 @@ use iceberg::io::{
     S3_ACCESS_KEY_ID, S3_ENDPOINT, S3_PATH_STYLE_ACCESS, S3_REGION, S3_SECRET_ACCESS_KEY,
 };
 use iceberg_catalog_rest::REST_CATALOG_PROP_URI;
-use iceberg_test_utils::{get_minio_endpoint, get_rest_catalog_endpoint, set_up};
+use iceberg_test_utils::{get_object_store_endpoint, get_rest_catalog_endpoint, set_up};
 
 /// Global test fixture that uses environment-based configuration.
 /// This assumes Docker containers are started externally (e.g., via `make docker-up`).
@@ -39,11 +39,11 @@ impl GlobalTestFixture {
         set_up();
 
         let rest_endpoint = get_rest_catalog_endpoint();
-        let minio_endpoint = get_minio_endpoint();
+        let object_store_endpoint = get_object_store_endpoint();
 
         let catalog_config = HashMap::from([
             (REST_CATALOG_PROP_URI.to_string(), rest_endpoint),
-            (S3_ENDPOINT.to_string(), minio_endpoint),
+            (S3_ENDPOINT.to_string(), object_store_endpoint),
             (S3_ACCESS_KEY_ID.to_string(), "admin".to_string()),
             (S3_SECRET_ACCESS_KEY.to_string(), "password".to_string()),
             (S3_REGION.to_string(), "us-east-1".to_string()),

crates/storage/opendal/tests/file_io_s3_test.rs

@@ -31,19 +31,19 @@ mod tests {
     use iceberg_storage_opendal::{
         AwsCredential, CustomAwsCredentialLoader, OpenDalStorageFactory, ProvideCredential,
     };
-    use iceberg_test_utils::{get_minio_endpoint, normalize_test_name_with_parts, set_up};
+    use iceberg_test_utils::{get_object_store_endpoint, normalize_test_name_with_parts, set_up};
     use reqsign_core::Context;
 
     async fn get_file_io() -> FileIO {
         set_up();
 
-        let minio_endpoint = get_minio_endpoint();
+        let object_store_endpoint = get_object_store_endpoint();
 
         FileIOBuilder::new(Arc::new(OpenDalStorageFactory::S3 {
             customized_credential_load: None,
         }))
         .with_props(vec![
-            (S3_ENDPOINT, minio_endpoint),
+            (S3_ENDPOINT, object_store_endpoint),
             (S3_ACCESS_KEY_ID, "admin".to_string()),
             (S3_SECRET_ACCESS_KEY, "password".to_string()),
             (S3_REGION, "us-east-1".to_string()),
@@ -109,7 +109,7 @@ mod tests {
             Self { credential }
         }
 
-        fn new_minio() -> Self {
+        fn new_object_store() -> Self {
             Self::new(Some(AwsCredential {
                 access_key_id: "admin".to_string(),
                 secret_access_key: "password".to_string(),
@@ -133,7 +133,7 @@ mod tests {
     #[test]
     fn test_custom_aws_credential_loader_instantiation() {
         // Test creating CustomAwsCredentialLoader with mock loader
-        let mock_loader = MockCredentialLoader::new_minio();
+        let mock_loader = MockCredentialLoader::new_object_store();
         let custom_loader = CustomAwsCredentialLoader::new(mock_loader);
 
         // Test that the loader can be used in FileIOBuilder with OpenDalStorageFactory
@@ -153,17 +153,17 @@ mod tests {
         let _file_io = get_file_io().await;
 
         // Create a mock credential loader
-        let mock_loader = MockCredentialLoader::new_minio();
+        let mock_loader = MockCredentialLoader::new_object_store();
         let custom_loader = CustomAwsCredentialLoader::new(mock_loader);
 
-        let minio_endpoint = get_minio_endpoint();
+        let object_store_endpoint = get_object_store_endpoint();
 
         // Build FileIO with custom credential loader via OpenDalStorageFactory
         let file_io_with_custom_creds = FileIOBuilder::new(Arc::new(OpenDalStorageFactory::S3 {
             customized_credential_load: Some(custom_loader),
         }))
         .with_props(vec![
-            (S3_ENDPOINT, minio_endpoint),
+            (S3_ENDPOINT, object_store_endpoint),
             (S3_REGION, "us-east-1".to_string()),
             (S3_PATH_STYLE_ACCESS, "true".to_string()),
         ])
@@ -184,14 +184,14 @@ mod tests {
         let mock_loader = MockCredentialLoader::new(None);
         let custom_loader = CustomAwsCredentialLoader::new(mock_loader);
 
-        let minio_endpoint = get_minio_endpoint();
+        let object_store_endpoint = get_object_store_endpoint();
 
         // Build FileIO with custom credential loader via OpenDalStorageFactory
         let file_io_with_custom_creds = FileIOBuilder::new(Arc::new(OpenDalStorageFactory::S3 {
             customized_credential_load: Some(custom_loader),
         }))
         .with_props(vec![
-            (S3_ENDPOINT, minio_endpoint),
+            (S3_ENDPOINT, object_store_endpoint),
             (S3_REGION, "us-east-1".to_string()),
             (S3_PATH_STYLE_ACCESS, "true".to_string()),
         ])

crates/storage/opendal/tests/resolving_storage_test.rs

@@ -33,16 +33,16 @@ mod tests {
         S3_SECRET_ACCESS_KEY,
     };
     use iceberg_storage_opendal::OpenDalResolvingStorageFactory;
-    use iceberg_test_utils::{get_minio_endpoint, normalize_test_name_with_parts, set_up};
+    use iceberg_test_utils::{get_object_store_endpoint, normalize_test_name_with_parts, set_up};
 
     fn get_resolving_file_io() -> iceberg::io::FileIO {
         set_up();
 
-        let minio_endpoint = get_minio_endpoint();
+        let object_store_endpoint = get_object_store_endpoint();
 
         FileIOBuilder::new(Arc::new(OpenDalResolvingStorageFactory::new()))
             .with_props(vec![
-                (S3_ENDPOINT, minio_endpoint),
+                (S3_ENDPOINT, object_store_endpoint),
                 (S3_ACCESS_KEY_ID, "admin".to_string()),
                 (S3_SECRET_ACCESS_KEY, "password".to_string()),
                 (S3_REGION, "us-east-1".to_string()),
@@ -263,9 +263,9 @@ mod tests {
         use reqsign_core::Context;
 
         #[derive(Debug)]
-        struct MinioCredentialLoader;
+        struct ObjectStoreCredentialLoader;
 
-        impl ProvideCredential for MinioCredentialLoader {
+        impl ProvideCredential for ObjectStoreCredentialLoader {
             type Credential = AwsCredential;
 
             async fn provide_credential(
@@ -282,14 +282,14 @@ mod tests {
         }
 
         set_up();
-        let minio_endpoint = get_minio_endpoint();
+        let object_store_endpoint = get_object_store_endpoint();
 
         let factory = OpenDalResolvingStorageFactory::new()
-            .with_s3_credential_loader(CustomAwsCredentialLoader::new(MinioCredentialLoader));
+            .with_s3_credential_loader(CustomAwsCredentialLoader::new(ObjectStoreCredentialLoader));
 
         let file_io = FileIOBuilder::new(Arc::new(factory))
             .with_props(vec![
-                (S3_ENDPOINT, minio_endpoint),
+                (S3_ENDPOINT, object_store_endpoint),
                 (S3_REGION, "us-east-1".to_string()),
                 (S3_PATH_STYLE_ACCESS, "true".to_string()),
             ])

crates/test_utils/src/lib.rs

@@ -37,24 +37,24 @@ mod common {
     }
 
     // Environment variable names for service endpoints
-    pub const ENV_MINIO_ENDPOINT: &str = "ICEBERG_TEST_MINIO_ENDPOINT";
+    pub const ENV_OBJECT_STORE_ENDPOINT: &str = "ICEBERG_TEST_OBJECT_STORE_ENDPOINT";
     pub const ENV_REST_CATALOG_ENDPOINT: &str = "ICEBERG_TEST_REST_ENDPOINT";
     pub const ENV_HMS_ENDPOINT: &str = "ICEBERG_TEST_HMS_ENDPOINT";
     pub const ENV_GLUE_ENDPOINT: &str = "ICEBERG_TEST_GLUE_ENDPOINT";
     pub const ENV_GCS_ENDPOINT: &str = "ICEBERG_TEST_GCS_ENDPOINT";
 
     // Default ports matching dev/docker-compose.yaml
-    pub const DEFAULT_MINIO_PORT: u16 = 9000;
+    pub const DEFAULT_OBJECT_STORE_PORT: u16 = 9000;
     pub const DEFAULT_REST_CATALOG_PORT: u16 = 8181;
     pub const DEFAULT_HMS_PORT: u16 = 9083;
     pub const DEFAULT_GLUE_PORT: u16 = 5001;
     pub const DEFAULT_GCS_PORT: u16 = 4443;
 
-    /// Returns the MinIO S3-compatible endpoint.
-    /// Checks ICEBERG_TEST_MINIO_ENDPOINT env var, otherwise returns localhost default.
-    pub fn get_minio_endpoint() -> String {
-        std::env::var(ENV_MINIO_ENDPOINT)
-            .unwrap_or_else(|_| format!("http://localhost:{DEFAULT_MINIO_PORT}"))
+    /// Returns the S3-compatible object store endpoint.
+    /// Checks ICEBERG_TEST_OBJECT_STORE_ENDPOINT env var, otherwise returns localhost default.
+    pub fn get_object_store_endpoint() -> String {
+        std::env::var(ENV_OBJECT_STORE_ENDPOINT)
+            .unwrap_or_else(|_| format!("http://localhost:{DEFAULT_OBJECT_STORE_PORT}"))
     }
 
     /// Returns the REST catalog endpoint.

dev/docker-compose.yaml

@@ -25,53 +25,43 @@ networks:
 
 services:
   # =============================================================================
-  # MinIO - S3-compatible storage (shared by all tests)
+  # Object store - S3-compatible storage backed by RustFS (shared by all tests)
   # =============================================================================
-  minio:
-    image: minio/minio:RELEASE.2025-05-24T17-08-30Z
+  object-store:
+    image: rustfs/rustfs:1.0.0
     environment:
-      - MINIO_ROOT_USER=admin
-      - MINIO_ROOT_PASSWORD=password
-      - MINIO_DOMAIN=minio
-    hostname: minio
+      - RUSTFS_ACCESS_KEY=admin
+      - RUSTFS_SECRET_KEY=password
+      - RUSTFS_SERVER_DOMAINS=object-store
+    hostname: object-store
     networks:
       iceberg_test:
         # Add aliases for virtual-hosted style bucket access
         aliases:
-          - icebergdata.minio
-          - warehouse.minio
-          - bucket1.minio
+          - icebergdata.object-store
+          - warehouse.object-store
+          - bucket1.object-store
     ports:
       - "9000:9000"
       - "9001:9001"
-    command: ["server", "/data", "--console-address", ":9001"]
     healthcheck:
-      test: ["CMD", "mc", "ready", "local"]
+      test: ["CMD", "curl", "-f", "http://localhost:9000/health/ready"]
       interval: 5s
       timeout: 5s
       retries: 5
 
-  # MinIO client - creates buckets for tests
-  mc:
+  # One-shot job that creates buckets for tests. Re-creating an existing bucket
+  # returns 200, so this is safe to re-run.
+  create-buckets:
     depends_on:
-      minio:
+      object-store:
         condition: service_healthy
-    image: minio/mc:RELEASE.2025-05-21T01-59-54Z
-    environment:
-      - AWS_ACCESS_KEY_ID=admin
-      - AWS_SECRET_ACCESS_KEY=password
-      - AWS_REGION=us-east-1
+    image: rustfs/rustfs:1.0.0
     entrypoint: >
       /bin/sh -c "
-        /usr/bin/mc alias set minio http://minio:9000 admin password;
-        /usr/bin/mc mb --ignore-existing minio/icebergdata;
-        /usr/bin/mc mb --ignore-existing minio/warehouse;
-        /usr/bin/mc mb --ignore-existing minio/bucket1;
-        /usr/bin/mc policy set public minio/icebergdata;
-        /usr/bin/mc policy set public minio/warehouse;
-        /usr/bin/mc policy set public minio/bucket1;
-        echo 'Buckets created successfully';
-        tail -f /dev/null
+        curl -fsS --aws-sigv4 aws:amz:us-east-1:s3 -u admin:password -X PUT http://object-store:9000/icebergdata &&
+        curl -fsS --aws-sigv4 aws:amz:us-east-1:s3 -u admin:password -X PUT http://object-store:9000/warehouse &&
+        curl -fsS --aws-sigv4 aws:amz:us-east-1:s3 -u admin:password -X PUT http://object-store:9000/bucket1
       "
     networks:
       iceberg_test:
@@ -89,9 +79,9 @@ services:
       - CATALOG_URI=jdbc:sqlite:file:/tmp/iceberg_rest_mode=memory
       - CATALOG_WAREHOUSE=s3://icebergdata/demo
       - CATALOG_IO__IMPL=org.apache.iceberg.aws.s3.S3FileIO
-      - CATALOG_S3_ENDPOINT=http://minio:9000
+      - CATALOG_S3_ENDPOINT=http://object-store:9000
     depends_on:
-      minio:
+      object-store:
         condition: service_healthy
     networks:
       iceberg_test:
@@ -113,7 +103,7 @@ services:
       dockerfile: Dockerfile
     platform: ${DOCKER_DEFAULT_PLATFORM:-linux/amd64}
     depends_on:
-      minio:
+      object-store:
         condition: service_healthy
     networks:
       iceberg_test:
@@ -170,8 +160,8 @@ services:
     depends_on:
       rest:
         condition: service_healthy
-      minio:
-        condition: service_healthy
+      create-buckets:
+        condition: service_completed_successfully
     environment:
       - AWS_ACCESS_KEY_ID=admin
       - AWS_SECRET_ACCESS_KEY=password

dev/hms/Dockerfile

@@ -20,10 +20,12 @@ ENV HADOOP_VERSION=3.1.0
 
 USER root
 
-RUN apt-get update -qq && apt-get -qq -y install curl && \
-    curl https://repo1.maven.org/maven2/org/apache/hadoop/hadoop-aws/${HADOOP_VERSION}/hadoop-aws-${HADOOP_VERSION}.jar -Lo /opt/hive/lib/hadoop-aws-${HADOOP_VERSION}.jar && \
-    curl https://repo1.maven.org/maven2/com/amazonaws/aws-java-sdk-bundle/1.11.271/aws-java-sdk-bundle-1.11.271.jar -Lo /opt/hive/lib/aws-java-sdk-bundle-1.11.271.jar && \
-    apt-get clean && rm -rf /var/lib/apt/lists/*
+ADD --chmod=644 --checksum=sha256:a18508b9348af095ea41301e439354dbd449e304ac44c6885b2b4fe78de88126 \
+    https://repo1.maven.org/maven2/org/apache/hadoop/hadoop-aws/${HADOOP_VERSION}/hadoop-aws-${HADOOP_VERSION}.jar \
+    /opt/hive/lib/hadoop-aws-${HADOOP_VERSION}.jar
+ADD --chmod=644 --checksum=sha256:faf78ac4880f56cf52791d84ec1068ce7c66acc4295d580a726104b734c01fcd \
+    https://repo1.maven.org/maven2/com/amazonaws/aws-java-sdk-bundle/1.11.271/aws-java-sdk-bundle-1.11.271.jar \
+    /opt/hive/lib/aws-java-sdk-bundle-1.11.271.jar
 
 COPY core-site.xml /opt/hadoop/etc/hadoop/core-site.xml
 

dev/hms/core-site.xml

@@ -30,7 +30,7 @@
     </property>
     <property>
       <name>fs.s3a.endpoint</name>
-      <value>http://minio:9000</value>
+      <value>http://object-store:9000</value>
     </property>
     <property>
       <name>fs.s3a.access.key</name>