From 0d786e4a7766f8d868ceee235db433e4e3966513 Mon Sep 17 00:00:00 2001 From: ovoievodin Date: Thu, 24 Sep 2026 10:07:03 -0700 Subject: [PATCH 1/5] ci: replace MinIO with RustFS for integration tests `quay.io/minio/minio` no longer allows anonymous pulls, so `make docker-up` fails and the integration tests cannot start. DataFusion hit the same failure and switched to RustFS in apache/datafusion#25706. PyIceberg moved to RustFS earlier in apache/iceberg-python#3928. Run `rustfs/rustfs:1.0.0` as the shared S3 service instead. It keeps the `admin`/`password` credentials, the 9000/9001 ports, and virtual-hosted-style access through `RUSTFS_SERVER_DOMAINS` and the bucket aliases. The healthcheck uses `/health/ready`, which waits for storage and IAM. The `mc` bucket setup becomes `curl --aws-sigv4` run from the RustFS image, and the public bucket policy is dropped because no test reads anonymously. Point the REST fixture, HMS, and Spark at `http://rustfs:9000`, and rename the MinIO-specific test helpers and the `ICEBERG_TEST_MINIO_ENDPOINT` override to RustFS. Closes #3272 --- crates/catalog/hms/tests/hms_catalog_test.rs | 6 +-- crates/catalog/loader/tests/common/mod.rs | 10 ++-- crates/integration_tests/src/lib.rs | 6 +-- .../storage/opendal/tests/file_io_s3_test.rs | 20 +++---- .../opendal/tests/resolving_storage_test.rs | 16 +++--- crates/test_utils/src/lib.rs | 14 ++--- dev/docker-compose.yaml | 53 ++++++++----------- dev/hms/core-site.xml | 2 +- dev/spark/spark-defaults.conf | 2 +- website/src/reference/container-runtimes.md | 2 +- 10 files changed, 61 insertions(+), 70 deletions(-) diff --git a/crates/catalog/hms/tests/hms_catalog_test.rs b/crates/catalog/hms/tests/hms_catalog_test.rs index d0e6486ad8..bfdb62b54f 100644 --- a/crates/catalog/hms/tests/hms_catalog_test.rs +++ b/crates/catalog/hms/tests/hms_catalog_test.rs @@ -33,7 +33,7 @@ use iceberg_catalog_hms::{ HmsCatalog, HmsCatalogBuilder, THRIFT_TRANSPORT_BUFFERED, }; use iceberg_storage_opendal::OpenDalStorageFactory; -use iceberg_test_utils::{get_hms_endpoint, get_minio_endpoint, set_up}; +use iceberg_test_utils::{get_hms_endpoint, get_rustfs_endpoint, set_up}; use tokio::time::sleep; use tracing::info; @@ -43,7 +43,7 @@ async fn get_catalog() -> HmsCatalog { set_up(); let hms_endpoint = get_hms_endpoint(); - let minio_endpoint = get_minio_endpoint(); + let rustfs_endpoint = get_rustfs_endpoint(); let props = HashMap::from([ (HMS_CATALOG_PROP_URI.to_string(), hms_endpoint), @@ -55,7 +55,7 @@ async fn get_catalog() -> HmsCatalog { HMS_CATALOG_PROP_WAREHOUSE.to_string(), "s3a://warehouse/hive".to_string(), ), - (S3_ENDPOINT.to_string(), minio_endpoint), + (S3_ENDPOINT.to_string(), rustfs_endpoint), (S3_ACCESS_KEY_ID.to_string(), "admin".to_string()), (S3_SECRET_ACCESS_KEY.to_string(), "password".to_string()), (S3_REGION.to_string(), "us-east-1".to_string()), diff --git a/crates/catalog/loader/tests/common/mod.rs b/crates/catalog/loader/tests/common/mod.rs index 2b5033e6f2..39475f5c93 100644 --- a/crates/catalog/loader/tests/common/mod.rs +++ b/crates/catalog/loader/tests/common/mod.rs @@ -50,7 +50,7 @@ use iceberg_catalog_sql::{ }; use iceberg_storage_opendal::OpenDalStorageFactory; use iceberg_test_utils::{ - get_glue_endpoint, get_hms_endpoint, get_minio_endpoint, get_rest_catalog_endpoint, set_up, + get_glue_endpoint, get_hms_endpoint, get_rest_catalog_endpoint, get_rustfs_endpoint, set_up, }; use sqlx::migrate::MigrateDatabase; use tempfile::TempDir; @@ -230,7 +230,7 @@ async fn rest_catalog(kms_client_factory: Arc) -> RestCata async fn glue_catalog(kms_client_factory: Arc) -> GlueCatalog { let glue_endpoint = get_glue_endpoint(); - let minio_endpoint = get_minio_endpoint(); + let rustfs_endpoint = get_rustfs_endpoint(); let props = HashMap::from([ (AWS_ACCESS_KEY_ID.to_string(), "my_access_id".to_string()), @@ -239,7 +239,7 @@ async fn glue_catalog(kms_client_factory: Arc) -> GlueCata "my_secret_key".to_string(), ), (AWS_REGION_NAME.to_string(), "us-east-1".to_string()), - (S3_ENDPOINT.to_string(), minio_endpoint), + (S3_ENDPOINT.to_string(), rustfs_endpoint), (S3_ACCESS_KEY_ID.to_string(), "admin".to_string()), (S3_SECRET_ACCESS_KEY.to_string(), "password".to_string()), (S3_REGION.to_string(), "us-east-1".to_string()), @@ -279,7 +279,7 @@ async fn glue_catalog(kms_client_factory: Arc) -> GlueCata async fn hms_catalog(kms_client_factory: Arc) -> HmsCatalog { let hms_endpoint = get_hms_endpoint(); - let minio_endpoint = get_minio_endpoint(); + let rustfs_endpoint = get_rustfs_endpoint(); let props = HashMap::from([ (HMS_CATALOG_PROP_URI.to_string(), hms_endpoint), @@ -291,7 +291,7 @@ async fn hms_catalog(kms_client_factory: Arc) -> HmsCatalo HMS_CATALOG_PROP_WAREHOUSE.to_string(), "s3a://warehouse/hive".to_string(), ), - (S3_ENDPOINT.to_string(), minio_endpoint), + (S3_ENDPOINT.to_string(), rustfs_endpoint), (S3_ACCESS_KEY_ID.to_string(), "admin".to_string()), (S3_SECRET_ACCESS_KEY.to_string(), "password".to_string()), (S3_REGION.to_string(), "us-east-1".to_string()), diff --git a/crates/integration_tests/src/lib.rs b/crates/integration_tests/src/lib.rs index feafa3ae9f..fe8bb7d318 100644 --- a/crates/integration_tests/src/lib.rs +++ b/crates/integration_tests/src/lib.rs @@ -22,7 +22,7 @@ use iceberg::io::{ S3_ACCESS_KEY_ID, S3_ENDPOINT, S3_PATH_STYLE_ACCESS, S3_REGION, S3_SECRET_ACCESS_KEY, }; use iceberg_catalog_rest::REST_CATALOG_PROP_URI; -use iceberg_test_utils::{get_minio_endpoint, get_rest_catalog_endpoint, set_up}; +use iceberg_test_utils::{get_rest_catalog_endpoint, get_rustfs_endpoint, set_up}; /// Global test fixture that uses environment-based configuration. /// This assumes Docker containers are started externally (e.g., via `make docker-up`). @@ -39,11 +39,11 @@ impl GlobalTestFixture { set_up(); let rest_endpoint = get_rest_catalog_endpoint(); - let minio_endpoint = get_minio_endpoint(); + let rustfs_endpoint = get_rustfs_endpoint(); let catalog_config = HashMap::from([ (REST_CATALOG_PROP_URI.to_string(), rest_endpoint), - (S3_ENDPOINT.to_string(), minio_endpoint), + (S3_ENDPOINT.to_string(), rustfs_endpoint), (S3_ACCESS_KEY_ID.to_string(), "admin".to_string()), (S3_SECRET_ACCESS_KEY.to_string(), "password".to_string()), (S3_REGION.to_string(), "us-east-1".to_string()), diff --git a/crates/storage/opendal/tests/file_io_s3_test.rs b/crates/storage/opendal/tests/file_io_s3_test.rs index 6b57f86607..365842793d 100644 --- a/crates/storage/opendal/tests/file_io_s3_test.rs +++ b/crates/storage/opendal/tests/file_io_s3_test.rs @@ -32,19 +32,19 @@ mod tests { use iceberg_storage_opendal::{ AwsCredential, CustomAwsCredentialLoader, OpenDalStorageFactory, ProvideCredential, }; - use iceberg_test_utils::{get_minio_endpoint, normalize_test_name_with_parts, set_up}; + use iceberg_test_utils::{get_rustfs_endpoint, normalize_test_name_with_parts, set_up}; use reqsign_core::Context; async fn get_file_io() -> FileIO { set_up(); - let minio_endpoint = get_minio_endpoint(); + let rustfs_endpoint = get_rustfs_endpoint(); FileIOBuilder::new(Arc::new(OpenDalStorageFactory::S3 { customized_credential_load: None, })) .with_props(vec![ - (S3_ENDPOINT, minio_endpoint), + (S3_ENDPOINT, rustfs_endpoint), (S3_ACCESS_KEY_ID, "admin".to_string()), (S3_SECRET_ACCESS_KEY, "password".to_string()), (S3_REGION, "us-east-1".to_string()), @@ -138,7 +138,7 @@ mod tests { Self { credential } } - fn new_minio() -> Self { + fn new_rustfs() -> Self { Self::new(Some(AwsCredential { access_key_id: "admin".to_string(), secret_access_key: "password".to_string(), @@ -162,7 +162,7 @@ mod tests { #[test] fn test_custom_aws_credential_loader_instantiation() { // Test creating CustomAwsCredentialLoader with mock loader - let mock_loader = MockCredentialLoader::new_minio(); + let mock_loader = MockCredentialLoader::new_rustfs(); let custom_loader = CustomAwsCredentialLoader::new(mock_loader); // Test that the loader can be used in FileIOBuilder with OpenDalStorageFactory @@ -182,17 +182,17 @@ mod tests { let _file_io = get_file_io().await; // Create a mock credential loader - let mock_loader = MockCredentialLoader::new_minio(); + let mock_loader = MockCredentialLoader::new_rustfs(); let custom_loader = CustomAwsCredentialLoader::new(mock_loader); - let minio_endpoint = get_minio_endpoint(); + let rustfs_endpoint = get_rustfs_endpoint(); // Build FileIO with custom credential loader via OpenDalStorageFactory let file_io_with_custom_creds = FileIOBuilder::new(Arc::new(OpenDalStorageFactory::S3 { customized_credential_load: Some(custom_loader), })) .with_props(vec![ - (S3_ENDPOINT, minio_endpoint), + (S3_ENDPOINT, rustfs_endpoint), (S3_REGION, "us-east-1".to_string()), (S3_PATH_STYLE_ACCESS, "true".to_string()), ]) @@ -213,14 +213,14 @@ mod tests { let mock_loader = MockCredentialLoader::new(None); let custom_loader = CustomAwsCredentialLoader::new(mock_loader); - let minio_endpoint = get_minio_endpoint(); + let rustfs_endpoint = get_rustfs_endpoint(); // Build FileIO with custom credential loader via OpenDalStorageFactory let file_io_with_custom_creds = FileIOBuilder::new(Arc::new(OpenDalStorageFactory::S3 { customized_credential_load: Some(custom_loader), })) .with_props(vec![ - (S3_ENDPOINT, minio_endpoint), + (S3_ENDPOINT, rustfs_endpoint), (S3_REGION, "us-east-1".to_string()), (S3_PATH_STYLE_ACCESS, "true".to_string()), ]) diff --git a/crates/storage/opendal/tests/resolving_storage_test.rs b/crates/storage/opendal/tests/resolving_storage_test.rs index ba94b8332d..615da2f4b3 100644 --- a/crates/storage/opendal/tests/resolving_storage_test.rs +++ b/crates/storage/opendal/tests/resolving_storage_test.rs @@ -33,16 +33,16 @@ mod tests { S3_SECRET_ACCESS_KEY, }; use iceberg_storage_opendal::OpenDalResolvingStorageFactory; - use iceberg_test_utils::{get_minio_endpoint, normalize_test_name_with_parts, set_up}; + use iceberg_test_utils::{get_rustfs_endpoint, normalize_test_name_with_parts, set_up}; fn get_resolving_file_io() -> iceberg::io::FileIO { set_up(); - let minio_endpoint = get_minio_endpoint(); + let rustfs_endpoint = get_rustfs_endpoint(); FileIOBuilder::new(Arc::new(OpenDalResolvingStorageFactory::new())) .with_props(vec![ - (S3_ENDPOINT, minio_endpoint), + (S3_ENDPOINT, rustfs_endpoint), (S3_ACCESS_KEY_ID, "admin".to_string()), (S3_SECRET_ACCESS_KEY, "password".to_string()), (S3_REGION, "us-east-1".to_string()), @@ -268,9 +268,9 @@ mod tests { use reqsign_core::Context; #[derive(Debug)] - struct MinioCredentialLoader; + struct RustfsCredentialLoader; - impl ProvideCredential for MinioCredentialLoader { + impl ProvideCredential for RustfsCredentialLoader { type Credential = AwsCredential; async fn provide_credential( @@ -287,14 +287,14 @@ mod tests { } set_up(); - let minio_endpoint = get_minio_endpoint(); + let rustfs_endpoint = get_rustfs_endpoint(); let factory = OpenDalResolvingStorageFactory::new() - .with_s3_credential_loader(CustomAwsCredentialLoader::new(MinioCredentialLoader)); + .with_s3_credential_loader(CustomAwsCredentialLoader::new(RustfsCredentialLoader)); let file_io = FileIOBuilder::new(Arc::new(factory)) .with_props(vec![ - (S3_ENDPOINT, minio_endpoint), + (S3_ENDPOINT, rustfs_endpoint), (S3_REGION, "us-east-1".to_string()), (S3_PATH_STYLE_ACCESS, "true".to_string()), ]) diff --git a/crates/test_utils/src/lib.rs b/crates/test_utils/src/lib.rs index e44d96c385..2b992c2271 100644 --- a/crates/test_utils/src/lib.rs +++ b/crates/test_utils/src/lib.rs @@ -37,24 +37,24 @@ mod common { } // Environment variable names for service endpoints - pub const ENV_MINIO_ENDPOINT: &str = "ICEBERG_TEST_MINIO_ENDPOINT"; + pub const ENV_RUSTFS_ENDPOINT: &str = "ICEBERG_TEST_RUSTFS_ENDPOINT"; pub const ENV_REST_CATALOG_ENDPOINT: &str = "ICEBERG_TEST_REST_ENDPOINT"; pub const ENV_HMS_ENDPOINT: &str = "ICEBERG_TEST_HMS_ENDPOINT"; pub const ENV_GLUE_ENDPOINT: &str = "ICEBERG_TEST_GLUE_ENDPOINT"; pub const ENV_GCS_ENDPOINT: &str = "ICEBERG_TEST_GCS_ENDPOINT"; // Default ports matching dev/docker-compose.yaml - pub const DEFAULT_MINIO_PORT: u16 = 9000; + pub const DEFAULT_RUSTFS_PORT: u16 = 9000; pub const DEFAULT_REST_CATALOG_PORT: u16 = 8181; pub const DEFAULT_HMS_PORT: u16 = 9083; pub const DEFAULT_GLUE_PORT: u16 = 5001; pub const DEFAULT_GCS_PORT: u16 = 4443; - /// Returns the MinIO S3-compatible endpoint. - /// Checks ICEBERG_TEST_MINIO_ENDPOINT env var, otherwise returns localhost default. - pub fn get_minio_endpoint() -> String { - std::env::var(ENV_MINIO_ENDPOINT) - .unwrap_or_else(|_| format!("http://localhost:{DEFAULT_MINIO_PORT}")) + /// Returns the RustFS S3-compatible endpoint. + /// Checks ICEBERG_TEST_RUSTFS_ENDPOINT env var, otherwise returns localhost default. + pub fn get_rustfs_endpoint() -> String { + std::env::var(ENV_RUSTFS_ENDPOINT) + .unwrap_or_else(|_| format!("http://localhost:{DEFAULT_RUSTFS_PORT}")) } /// Returns the REST catalog endpoint. diff --git a/dev/docker-compose.yaml b/dev/docker-compose.yaml index 16c999cb9a..93b5769918 100644 --- a/dev/docker-compose.yaml +++ b/dev/docker-compose.yaml @@ -25,51 +25,42 @@ networks: services: # ============================================================================= - # MinIO - S3-compatible storage (shared by all tests) + # RustFS - S3-compatible storage (shared by all tests) # ============================================================================= - minio: - image: quay.io/minio/minio:RELEASE.2025-05-24T17-08-30Z + rustfs: + image: rustfs/rustfs:1.0.0 environment: - - MINIO_ROOT_USER=admin - - MINIO_ROOT_PASSWORD=password - - MINIO_DOMAIN=minio - hostname: minio + - RUSTFS_ACCESS_KEY=admin + - RUSTFS_SECRET_KEY=password + - RUSTFS_SERVER_DOMAINS=rustfs + hostname: rustfs networks: iceberg_test: # Add aliases for virtual-hosted style bucket access aliases: - - icebergdata.minio - - warehouse.minio - - bucket1.minio + - icebergdata.rustfs + - warehouse.rustfs + - bucket1.rustfs ports: - "9000:9000" - "9001:9001" - command: ["server", "/data", "--console-address", ":9001"] healthcheck: - test: ["CMD", "mc", "ready", "local"] + test: ["CMD", "curl", "-f", "http://localhost:9000/health/ready"] interval: 5s timeout: 5s retries: 5 - # MinIO client - creates buckets for tests - mc: + # Creates buckets for tests + create-buckets: depends_on: - minio: + rustfs: condition: service_healthy - image: quay.io/minio/mc:RELEASE.2025-05-21T01-59-54Z - environment: - - AWS_ACCESS_KEY_ID=admin - - AWS_SECRET_ACCESS_KEY=password - - AWS_REGION=us-east-1 + image: rustfs/rustfs:1.0.0 entrypoint: > /bin/sh -c " - /usr/bin/mc alias set minio http://minio:9000 admin password; - /usr/bin/mc mb --ignore-existing minio/icebergdata; - /usr/bin/mc mb --ignore-existing minio/warehouse; - /usr/bin/mc mb --ignore-existing minio/bucket1; - /usr/bin/mc policy set public minio/icebergdata; - /usr/bin/mc policy set public minio/warehouse; - /usr/bin/mc policy set public minio/bucket1; + curl -fsS --aws-sigv4 aws:amz:us-east-1:s3 -u admin:password -X PUT http://rustfs:9000/icebergdata; + curl -fsS --aws-sigv4 aws:amz:us-east-1:s3 -u admin:password -X PUT http://rustfs:9000/warehouse; + curl -fsS --aws-sigv4 aws:amz:us-east-1:s3 -u admin:password -X PUT http://rustfs:9000/bucket1; echo 'Buckets created successfully'; tail -f /dev/null " @@ -89,9 +80,9 @@ services: - CATALOG_URI=jdbc:sqlite:file:/tmp/iceberg_rest.db?journal_mode=WAL - CATALOG_WAREHOUSE=s3://icebergdata/demo - CATALOG_IO__IMPL=org.apache.iceberg.aws.s3.S3FileIO - - CATALOG_S3_ENDPOINT=http://minio:9000 + - CATALOG_S3_ENDPOINT=http://rustfs:9000 depends_on: - minio: + rustfs: condition: service_healthy networks: iceberg_test: @@ -113,7 +104,7 @@ services: dockerfile: Dockerfile platform: ${DOCKER_DEFAULT_PLATFORM:-linux/amd64} depends_on: - minio: + rustfs: condition: service_healthy networks: iceberg_test: @@ -170,7 +161,7 @@ services: depends_on: rest: condition: service_healthy - minio: + rustfs: condition: service_healthy environment: - AWS_ACCESS_KEY_ID=admin diff --git a/dev/hms/core-site.xml b/dev/hms/core-site.xml index f23efa8885..60fcdf7331 100644 --- a/dev/hms/core-site.xml +++ b/dev/hms/core-site.xml @@ -30,7 +30,7 @@ fs.s3a.endpoint - http://minio:9000 + http://rustfs:9000 fs.s3a.access.key diff --git a/dev/spark/spark-defaults.conf b/dev/spark/spark-defaults.conf index b576f624b9..8b3f3b62fe 100644 --- a/dev/spark/spark-defaults.conf +++ b/dev/spark/spark-defaults.conf @@ -23,7 +23,7 @@ spark.sql.catalog.rest.type rest spark.sql.catalog.rest.uri http://rest:8181 spark.sql.catalog.rest.io-impl org.apache.iceberg.aws.s3.S3FileIO spark.sql.catalog.rest.warehouse s3://warehouse/rest/ -spark.sql.catalog.rest.s3.endpoint http://minio:9000 +spark.sql.catalog.rest.s3.endpoint http://rustfs:9000 spark.sql.catalog.rest.cache-enabled false spark.sql.defaultCatalog rest diff --git a/website/src/reference/container-runtimes.md b/website/src/reference/container-runtimes.md index 7b4ac38511..258afd48e2 100644 --- a/website/src/reference/container-runtimes.md +++ b/website/src/reference/container-runtimes.md @@ -19,7 +19,7 @@ # Container Runtimes -Iceberg-rust uses containers for integration tests, where `docker` and `docker compose` start containers for MinIO and various catalogs. You can use any of the following container runtimes. +Iceberg-rust uses containers for integration tests, where `docker` and `docker compose` start containers for RustFS and various catalogs. You can use any of the following container runtimes. ## Docker Desktop From 2f227a260e6f965d5aba5f41a98fac228ea16f36 Mon Sep 17 00:00:00 2001 From: Kevin Liu Date: Thu, 24 Sep 2026 11:28:52 -0700 Subject: [PATCH 2/5] ci: use vendor-neutral object-store naming for S3 test service Rename the RustFS service, hostname, bucket aliases, test helpers and env var to a generic object-store name (matching PyIceberg), so future backend swaps only need an image change. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- crates/catalog/hms/tests/hms_catalog_test.rs | 6 ++-- crates/catalog/loader/tests/common/mod.rs | 11 +++---- crates/integration_tests/src/lib.rs | 6 ++-- .../storage/opendal/tests/file_io_s3_test.rs | 20 ++++++------- .../opendal/tests/resolving_storage_test.rs | 16 +++++----- crates/test_utils/src/lib.rs | 14 ++++----- dev/docker-compose.yaml | 30 +++++++++---------- dev/hms/core-site.xml | 2 +- dev/spark/spark-defaults.conf | 2 +- website/src/reference/container-runtimes.md | 2 +- 10 files changed, 55 insertions(+), 54 deletions(-) diff --git a/crates/catalog/hms/tests/hms_catalog_test.rs b/crates/catalog/hms/tests/hms_catalog_test.rs index bfdb62b54f..b64184ce77 100644 --- a/crates/catalog/hms/tests/hms_catalog_test.rs +++ b/crates/catalog/hms/tests/hms_catalog_test.rs @@ -33,7 +33,7 @@ use iceberg_catalog_hms::{ HmsCatalog, HmsCatalogBuilder, THRIFT_TRANSPORT_BUFFERED, }; use iceberg_storage_opendal::OpenDalStorageFactory; -use iceberg_test_utils::{get_hms_endpoint, get_rustfs_endpoint, set_up}; +use iceberg_test_utils::{get_hms_endpoint, get_object_store_endpoint, set_up}; use tokio::time::sleep; use tracing::info; @@ -43,7 +43,7 @@ async fn get_catalog() -> HmsCatalog { set_up(); let hms_endpoint = get_hms_endpoint(); - let rustfs_endpoint = get_rustfs_endpoint(); + let object_store_endpoint = get_object_store_endpoint(); let props = HashMap::from([ (HMS_CATALOG_PROP_URI.to_string(), hms_endpoint), @@ -55,7 +55,7 @@ async fn get_catalog() -> HmsCatalog { HMS_CATALOG_PROP_WAREHOUSE.to_string(), "s3a://warehouse/hive".to_string(), ), - (S3_ENDPOINT.to_string(), rustfs_endpoint), + (S3_ENDPOINT.to_string(), object_store_endpoint), (S3_ACCESS_KEY_ID.to_string(), "admin".to_string()), (S3_SECRET_ACCESS_KEY.to_string(), "password".to_string()), (S3_REGION.to_string(), "us-east-1".to_string()), diff --git a/crates/catalog/loader/tests/common/mod.rs b/crates/catalog/loader/tests/common/mod.rs index 39475f5c93..0b7e22cea1 100644 --- a/crates/catalog/loader/tests/common/mod.rs +++ b/crates/catalog/loader/tests/common/mod.rs @@ -50,7 +50,8 @@ use iceberg_catalog_sql::{ }; use iceberg_storage_opendal::OpenDalStorageFactory; use iceberg_test_utils::{ - get_glue_endpoint, get_hms_endpoint, get_rest_catalog_endpoint, get_rustfs_endpoint, set_up, + get_glue_endpoint, get_hms_endpoint, get_object_store_endpoint, get_rest_catalog_endpoint, + set_up, }; use sqlx::migrate::MigrateDatabase; use tempfile::TempDir; @@ -230,7 +231,7 @@ async fn rest_catalog(kms_client_factory: Arc) -> RestCata async fn glue_catalog(kms_client_factory: Arc) -> GlueCatalog { let glue_endpoint = get_glue_endpoint(); - let rustfs_endpoint = get_rustfs_endpoint(); + let object_store_endpoint = get_object_store_endpoint(); let props = HashMap::from([ (AWS_ACCESS_KEY_ID.to_string(), "my_access_id".to_string()), @@ -239,7 +240,7 @@ async fn glue_catalog(kms_client_factory: Arc) -> GlueCata "my_secret_key".to_string(), ), (AWS_REGION_NAME.to_string(), "us-east-1".to_string()), - (S3_ENDPOINT.to_string(), rustfs_endpoint), + (S3_ENDPOINT.to_string(), object_store_endpoint), (S3_ACCESS_KEY_ID.to_string(), "admin".to_string()), (S3_SECRET_ACCESS_KEY.to_string(), "password".to_string()), (S3_REGION.to_string(), "us-east-1".to_string()), @@ -279,7 +280,7 @@ async fn glue_catalog(kms_client_factory: Arc) -> GlueCata async fn hms_catalog(kms_client_factory: Arc) -> HmsCatalog { let hms_endpoint = get_hms_endpoint(); - let rustfs_endpoint = get_rustfs_endpoint(); + let object_store_endpoint = get_object_store_endpoint(); let props = HashMap::from([ (HMS_CATALOG_PROP_URI.to_string(), hms_endpoint), @@ -291,7 +292,7 @@ async fn hms_catalog(kms_client_factory: Arc) -> HmsCatalo HMS_CATALOG_PROP_WAREHOUSE.to_string(), "s3a://warehouse/hive".to_string(), ), - (S3_ENDPOINT.to_string(), rustfs_endpoint), + (S3_ENDPOINT.to_string(), object_store_endpoint), (S3_ACCESS_KEY_ID.to_string(), "admin".to_string()), (S3_SECRET_ACCESS_KEY.to_string(), "password".to_string()), (S3_REGION.to_string(), "us-east-1".to_string()), diff --git a/crates/integration_tests/src/lib.rs b/crates/integration_tests/src/lib.rs index fe8bb7d318..c137b46f86 100644 --- a/crates/integration_tests/src/lib.rs +++ b/crates/integration_tests/src/lib.rs @@ -22,7 +22,7 @@ use iceberg::io::{ S3_ACCESS_KEY_ID, S3_ENDPOINT, S3_PATH_STYLE_ACCESS, S3_REGION, S3_SECRET_ACCESS_KEY, }; use iceberg_catalog_rest::REST_CATALOG_PROP_URI; -use iceberg_test_utils::{get_rest_catalog_endpoint, get_rustfs_endpoint, set_up}; +use iceberg_test_utils::{get_object_store_endpoint, get_rest_catalog_endpoint, set_up}; /// Global test fixture that uses environment-based configuration. /// This assumes Docker containers are started externally (e.g., via `make docker-up`). @@ -39,11 +39,11 @@ impl GlobalTestFixture { set_up(); let rest_endpoint = get_rest_catalog_endpoint(); - let rustfs_endpoint = get_rustfs_endpoint(); + let object_store_endpoint = get_object_store_endpoint(); let catalog_config = HashMap::from([ (REST_CATALOG_PROP_URI.to_string(), rest_endpoint), - (S3_ENDPOINT.to_string(), rustfs_endpoint), + (S3_ENDPOINT.to_string(), object_store_endpoint), (S3_ACCESS_KEY_ID.to_string(), "admin".to_string()), (S3_SECRET_ACCESS_KEY.to_string(), "password".to_string()), (S3_REGION.to_string(), "us-east-1".to_string()), diff --git a/crates/storage/opendal/tests/file_io_s3_test.rs b/crates/storage/opendal/tests/file_io_s3_test.rs index 365842793d..ac6342dddb 100644 --- a/crates/storage/opendal/tests/file_io_s3_test.rs +++ b/crates/storage/opendal/tests/file_io_s3_test.rs @@ -32,19 +32,19 @@ mod tests { use iceberg_storage_opendal::{ AwsCredential, CustomAwsCredentialLoader, OpenDalStorageFactory, ProvideCredential, }; - use iceberg_test_utils::{get_rustfs_endpoint, normalize_test_name_with_parts, set_up}; + use iceberg_test_utils::{get_object_store_endpoint, normalize_test_name_with_parts, set_up}; use reqsign_core::Context; async fn get_file_io() -> FileIO { set_up(); - let rustfs_endpoint = get_rustfs_endpoint(); + let object_store_endpoint = get_object_store_endpoint(); FileIOBuilder::new(Arc::new(OpenDalStorageFactory::S3 { customized_credential_load: None, })) .with_props(vec![ - (S3_ENDPOINT, rustfs_endpoint), + (S3_ENDPOINT, object_store_endpoint), (S3_ACCESS_KEY_ID, "admin".to_string()), (S3_SECRET_ACCESS_KEY, "password".to_string()), (S3_REGION, "us-east-1".to_string()), @@ -138,7 +138,7 @@ mod tests { Self { credential } } - fn new_rustfs() -> Self { + fn new_object_store() -> Self { Self::new(Some(AwsCredential { access_key_id: "admin".to_string(), secret_access_key: "password".to_string(), @@ -162,7 +162,7 @@ mod tests { #[test] fn test_custom_aws_credential_loader_instantiation() { // Test creating CustomAwsCredentialLoader with mock loader - let mock_loader = MockCredentialLoader::new_rustfs(); + let mock_loader = MockCredentialLoader::new_object_store(); let custom_loader = CustomAwsCredentialLoader::new(mock_loader); // Test that the loader can be used in FileIOBuilder with OpenDalStorageFactory @@ -182,17 +182,17 @@ mod tests { let _file_io = get_file_io().await; // Create a mock credential loader - let mock_loader = MockCredentialLoader::new_rustfs(); + let mock_loader = MockCredentialLoader::new_object_store(); let custom_loader = CustomAwsCredentialLoader::new(mock_loader); - let rustfs_endpoint = get_rustfs_endpoint(); + let object_store_endpoint = get_object_store_endpoint(); // Build FileIO with custom credential loader via OpenDalStorageFactory let file_io_with_custom_creds = FileIOBuilder::new(Arc::new(OpenDalStorageFactory::S3 { customized_credential_load: Some(custom_loader), })) .with_props(vec![ - (S3_ENDPOINT, rustfs_endpoint), + (S3_ENDPOINT, object_store_endpoint), (S3_REGION, "us-east-1".to_string()), (S3_PATH_STYLE_ACCESS, "true".to_string()), ]) @@ -213,14 +213,14 @@ mod tests { let mock_loader = MockCredentialLoader::new(None); let custom_loader = CustomAwsCredentialLoader::new(mock_loader); - let rustfs_endpoint = get_rustfs_endpoint(); + let object_store_endpoint = get_object_store_endpoint(); // Build FileIO with custom credential loader via OpenDalStorageFactory let file_io_with_custom_creds = FileIOBuilder::new(Arc::new(OpenDalStorageFactory::S3 { customized_credential_load: Some(custom_loader), })) .with_props(vec![ - (S3_ENDPOINT, rustfs_endpoint), + (S3_ENDPOINT, object_store_endpoint), (S3_REGION, "us-east-1".to_string()), (S3_PATH_STYLE_ACCESS, "true".to_string()), ]) diff --git a/crates/storage/opendal/tests/resolving_storage_test.rs b/crates/storage/opendal/tests/resolving_storage_test.rs index 615da2f4b3..bcc3d51842 100644 --- a/crates/storage/opendal/tests/resolving_storage_test.rs +++ b/crates/storage/opendal/tests/resolving_storage_test.rs @@ -33,16 +33,16 @@ mod tests { S3_SECRET_ACCESS_KEY, }; use iceberg_storage_opendal::OpenDalResolvingStorageFactory; - use iceberg_test_utils::{get_rustfs_endpoint, normalize_test_name_with_parts, set_up}; + use iceberg_test_utils::{get_object_store_endpoint, normalize_test_name_with_parts, set_up}; fn get_resolving_file_io() -> iceberg::io::FileIO { set_up(); - let rustfs_endpoint = get_rustfs_endpoint(); + let object_store_endpoint = get_object_store_endpoint(); FileIOBuilder::new(Arc::new(OpenDalResolvingStorageFactory::new())) .with_props(vec![ - (S3_ENDPOINT, rustfs_endpoint), + (S3_ENDPOINT, object_store_endpoint), (S3_ACCESS_KEY_ID, "admin".to_string()), (S3_SECRET_ACCESS_KEY, "password".to_string()), (S3_REGION, "us-east-1".to_string()), @@ -268,9 +268,9 @@ mod tests { use reqsign_core::Context; #[derive(Debug)] - struct RustfsCredentialLoader; + struct ObjectStoreCredentialLoader; - impl ProvideCredential for RustfsCredentialLoader { + impl ProvideCredential for ObjectStoreCredentialLoader { type Credential = AwsCredential; async fn provide_credential( @@ -287,14 +287,14 @@ mod tests { } set_up(); - let rustfs_endpoint = get_rustfs_endpoint(); + let object_store_endpoint = get_object_store_endpoint(); let factory = OpenDalResolvingStorageFactory::new() - .with_s3_credential_loader(CustomAwsCredentialLoader::new(RustfsCredentialLoader)); + .with_s3_credential_loader(CustomAwsCredentialLoader::new(ObjectStoreCredentialLoader)); let file_io = FileIOBuilder::new(Arc::new(factory)) .with_props(vec![ - (S3_ENDPOINT, rustfs_endpoint), + (S3_ENDPOINT, object_store_endpoint), (S3_REGION, "us-east-1".to_string()), (S3_PATH_STYLE_ACCESS, "true".to_string()), ]) diff --git a/crates/test_utils/src/lib.rs b/crates/test_utils/src/lib.rs index 2b992c2271..b64e462696 100644 --- a/crates/test_utils/src/lib.rs +++ b/crates/test_utils/src/lib.rs @@ -37,24 +37,24 @@ mod common { } // Environment variable names for service endpoints - pub const ENV_RUSTFS_ENDPOINT: &str = "ICEBERG_TEST_RUSTFS_ENDPOINT"; + pub const ENV_OBJECT_STORE_ENDPOINT: &str = "ICEBERG_TEST_OBJECT_STORE_ENDPOINT"; pub const ENV_REST_CATALOG_ENDPOINT: &str = "ICEBERG_TEST_REST_ENDPOINT"; pub const ENV_HMS_ENDPOINT: &str = "ICEBERG_TEST_HMS_ENDPOINT"; pub const ENV_GLUE_ENDPOINT: &str = "ICEBERG_TEST_GLUE_ENDPOINT"; pub const ENV_GCS_ENDPOINT: &str = "ICEBERG_TEST_GCS_ENDPOINT"; // Default ports matching dev/docker-compose.yaml - pub const DEFAULT_RUSTFS_PORT: u16 = 9000; + pub const DEFAULT_OBJECT_STORE_PORT: u16 = 9000; pub const DEFAULT_REST_CATALOG_PORT: u16 = 8181; pub const DEFAULT_HMS_PORT: u16 = 9083; pub const DEFAULT_GLUE_PORT: u16 = 5001; pub const DEFAULT_GCS_PORT: u16 = 4443; - /// Returns the RustFS S3-compatible endpoint. - /// Checks ICEBERG_TEST_RUSTFS_ENDPOINT env var, otherwise returns localhost default. - pub fn get_rustfs_endpoint() -> String { - std::env::var(ENV_RUSTFS_ENDPOINT) - .unwrap_or_else(|_| format!("http://localhost:{DEFAULT_RUSTFS_PORT}")) + /// Returns the S3-compatible object store endpoint. + /// Checks ICEBERG_TEST_OBJECT_STORE_ENDPOINT env var, otherwise returns localhost default. + pub fn get_object_store_endpoint() -> String { + std::env::var(ENV_OBJECT_STORE_ENDPOINT) + .unwrap_or_else(|_| format!("http://localhost:{DEFAULT_OBJECT_STORE_PORT}")) } /// Returns the REST catalog endpoint. diff --git a/dev/docker-compose.yaml b/dev/docker-compose.yaml index 93b5769918..4c5b7d0155 100644 --- a/dev/docker-compose.yaml +++ b/dev/docker-compose.yaml @@ -25,22 +25,22 @@ networks: services: # ============================================================================= - # RustFS - S3-compatible storage (shared by all tests) + # Object store - S3-compatible storage backed by RustFS (shared by all tests) # ============================================================================= - rustfs: + object-store: image: rustfs/rustfs:1.0.0 environment: - RUSTFS_ACCESS_KEY=admin - RUSTFS_SECRET_KEY=password - - RUSTFS_SERVER_DOMAINS=rustfs - hostname: rustfs + - RUSTFS_SERVER_DOMAINS=object-store + hostname: object-store networks: iceberg_test: # Add aliases for virtual-hosted style bucket access aliases: - - icebergdata.rustfs - - warehouse.rustfs - - bucket1.rustfs + - icebergdata.object-store + - warehouse.object-store + - bucket1.object-store ports: - "9000:9000" - "9001:9001" @@ -53,14 +53,14 @@ services: # Creates buckets for tests create-buckets: depends_on: - rustfs: + object-store: condition: service_healthy image: rustfs/rustfs:1.0.0 entrypoint: > /bin/sh -c " - curl -fsS --aws-sigv4 aws:amz:us-east-1:s3 -u admin:password -X PUT http://rustfs:9000/icebergdata; - curl -fsS --aws-sigv4 aws:amz:us-east-1:s3 -u admin:password -X PUT http://rustfs:9000/warehouse; - curl -fsS --aws-sigv4 aws:amz:us-east-1:s3 -u admin:password -X PUT http://rustfs:9000/bucket1; + curl -fsS --aws-sigv4 aws:amz:us-east-1:s3 -u admin:password -X PUT http://object-store:9000/icebergdata; + curl -fsS --aws-sigv4 aws:amz:us-east-1:s3 -u admin:password -X PUT http://object-store:9000/warehouse; + curl -fsS --aws-sigv4 aws:amz:us-east-1:s3 -u admin:password -X PUT http://object-store:9000/bucket1; echo 'Buckets created successfully'; tail -f /dev/null " @@ -80,9 +80,9 @@ services: - CATALOG_URI=jdbc:sqlite:file:/tmp/iceberg_rest.db?journal_mode=WAL - CATALOG_WAREHOUSE=s3://icebergdata/demo - CATALOG_IO__IMPL=org.apache.iceberg.aws.s3.S3FileIO - - CATALOG_S3_ENDPOINT=http://rustfs:9000 + - CATALOG_S3_ENDPOINT=http://object-store:9000 depends_on: - rustfs: + object-store: condition: service_healthy networks: iceberg_test: @@ -104,7 +104,7 @@ services: dockerfile: Dockerfile platform: ${DOCKER_DEFAULT_PLATFORM:-linux/amd64} depends_on: - rustfs: + object-store: condition: service_healthy networks: iceberg_test: @@ -161,7 +161,7 @@ services: depends_on: rest: condition: service_healthy - rustfs: + object-store: condition: service_healthy environment: - AWS_ACCESS_KEY_ID=admin diff --git a/dev/hms/core-site.xml b/dev/hms/core-site.xml index 60fcdf7331..d88ef93570 100644 --- a/dev/hms/core-site.xml +++ b/dev/hms/core-site.xml @@ -30,7 +30,7 @@ fs.s3a.endpoint - http://rustfs:9000 + http://object-store:9000 fs.s3a.access.key diff --git a/dev/spark/spark-defaults.conf b/dev/spark/spark-defaults.conf index 8b3f3b62fe..a02640bbdd 100644 --- a/dev/spark/spark-defaults.conf +++ b/dev/spark/spark-defaults.conf @@ -23,7 +23,7 @@ spark.sql.catalog.rest.type rest spark.sql.catalog.rest.uri http://rest:8181 spark.sql.catalog.rest.io-impl org.apache.iceberg.aws.s3.S3FileIO spark.sql.catalog.rest.warehouse s3://warehouse/rest/ -spark.sql.catalog.rest.s3.endpoint http://rustfs:9000 +spark.sql.catalog.rest.s3.endpoint http://object-store:9000 spark.sql.catalog.rest.cache-enabled false spark.sql.defaultCatalog rest diff --git a/website/src/reference/container-runtimes.md b/website/src/reference/container-runtimes.md index 258afd48e2..412aa70053 100644 --- a/website/src/reference/container-runtimes.md +++ b/website/src/reference/container-runtimes.md @@ -19,7 +19,7 @@ # Container Runtimes -Iceberg-rust uses containers for integration tests, where `docker` and `docker compose` start containers for RustFS and various catalogs. You can use any of the following container runtimes. +Iceberg-rust uses containers for integration tests, where `docker` and `docker compose` start containers for an S3-compatible object store (RustFS) and various catalogs. You can use any of the following container runtimes. ## Docker Desktop From 688439867ba8ae9171147454937bbb2dcc070de5 Mon Sep 17 00:00:00 2001 From: Kevin Liu Date: Thu, 24 Sep 2026 11:38:12 -0700 Subject: [PATCH 3/5] ci: fail fast and gate readiness on test bucket creation Treat any non-200/409 response as a failure instead of ignoring it, and add a healthcheck so `docker compose up --wait` blocks until the buckets exist. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- dev/docker-compose.yaml | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/dev/docker-compose.yaml b/dev/docker-compose.yaml index 4c5b7d0155..3d1fd70e48 100644 --- a/dev/docker-compose.yaml +++ b/dev/docker-compose.yaml @@ -56,14 +56,24 @@ services: object-store: condition: service_healthy image: rustfs/rustfs:1.0.0 + # 409 means the bucket already exists, which is fine on container restarts. entrypoint: > /bin/sh -c " - curl -fsS --aws-sigv4 aws:amz:us-east-1:s3 -u admin:password -X PUT http://object-store:9000/icebergdata; - curl -fsS --aws-sigv4 aws:amz:us-east-1:s3 -u admin:password -X PUT http://object-store:9000/warehouse; - curl -fsS --aws-sigv4 aws:amz:us-east-1:s3 -u admin:password -X PUT http://object-store:9000/bucket1; + set -e; + for bucket in icebergdata warehouse bucket1; do + code=$$(curl -sS -o /dev/null -w '%{http_code}' --aws-sigv4 aws:amz:us-east-1:s3 -u admin:password -X PUT http://object-store:9000/$$bucket); + case $$code in 200|409) ;; *) echo \"Failed to create bucket $$bucket: HTTP $$code\"; exit 1;; esac; + done; + touch /tmp/buckets-ready; echo 'Buckets created successfully'; tail -f /dev/null " + # Lets `docker compose up --wait` block until the buckets exist. + healthcheck: + test: ["CMD-SHELL", "test -f /tmp/buckets-ready"] + interval: 2s + timeout: 2s + retries: 15 networks: iceberg_test: From 21f163c9088d5f0cc35805cfd8b2c4359db4006d Mon Sep 17 00:00:00 2001 From: Kevin Liu Date: Thu, 24 Sep 2026 11:40:51 -0700 Subject: [PATCH 4/5] ci: remove orphan containers on docker-up Renaming the minio service leaves stale containers holding port 9000 for anyone with a previously started stack; clean them up automatically. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index d27893390a..5371c42287 100644 --- a/Makefile +++ b/Makefile @@ -112,7 +112,7 @@ site: install-mdbook # Docker targets for integration tests docker-up: - docker compose -f dev/docker-compose.yaml up -d --build --wait + docker compose -f dev/docker-compose.yaml up -d --build --wait --remove-orphans docker-down: docker compose -f dev/docker-compose.yaml down -v --remove-orphans --timeout 0 From 1f04ccbc4124767d81902f328e58f0c28a79cc2b Mon Sep 17 00:00:00 2001 From: Kevin Liu Date: Thu, 24 Sep 2026 11:43:43 -0700 Subject: [PATCH 5/5] ci: make create-buckets a one-shot job RustFS returns 200 when re-creating an existing bucket, so plain chained curl -f calls are idempotent. Replace the status-code parsing, marker-file healthcheck and tail with a one-shot container, and gate spark-iceberg on its successful completion. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- dev/docker-compose.yaml | 25 +++++++------------------ 1 file changed, 7 insertions(+), 18 deletions(-) diff --git a/dev/docker-compose.yaml b/dev/docker-compose.yaml index 3d1fd70e48..cb89e3042d 100644 --- a/dev/docker-compose.yaml +++ b/dev/docker-compose.yaml @@ -50,30 +50,19 @@ services: timeout: 5s retries: 5 - # Creates buckets for tests + # One-shot job that creates buckets for tests. Re-creating an existing bucket + # returns 200, so this is safe to re-run. create-buckets: depends_on: object-store: condition: service_healthy image: rustfs/rustfs:1.0.0 - # 409 means the bucket already exists, which is fine on container restarts. entrypoint: > /bin/sh -c " - set -e; - for bucket in icebergdata warehouse bucket1; do - code=$$(curl -sS -o /dev/null -w '%{http_code}' --aws-sigv4 aws:amz:us-east-1:s3 -u admin:password -X PUT http://object-store:9000/$$bucket); - case $$code in 200|409) ;; *) echo \"Failed to create bucket $$bucket: HTTP $$code\"; exit 1;; esac; - done; - touch /tmp/buckets-ready; - echo 'Buckets created successfully'; - tail -f /dev/null + curl -fsS --aws-sigv4 aws:amz:us-east-1:s3 -u admin:password -X PUT http://object-store:9000/icebergdata && + curl -fsS --aws-sigv4 aws:amz:us-east-1:s3 -u admin:password -X PUT http://object-store:9000/warehouse && + curl -fsS --aws-sigv4 aws:amz:us-east-1:s3 -u admin:password -X PUT http://object-store:9000/bucket1 " - # Lets `docker compose up --wait` block until the buckets exist. - healthcheck: - test: ["CMD-SHELL", "test -f /tmp/buckets-ready"] - interval: 2s - timeout: 2s - retries: 15 networks: iceberg_test: @@ -171,8 +160,8 @@ services: depends_on: rest: condition: service_healthy - object-store: - condition: service_healthy + create-buckets: + condition: service_completed_successfully environment: - AWS_ACCESS_KEY_ID=admin - AWS_SECRET_ACCESS_KEY=password