From 7ce5eecdf79023460a6d147e3bd01f5465891658 Mon Sep 17 00:00:00 2001 From: Sreesh Maheshwar Date: Thu, 24 Sep 2026 20:43:49 +0100 Subject: [PATCH 1/3] ci: replace MinIO with RustFS in S3 tests Use RustFS 1.0.0 for Docker and native test fixtures, and initialize the S3 bucket with the AWS CLI. Generated-by: Codex --- .github/workflows/aws_test.yml | 8 +- ci/scripts/start_minio.sh | 158 ------------------------------- ci/scripts/start_object_store.sh | 114 ++++++++++++++++++++++ 3 files changed, 118 insertions(+), 162 deletions(-) delete mode 100644 ci/scripts/start_minio.sh create mode 100644 ci/scripts/start_object_store.sh diff --git a/.github/workflows/aws_test.yml b/.github/workflows/aws_test.yml index e59185617..f6c667651 100644 --- a/.github/workflows/aws_test.yml +++ b/.github/workflows/aws_test.yml @@ -74,8 +74,8 @@ jobs: bundle_awssdk: "ON" env: ICEBERG_TEST_S3_URI: s3://iceberg-test - AWS_ACCESS_KEY_ID: minio - AWS_SECRET_ACCESS_KEY: minio123 + AWS_ACCESS_KEY_ID: admin + AWS_SECRET_ACCESS_KEY: password AWS_DEFAULT_REGION: us-east-1 AWS_ENDPOINT_URL: http://127.0.0.1:9000 AWS_EC2_METADATA_DISABLED: "TRUE" @@ -114,10 +114,10 @@ jobs: run: | echo "CC=${{ matrix.CC }}" >> $GITHUB_ENV echo "CXX=${{ matrix.CXX }}" >> $GITHUB_ENV - - name: Start MinIO + - name: Start RustFS if: ${{ matrix.s3 == 'ON' }} shell: bash - run: bash ci/scripts/start_minio.sh + run: bash ci/scripts/start_object_store.sh - name: Set up sccache uses: ./.github/actions/setup-sccache with: diff --git a/ci/scripts/start_minio.sh b/ci/scripts/start_minio.sh deleted file mode 100644 index 793c7852b..000000000 --- a/ci/scripts/start_minio.sh +++ /dev/null @@ -1,158 +0,0 @@ -#!/usr/bin/env bash -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -set -eux - -MINIO_ROOT_USER="${MINIO_ROOT_USER:-minio}" -MINIO_ROOT_PASSWORD="${MINIO_ROOT_PASSWORD:-minio123}" -MINIO_IMAGE="${MINIO_IMAGE:-quay.io/minio/minio:latest}" -MINIO_CONTAINER_NAME="${MINIO_CONTAINER_NAME:-iceberg-minio}" -MINIO_PORT="${MINIO_PORT:-9000}" -MINIO_CONSOLE_PORT="${MINIO_CONSOLE_PORT:-9001}" -MINIO_BUCKET="${MINIO_BUCKET:-iceberg-test}" -MINIO_ENDPOINT="${MINIO_ENDPOINT:-http://127.0.0.1:${MINIO_PORT}}" - -wait_for_minio() { - for i in {1..30}; do - if curl -fsS "${MINIO_ENDPOINT}/minio/health/ready" >/dev/null; then - return 0 - fi - sleep 1 - done - echo "MinIO did not become ready after 30 seconds." >&2 - echo "Endpoint: ${MINIO_ENDPOINT}" >&2 - if command -v docker >/dev/null 2>&1; then - docker logs "${MINIO_CONTAINER_NAME}" 2>&1 || true - fi - return 1 -} - -start_minio_docker() { - if ! command -v docker >/dev/null 2>&1; then - return 1 - fi - - if docker ps -a --format '{{.Names}}' | grep -q "^${MINIO_CONTAINER_NAME}\$"; then - docker rm -f "${MINIO_CONTAINER_NAME}" - fi - - docker run -d --name "${MINIO_CONTAINER_NAME}" \ - -p "${MINIO_PORT}:9000" -p "${MINIO_CONSOLE_PORT}:9001" \ - -e "MINIO_ROOT_USER=${MINIO_ROOT_USER}" \ - -e "MINIO_ROOT_PASSWORD=${MINIO_ROOT_PASSWORD}" \ - "${MINIO_IMAGE}" \ - server /data --console-address ":${MINIO_CONSOLE_PORT}" - - wait_for_minio -} - -start_minio_macos() { - if ! command -v brew >/dev/null 2>&1; then - echo "brew is required to start MinIO on macOS without Docker" >&2 - return 1 - fi - - brew install minio - MINIO_ROOT_USER="${MINIO_ROOT_USER}" MINIO_ROOT_PASSWORD="${MINIO_ROOT_PASSWORD}" \ - minio server /tmp/minio --console-address ":${MINIO_CONSOLE_PORT}" & - wait_for_minio -} - -download_mc() { - local uname_out - uname_out="$(uname -s)" - local mc_release="RELEASE.2025-08-13T08-35-41Z" - local mc_download_url="https://github.com/minio/mc/releases/download/${mc_release}" - - local mc_dir - mc_dir="${RUNNER_TEMP:-/tmp}" - mkdir -p "${mc_dir}" - - case "${uname_out}" in - Linux*) - MC_BIN="${mc_dir}/mc" - curl -fsSL "${mc_download_url}/mc.linux-amd64.${mc_release}" -o "${MC_BIN}" - chmod +x "${MC_BIN}" - ;; - Darwin*) - MC_BIN="${mc_dir}/mc" - local arch - arch="$(uname -m)" - if [ "${arch}" = "arm64" ]; then - curl -fsSL "${mc_download_url}/mc.darwin-arm64.${mc_release}" -o "${MC_BIN}" - else - curl -fsSL "${mc_download_url}/mc.darwin-amd64.${mc_release}" -o "${MC_BIN}" - fi - chmod +x "${MC_BIN}" - ;; - MINGW*|MSYS*|CYGWIN*) - MC_BIN="${mc_dir}/mc.exe" - curl -fsSL "${mc_download_url}/mc.windows-amd64.${mc_release}.exe" -o "${MC_BIN}" - ;; - *) - echo "Unsupported OS for mc: ${uname_out}" >&2 - return 1 - ;; - esac -} - -create_bucket() { - download_mc - for i in {1..30}; do - if "${MC_BIN}" alias set local "${MINIO_ENDPOINT}" "${MINIO_ROOT_USER}" "${MINIO_ROOT_PASSWORD}"; then - break - fi - sleep 1 - done - "${MC_BIN}" mb --ignore-existing "local/${MINIO_BUCKET}" -} - -start_minio_windows() { - local minio_dir="${RUNNER_TEMP:-/tmp}" - local minio_bin="${minio_dir}/minio.exe" - local minio_release="RELEASE.2025-09-07T16-13-09Z" - curl -fsSL \ - "https://github.com/minio/minio/releases/download/${minio_release}/minio.windows-amd64.${minio_release}.exe" \ - -o "${minio_bin}" - MINIO_ROOT_USER="${MINIO_ROOT_USER}" MINIO_ROOT_PASSWORD="${MINIO_ROOT_PASSWORD}" \ - "${minio_bin}" server "${minio_dir}/minio-data" --console-address ":${MINIO_CONSOLE_PORT}" & - wait_for_minio -} - -case "$(uname -s)" in - Darwin*) - if ! start_minio_docker; then - start_minio_macos - fi - ;; - MINGW*|MSYS*|CYGWIN*) - if ! start_minio_docker; then - start_minio_windows - fi - ;; - Linux*) - start_minio_docker - ;; - *) - echo "Unsupported OS: $(uname -s)" >&2 - exit 1 - ;; -esac - -create_bucket diff --git a/ci/scripts/start_object_store.sh b/ci/scripts/start_object_store.sh new file mode 100644 index 000000000..9d028c634 --- /dev/null +++ b/ci/scripts/start_object_store.sh @@ -0,0 +1,114 @@ +#!/usr/bin/env bash +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +set -euo pipefail + +OBJECT_STORE_VERSION="${OBJECT_STORE_VERSION:-1.0.0}" +OBJECT_STORE_IMAGE="${OBJECT_STORE_IMAGE:-rustfs/rustfs:${OBJECT_STORE_VERSION}}" +OBJECT_STORE_CONTAINER_NAME="${OBJECT_STORE_CONTAINER_NAME:-iceberg-object-store}" +OBJECT_STORE_ACCESS_KEY="${AWS_ACCESS_KEY_ID:-admin}" +OBJECT_STORE_SECRET_KEY="${AWS_SECRET_ACCESS_KEY:-password}" +OBJECT_STORE_PORT="${OBJECT_STORE_PORT:-9000}" +OBJECT_STORE_CONSOLE_PORT="${OBJECT_STORE_CONSOLE_PORT:-9001}" +OBJECT_STORE_BUCKET="${OBJECT_STORE_BUCKET:-iceberg-test}" +OBJECT_STORE_ENDPOINT="${AWS_ENDPOINT_URL:-http://127.0.0.1:${OBJECT_STORE_PORT}}" +OBJECT_STORE_DIR="${RUNNER_TEMP:-/tmp}/iceberg-object-store" + +wait_for_object_store() { + for ((attempt = 0; attempt < 60; attempt++)); do + if curl -fsS --connect-timeout 1 --max-time 2 "${OBJECT_STORE_ENDPOINT}/health/ready" >/dev/null; then + return 0 + fi + sleep 1 + done + echo "RustFS did not become ready at ${OBJECT_STORE_ENDPOINT}." >&2 + if [ -f "${OBJECT_STORE_DIR}/rustfs.log" ]; then + cat "${OBJECT_STORE_DIR}/rustfs.log" >&2 + else + docker logs "${OBJECT_STORE_CONTAINER_NAME}" >&2 || true + fi + return 1 +} + +start_object_store_docker() { + if docker container inspect "${OBJECT_STORE_CONTAINER_NAME}" >/dev/null 2>&1; then + docker rm -f "${OBJECT_STORE_CONTAINER_NAME}" + fi + + docker run -d --name "${OBJECT_STORE_CONTAINER_NAME}" \ + -p "${OBJECT_STORE_PORT}:9000" -p "${OBJECT_STORE_CONSOLE_PORT}:9001" \ + -e "RUSTFS_ACCESS_KEY=${OBJECT_STORE_ACCESS_KEY}" \ + -e "RUSTFS_SECRET_KEY=${OBJECT_STORE_SECRET_KEY}" \ + -e RUSTFS_CONSOLE_ENABLE=true \ + "${OBJECT_STORE_IMAGE}" /data +} + +start_object_store_native() { + local platform binary + case "$(uname -s)-$(uname -m)" in + Darwin-arm64) + platform=macos-aarch64 + binary=rustfs + ;; + MINGW*-x86_64|MSYS*-x86_64|CYGWIN*-x86_64) + platform=windows-x86_64 + binary=rustfs.exe + ;; + *) + echo "Use Docker to run RustFS on $(uname -s)-$(uname -m)." >&2 + return 1 + ;; + esac + + mkdir -p "${OBJECT_STORE_DIR}/data" + curl -fsSL --retry 3 \ + "https://github.com/rustfs/rustfs/releases/download/${OBJECT_STORE_VERSION}/rustfs-${platform}-v${OBJECT_STORE_VERSION}.zip" \ + -o "${OBJECT_STORE_DIR}/rustfs.zip" + unzip -o "${OBJECT_STORE_DIR}/rustfs.zip" -d "${OBJECT_STORE_DIR}" + chmod +x "${OBJECT_STORE_DIR}/${binary}" + RUSTFS_ACCESS_KEY="${OBJECT_STORE_ACCESS_KEY}" \ + RUSTFS_SECRET_KEY="${OBJECT_STORE_SECRET_KEY}" \ + RUSTFS_ADDRESS=":${OBJECT_STORE_PORT}" \ + RUSTFS_CONSOLE_ADDRESS=":${OBJECT_STORE_CONSOLE_PORT}" \ + RUSTFS_CONSOLE_ENABLE=true \ + "${OBJECT_STORE_DIR}/${binary}" "${OBJECT_STORE_DIR}/data" \ + >"${OBJECT_STORE_DIR}/rustfs.log" 2>&1 & +} + +create_bucket() { + export AWS_ACCESS_KEY_ID="${OBJECT_STORE_ACCESS_KEY}" + export AWS_SECRET_ACCESS_KEY="${OBJECT_STORE_SECRET_KEY}" + export AWS_DEFAULT_REGION="${AWS_DEFAULT_REGION:-us-east-1}" + aws --endpoint-url "${OBJECT_STORE_ENDPOINT}" s3api head-bucket --bucket "${OBJECT_STORE_BUCKET}" || \ + aws --endpoint-url "${OBJECT_STORE_ENDPOINT}" s3api create-bucket --bucket "${OBJECT_STORE_BUCKET}" +} + +if ! command -v aws >/dev/null 2>&1; then + echo "AWS CLI is required to create the test bucket." >&2 + exit 1 +fi + +if command -v docker >/dev/null 2>&1 && [ "$(docker info --format '{{.OSType}}' 2>/dev/null)" = linux ]; then + start_object_store_docker +else + start_object_store_native +fi + +wait_for_object_store +create_bucket From beb35bc8bb966f2a8a2d78544033c1ad0d089c6a Mon Sep 17 00:00:00 2001 From: Sreesh Maheshwar Date: Thu, 24 Sep 2026 21:22:35 +0100 Subject: [PATCH 2/3] ci: improve RustFS startup diagnostics and readiness Expose server logs, retry bucket initialization, verify native downloads, and disable the unused console. Generated-by: Codex --- ci/scripts/start_object_store.sh | 40 +++++++++++++++++++++++--------- 1 file changed, 29 insertions(+), 11 deletions(-) diff --git a/ci/scripts/start_object_store.sh b/ci/scripts/start_object_store.sh index 9d028c634..93194f859 100644 --- a/ci/scripts/start_object_store.sh +++ b/ci/scripts/start_object_store.sh @@ -25,14 +25,13 @@ OBJECT_STORE_CONTAINER_NAME="${OBJECT_STORE_CONTAINER_NAME:-iceberg-object-store OBJECT_STORE_ACCESS_KEY="${AWS_ACCESS_KEY_ID:-admin}" OBJECT_STORE_SECRET_KEY="${AWS_SECRET_ACCESS_KEY:-password}" OBJECT_STORE_PORT="${OBJECT_STORE_PORT:-9000}" -OBJECT_STORE_CONSOLE_PORT="${OBJECT_STORE_CONSOLE_PORT:-9001}" OBJECT_STORE_BUCKET="${OBJECT_STORE_BUCKET:-iceberg-test}" OBJECT_STORE_ENDPOINT="${AWS_ENDPOINT_URL:-http://127.0.0.1:${OBJECT_STORE_PORT}}" OBJECT_STORE_DIR="${RUNNER_TEMP:-/tmp}/iceberg-object-store" wait_for_object_store() { for ((attempt = 0; attempt < 60; attempt++)); do - if curl -fsS --connect-timeout 1 --max-time 2 "${OBJECT_STORE_ENDPOINT}/health/ready" >/dev/null; then + if curl -fs --connect-timeout 1 --max-time 2 "${OBJECT_STORE_ENDPOINT}/health/ready" >/dev/null; then return 0 fi sleep 1 @@ -52,10 +51,11 @@ start_object_store_docker() { fi docker run -d --name "${OBJECT_STORE_CONTAINER_NAME}" \ - -p "${OBJECT_STORE_PORT}:9000" -p "${OBJECT_STORE_CONSOLE_PORT}:9001" \ + -p "${OBJECT_STORE_PORT}:9000" \ -e "RUSTFS_ACCESS_KEY=${OBJECT_STORE_ACCESS_KEY}" \ -e "RUSTFS_SECRET_KEY=${OBJECT_STORE_SECRET_KEY}" \ - -e RUSTFS_CONSOLE_ENABLE=true \ + -e RUSTFS_CONSOLE_ENABLE=false \ + -e RUSTFS_OBS_LOG_STDOUT_ENABLED=true \ "${OBJECT_STORE_IMAGE}" /data } @@ -76,17 +76,27 @@ start_object_store_native() { ;; esac + local archive="rustfs-${platform}-v${OBJECT_STORE_VERSION}.zip" + local release_url="https://github.com/rustfs/rustfs/releases/download/${OBJECT_STORE_VERSION}" mkdir -p "${OBJECT_STORE_DIR}/data" curl -fsSL --retry 3 \ - "https://github.com/rustfs/rustfs/releases/download/${OBJECT_STORE_VERSION}/rustfs-${platform}-v${OBJECT_STORE_VERSION}.zip" \ - -o "${OBJECT_STORE_DIR}/rustfs.zip" - unzip -o "${OBJECT_STORE_DIR}/rustfs.zip" -d "${OBJECT_STORE_DIR}" + "${release_url}/${archive}" -o "${OBJECT_STORE_DIR}/${archive}" + curl -fsSL --retry 3 "${release_url}/SHA256SUMS" -o "${OBJECT_STORE_DIR}/SHA256SUMS" + ( + cd "${OBJECT_STORE_DIR}" + grep -F " ${archive}" SHA256SUMS > rustfs.sha256 + if command -v sha256sum >/dev/null 2>&1; then + sha256sum --check rustfs.sha256 + else + shasum -a 256 --check rustfs.sha256 + fi + ) + unzip -o "${OBJECT_STORE_DIR}/${archive}" -d "${OBJECT_STORE_DIR}" chmod +x "${OBJECT_STORE_DIR}/${binary}" RUSTFS_ACCESS_KEY="${OBJECT_STORE_ACCESS_KEY}" \ RUSTFS_SECRET_KEY="${OBJECT_STORE_SECRET_KEY}" \ RUSTFS_ADDRESS=":${OBJECT_STORE_PORT}" \ - RUSTFS_CONSOLE_ADDRESS=":${OBJECT_STORE_CONSOLE_PORT}" \ - RUSTFS_CONSOLE_ENABLE=true \ + RUSTFS_CONSOLE_ENABLE=false \ "${OBJECT_STORE_DIR}/${binary}" "${OBJECT_STORE_DIR}/data" \ >"${OBJECT_STORE_DIR}/rustfs.log" 2>&1 & } @@ -95,8 +105,16 @@ create_bucket() { export AWS_ACCESS_KEY_ID="${OBJECT_STORE_ACCESS_KEY}" export AWS_SECRET_ACCESS_KEY="${OBJECT_STORE_SECRET_KEY}" export AWS_DEFAULT_REGION="${AWS_DEFAULT_REGION:-us-east-1}" - aws --endpoint-url "${OBJECT_STORE_ENDPOINT}" s3api head-bucket --bucket "${OBJECT_STORE_BUCKET}" || \ - aws --endpoint-url "${OBJECT_STORE_ENDPOINT}" s3api create-bucket --bucket "${OBJECT_STORE_BUCKET}" + # The health endpoint can pass before S3 requests are accepted. + for ((attempt = 0; attempt < 5; attempt++)); do + if aws --endpoint-url "${OBJECT_STORE_ENDPOINT}" s3api head-bucket --bucket "${OBJECT_STORE_BUCKET}" >/dev/null 2>&1 || \ + aws --endpoint-url "${OBJECT_STORE_ENDPOINT}" s3api create-bucket --bucket "${OBJECT_STORE_BUCKET}"; then + return 0 + fi + sleep 2 + done + echo "Failed to create bucket ${OBJECT_STORE_BUCKET} at ${OBJECT_STORE_ENDPOINT}." >&2 + return 1 } if ! command -v aws >/dev/null 2>&1; then From a4bbe27b021de4031af1fe6e039abb59cc33b858 Mon Sep 17 00:00:00 2001 From: Kevin Liu Date: Thu, 24 Sep 2026 18:54:08 -0700 Subject: [PATCH 3/3] ci: use generic object store naming and improve startup diagnostics - Rename the CI step to "Start Object Store" - Enable shell tracing in start_object_store.sh - Print the native server log only when it was started natively - Replace MinIO-specific wording in S3 comments Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- .github/workflows/aws_test.yml | 2 +- ci/scripts/start_object_store.sh | 12 +++++++----- src/iceberg/arrow/s3/arrow_s3_file_io.cc | 2 +- src/iceberg/arrow/s3/s3_properties.h | 4 ++-- 4 files changed, 11 insertions(+), 9 deletions(-) diff --git a/.github/workflows/aws_test.yml b/.github/workflows/aws_test.yml index f6c667651..9214027ae 100644 --- a/.github/workflows/aws_test.yml +++ b/.github/workflows/aws_test.yml @@ -114,7 +114,7 @@ jobs: run: | echo "CC=${{ matrix.CC }}" >> $GITHUB_ENV echo "CXX=${{ matrix.CXX }}" >> $GITHUB_ENV - - name: Start RustFS + - name: Start Object Store if: ${{ matrix.s3 == 'ON' }} shell: bash run: bash ci/scripts/start_object_store.sh diff --git a/ci/scripts/start_object_store.sh b/ci/scripts/start_object_store.sh index 93194f859..dfd825142 100644 --- a/ci/scripts/start_object_store.sh +++ b/ci/scripts/start_object_store.sh @@ -17,7 +17,7 @@ # specific language governing permissions and limitations # under the License. -set -euo pipefail +set -euxo pipefail OBJECT_STORE_VERSION="${OBJECT_STORE_VERSION:-1.0.0}" OBJECT_STORE_IMAGE="${OBJECT_STORE_IMAGE:-rustfs/rustfs:${OBJECT_STORE_VERSION}}" @@ -28,6 +28,7 @@ OBJECT_STORE_PORT="${OBJECT_STORE_PORT:-9000}" OBJECT_STORE_BUCKET="${OBJECT_STORE_BUCKET:-iceberg-test}" OBJECT_STORE_ENDPOINT="${AWS_ENDPOINT_URL:-http://127.0.0.1:${OBJECT_STORE_PORT}}" OBJECT_STORE_DIR="${RUNNER_TEMP:-/tmp}/iceberg-object-store" +OBJECT_STORE_LOG="" wait_for_object_store() { for ((attempt = 0; attempt < 60; attempt++)); do @@ -36,9 +37,9 @@ wait_for_object_store() { fi sleep 1 done - echo "RustFS did not become ready at ${OBJECT_STORE_ENDPOINT}." >&2 - if [ -f "${OBJECT_STORE_DIR}/rustfs.log" ]; then - cat "${OBJECT_STORE_DIR}/rustfs.log" >&2 + echo "Object store did not become ready at ${OBJECT_STORE_ENDPOINT}." >&2 + if [ -n "${OBJECT_STORE_LOG}" ]; then + cat "${OBJECT_STORE_LOG}" >&2 else docker logs "${OBJECT_STORE_CONTAINER_NAME}" >&2 || true fi @@ -93,12 +94,13 @@ start_object_store_native() { ) unzip -o "${OBJECT_STORE_DIR}/${archive}" -d "${OBJECT_STORE_DIR}" chmod +x "${OBJECT_STORE_DIR}/${binary}" + OBJECT_STORE_LOG="${OBJECT_STORE_DIR}/rustfs.log" RUSTFS_ACCESS_KEY="${OBJECT_STORE_ACCESS_KEY}" \ RUSTFS_SECRET_KEY="${OBJECT_STORE_SECRET_KEY}" \ RUSTFS_ADDRESS=":${OBJECT_STORE_PORT}" \ RUSTFS_CONSOLE_ENABLE=false \ "${OBJECT_STORE_DIR}/${binary}" "${OBJECT_STORE_DIR}/data" \ - >"${OBJECT_STORE_DIR}/rustfs.log" 2>&1 & + >"${OBJECT_STORE_LOG}" 2>&1 & } create_bucket() { diff --git a/src/iceberg/arrow/s3/arrow_s3_file_io.cc b/src/iceberg/arrow/s3/arrow_s3_file_io.cc index 7ce0d0fda..c981759ce 100644 --- a/src/iceberg/arrow/s3/arrow_s3_file_io.cc +++ b/src/iceberg/arrow/s3/arrow_s3_file_io.cc @@ -109,7 +109,7 @@ Result<::arrow::fs::S3Options> ConfigureS3Options( options.region = *region; } - // Configure endpoint (for MinIO, LocalStack, etc.) + // Configure endpoint (for S3-compatible object stores) if (const auto* endpoint = FindProperty(properties, S3Properties::kEndpoint); endpoint != nullptr) { options.endpoint_override = SplitEndpointScheme(*endpoint, options); diff --git a/src/iceberg/arrow/s3/s3_properties.h b/src/iceberg/arrow/s3/s3_properties.h index 605c748ca..50dafa56f 100644 --- a/src/iceberg/arrow/s3/s3_properties.h +++ b/src/iceberg/arrow/s3/s3_properties.h @@ -44,9 +44,9 @@ struct S3Properties { static constexpr std::string_view kSessionToken = "s3.session-token"; /// AWS region, standard Iceberg client property. static constexpr std::string_view kClientRegion = "client.region"; - /// Custom endpoint override (for MinIO, LocalStack, etc.) + /// Custom endpoint override (for S3-compatible object stores) static constexpr std::string_view kEndpoint = "s3.endpoint"; - /// Whether to use path-style access (needed for MinIO) + /// Whether to use path-style access (needed for some S3-compatible object stores) static constexpr std::string_view kPathStyleAccess = "s3.path-style-access"; /// Whether SSL is enabled static constexpr std::string_view kSslEnabled = "s3.ssl.enabled";