diff --git a/.github/workflows/aws_test.yml b/.github/workflows/aws_test.yml index e59185617..9214027ae 100644 --- a/.github/workflows/aws_test.yml +++ b/.github/workflows/aws_test.yml @@ -74,8 +74,8 @@ jobs: bundle_awssdk: "ON" env: ICEBERG_TEST_S3_URI: s3://iceberg-test - AWS_ACCESS_KEY_ID: minio - AWS_SECRET_ACCESS_KEY: minio123 + AWS_ACCESS_KEY_ID: admin + AWS_SECRET_ACCESS_KEY: password AWS_DEFAULT_REGION: us-east-1 AWS_ENDPOINT_URL: http://127.0.0.1:9000 AWS_EC2_METADATA_DISABLED: "TRUE" @@ -114,10 +114,10 @@ jobs: run: | echo "CC=${{ matrix.CC }}" >> $GITHUB_ENV echo "CXX=${{ matrix.CXX }}" >> $GITHUB_ENV - - name: Start MinIO + - name: Start Object Store if: ${{ matrix.s3 == 'ON' }} shell: bash - run: bash ci/scripts/start_minio.sh + run: bash ci/scripts/start_object_store.sh - name: Set up sccache uses: ./.github/actions/setup-sccache with: diff --git a/ci/scripts/start_minio.sh b/ci/scripts/start_minio.sh deleted file mode 100644 index 793c7852b..000000000 --- a/ci/scripts/start_minio.sh +++ /dev/null @@ -1,158 +0,0 @@ -#!/usr/bin/env bash -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -set -eux - -MINIO_ROOT_USER="${MINIO_ROOT_USER:-minio}" -MINIO_ROOT_PASSWORD="${MINIO_ROOT_PASSWORD:-minio123}" -MINIO_IMAGE="${MINIO_IMAGE:-quay.io/minio/minio:latest}" -MINIO_CONTAINER_NAME="${MINIO_CONTAINER_NAME:-iceberg-minio}" -MINIO_PORT="${MINIO_PORT:-9000}" -MINIO_CONSOLE_PORT="${MINIO_CONSOLE_PORT:-9001}" -MINIO_BUCKET="${MINIO_BUCKET:-iceberg-test}" -MINIO_ENDPOINT="${MINIO_ENDPOINT:-http://127.0.0.1:${MINIO_PORT}}" - -wait_for_minio() { - for i in {1..30}; do - if curl -fsS "${MINIO_ENDPOINT}/minio/health/ready" >/dev/null; then - return 0 - fi - sleep 1 - done - echo "MinIO did not become ready after 30 seconds." >&2 - echo "Endpoint: ${MINIO_ENDPOINT}" >&2 - if command -v docker >/dev/null 2>&1; then - docker logs "${MINIO_CONTAINER_NAME}" 2>&1 || true - fi - return 1 -} - -start_minio_docker() { - if ! command -v docker >/dev/null 2>&1; then - return 1 - fi - - if docker ps -a --format '{{.Names}}' | grep -q "^${MINIO_CONTAINER_NAME}\$"; then - docker rm -f "${MINIO_CONTAINER_NAME}" - fi - - docker run -d --name "${MINIO_CONTAINER_NAME}" \ - -p "${MINIO_PORT}:9000" -p "${MINIO_CONSOLE_PORT}:9001" \ - -e "MINIO_ROOT_USER=${MINIO_ROOT_USER}" \ - -e "MINIO_ROOT_PASSWORD=${MINIO_ROOT_PASSWORD}" \ - "${MINIO_IMAGE}" \ - server /data --console-address ":${MINIO_CONSOLE_PORT}" - - wait_for_minio -} - -start_minio_macos() { - if ! command -v brew >/dev/null 2>&1; then - echo "brew is required to start MinIO on macOS without Docker" >&2 - return 1 - fi - - brew install minio - MINIO_ROOT_USER="${MINIO_ROOT_USER}" MINIO_ROOT_PASSWORD="${MINIO_ROOT_PASSWORD}" \ - minio server /tmp/minio --console-address ":${MINIO_CONSOLE_PORT}" & - wait_for_minio -} - -download_mc() { - local uname_out - uname_out="$(uname -s)" - local mc_release="RELEASE.2025-08-13T08-35-41Z" - local mc_download_url="https://github.com/minio/mc/releases/download/${mc_release}" - - local mc_dir - mc_dir="${RUNNER_TEMP:-/tmp}" - mkdir -p "${mc_dir}" - - case "${uname_out}" in - Linux*) - MC_BIN="${mc_dir}/mc" - curl -fsSL "${mc_download_url}/mc.linux-amd64.${mc_release}" -o "${MC_BIN}" - chmod +x "${MC_BIN}" - ;; - Darwin*) - MC_BIN="${mc_dir}/mc" - local arch - arch="$(uname -m)" - if [ "${arch}" = "arm64" ]; then - curl -fsSL "${mc_download_url}/mc.darwin-arm64.${mc_release}" -o "${MC_BIN}" - else - curl -fsSL "${mc_download_url}/mc.darwin-amd64.${mc_release}" -o "${MC_BIN}" - fi - chmod +x "${MC_BIN}" - ;; - MINGW*|MSYS*|CYGWIN*) - MC_BIN="${mc_dir}/mc.exe" - curl -fsSL "${mc_download_url}/mc.windows-amd64.${mc_release}.exe" -o "${MC_BIN}" - ;; - *) - echo "Unsupported OS for mc: ${uname_out}" >&2 - return 1 - ;; - esac -} - -create_bucket() { - download_mc - for i in {1..30}; do - if "${MC_BIN}" alias set local "${MINIO_ENDPOINT}" "${MINIO_ROOT_USER}" "${MINIO_ROOT_PASSWORD}"; then - break - fi - sleep 1 - done - "${MC_BIN}" mb --ignore-existing "local/${MINIO_BUCKET}" -} - -start_minio_windows() { - local minio_dir="${RUNNER_TEMP:-/tmp}" - local minio_bin="${minio_dir}/minio.exe" - local minio_release="RELEASE.2025-09-07T16-13-09Z" - curl -fsSL \ - "https://github.com/minio/minio/releases/download/${minio_release}/minio.windows-amd64.${minio_release}.exe" \ - -o "${minio_bin}" - MINIO_ROOT_USER="${MINIO_ROOT_USER}" MINIO_ROOT_PASSWORD="${MINIO_ROOT_PASSWORD}" \ - "${minio_bin}" server "${minio_dir}/minio-data" --console-address ":${MINIO_CONSOLE_PORT}" & - wait_for_minio -} - -case "$(uname -s)" in - Darwin*) - if ! start_minio_docker; then - start_minio_macos - fi - ;; - MINGW*|MSYS*|CYGWIN*) - if ! start_minio_docker; then - start_minio_windows - fi - ;; - Linux*) - start_minio_docker - ;; - *) - echo "Unsupported OS: $(uname -s)" >&2 - exit 1 - ;; -esac - -create_bucket diff --git a/ci/scripts/start_object_store.sh b/ci/scripts/start_object_store.sh new file mode 100644 index 000000000..dfd825142 --- /dev/null +++ b/ci/scripts/start_object_store.sh @@ -0,0 +1,134 @@ +#!/usr/bin/env bash +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +set -euxo pipefail + +OBJECT_STORE_VERSION="${OBJECT_STORE_VERSION:-1.0.0}" +OBJECT_STORE_IMAGE="${OBJECT_STORE_IMAGE:-rustfs/rustfs:${OBJECT_STORE_VERSION}}" +OBJECT_STORE_CONTAINER_NAME="${OBJECT_STORE_CONTAINER_NAME:-iceberg-object-store}" +OBJECT_STORE_ACCESS_KEY="${AWS_ACCESS_KEY_ID:-admin}" +OBJECT_STORE_SECRET_KEY="${AWS_SECRET_ACCESS_KEY:-password}" +OBJECT_STORE_PORT="${OBJECT_STORE_PORT:-9000}" +OBJECT_STORE_BUCKET="${OBJECT_STORE_BUCKET:-iceberg-test}" +OBJECT_STORE_ENDPOINT="${AWS_ENDPOINT_URL:-http://127.0.0.1:${OBJECT_STORE_PORT}}" +OBJECT_STORE_DIR="${RUNNER_TEMP:-/tmp}/iceberg-object-store" +OBJECT_STORE_LOG="" + +wait_for_object_store() { + for ((attempt = 0; attempt < 60; attempt++)); do + if curl -fs --connect-timeout 1 --max-time 2 "${OBJECT_STORE_ENDPOINT}/health/ready" >/dev/null; then + return 0 + fi + sleep 1 + done + echo "Object store did not become ready at ${OBJECT_STORE_ENDPOINT}." >&2 + if [ -n "${OBJECT_STORE_LOG}" ]; then + cat "${OBJECT_STORE_LOG}" >&2 + else + docker logs "${OBJECT_STORE_CONTAINER_NAME}" >&2 || true + fi + return 1 +} + +start_object_store_docker() { + if docker container inspect "${OBJECT_STORE_CONTAINER_NAME}" >/dev/null 2>&1; then + docker rm -f "${OBJECT_STORE_CONTAINER_NAME}" + fi + + docker run -d --name "${OBJECT_STORE_CONTAINER_NAME}" \ + -p "${OBJECT_STORE_PORT}:9000" \ + -e "RUSTFS_ACCESS_KEY=${OBJECT_STORE_ACCESS_KEY}" \ + -e "RUSTFS_SECRET_KEY=${OBJECT_STORE_SECRET_KEY}" \ + -e RUSTFS_CONSOLE_ENABLE=false \ + -e RUSTFS_OBS_LOG_STDOUT_ENABLED=true \ + "${OBJECT_STORE_IMAGE}" /data +} + +start_object_store_native() { + local platform binary + case "$(uname -s)-$(uname -m)" in + Darwin-arm64) + platform=macos-aarch64 + binary=rustfs + ;; + MINGW*-x86_64|MSYS*-x86_64|CYGWIN*-x86_64) + platform=windows-x86_64 + binary=rustfs.exe + ;; + *) + echo "Use Docker to run RustFS on $(uname -s)-$(uname -m)." >&2 + return 1 + ;; + esac + + local archive="rustfs-${platform}-v${OBJECT_STORE_VERSION}.zip" + local release_url="https://github.com/rustfs/rustfs/releases/download/${OBJECT_STORE_VERSION}" + mkdir -p "${OBJECT_STORE_DIR}/data" + curl -fsSL --retry 3 \ + "${release_url}/${archive}" -o "${OBJECT_STORE_DIR}/${archive}" + curl -fsSL --retry 3 "${release_url}/SHA256SUMS" -o "${OBJECT_STORE_DIR}/SHA256SUMS" + ( + cd "${OBJECT_STORE_DIR}" + grep -F " ${archive}" SHA256SUMS > rustfs.sha256 + if command -v sha256sum >/dev/null 2>&1; then + sha256sum --check rustfs.sha256 + else + shasum -a 256 --check rustfs.sha256 + fi + ) + unzip -o "${OBJECT_STORE_DIR}/${archive}" -d "${OBJECT_STORE_DIR}" + chmod +x "${OBJECT_STORE_DIR}/${binary}" + OBJECT_STORE_LOG="${OBJECT_STORE_DIR}/rustfs.log" + RUSTFS_ACCESS_KEY="${OBJECT_STORE_ACCESS_KEY}" \ + RUSTFS_SECRET_KEY="${OBJECT_STORE_SECRET_KEY}" \ + RUSTFS_ADDRESS=":${OBJECT_STORE_PORT}" \ + RUSTFS_CONSOLE_ENABLE=false \ + "${OBJECT_STORE_DIR}/${binary}" "${OBJECT_STORE_DIR}/data" \ + >"${OBJECT_STORE_LOG}" 2>&1 & +} + +create_bucket() { + export AWS_ACCESS_KEY_ID="${OBJECT_STORE_ACCESS_KEY}" + export AWS_SECRET_ACCESS_KEY="${OBJECT_STORE_SECRET_KEY}" + export AWS_DEFAULT_REGION="${AWS_DEFAULT_REGION:-us-east-1}" + # The health endpoint can pass before S3 requests are accepted. + for ((attempt = 0; attempt < 5; attempt++)); do + if aws --endpoint-url "${OBJECT_STORE_ENDPOINT}" s3api head-bucket --bucket "${OBJECT_STORE_BUCKET}" >/dev/null 2>&1 || \ + aws --endpoint-url "${OBJECT_STORE_ENDPOINT}" s3api create-bucket --bucket "${OBJECT_STORE_BUCKET}"; then + return 0 + fi + sleep 2 + done + echo "Failed to create bucket ${OBJECT_STORE_BUCKET} at ${OBJECT_STORE_ENDPOINT}." >&2 + return 1 +} + +if ! command -v aws >/dev/null 2>&1; then + echo "AWS CLI is required to create the test bucket." >&2 + exit 1 +fi + +if command -v docker >/dev/null 2>&1 && [ "$(docker info --format '{{.OSType}}' 2>/dev/null)" = linux ]; then + start_object_store_docker +else + start_object_store_native +fi + +wait_for_object_store +create_bucket diff --git a/src/iceberg/arrow/s3/arrow_s3_file_io.cc b/src/iceberg/arrow/s3/arrow_s3_file_io.cc index 7ce0d0fda..c981759ce 100644 --- a/src/iceberg/arrow/s3/arrow_s3_file_io.cc +++ b/src/iceberg/arrow/s3/arrow_s3_file_io.cc @@ -109,7 +109,7 @@ Result<::arrow::fs::S3Options> ConfigureS3Options( options.region = *region; } - // Configure endpoint (for MinIO, LocalStack, etc.) + // Configure endpoint (for S3-compatible object stores) if (const auto* endpoint = FindProperty(properties, S3Properties::kEndpoint); endpoint != nullptr) { options.endpoint_override = SplitEndpointScheme(*endpoint, options); diff --git a/src/iceberg/arrow/s3/s3_properties.h b/src/iceberg/arrow/s3/s3_properties.h index 605c748ca..50dafa56f 100644 --- a/src/iceberg/arrow/s3/s3_properties.h +++ b/src/iceberg/arrow/s3/s3_properties.h @@ -44,9 +44,9 @@ struct S3Properties { static constexpr std::string_view kSessionToken = "s3.session-token"; /// AWS region, standard Iceberg client property. static constexpr std::string_view kClientRegion = "client.region"; - /// Custom endpoint override (for MinIO, LocalStack, etc.) + /// Custom endpoint override (for S3-compatible object stores) static constexpr std::string_view kEndpoint = "s3.endpoint"; - /// Whether to use path-style access (needed for MinIO) + /// Whether to use path-style access (needed for some S3-compatible object stores) static constexpr std::string_view kPathStyleAccess = "s3.path-style-access"; /// Whether SSL is enabled static constexpr std::string_view kSslEnabled = "s3.ssl.enabled";