|
| 1 | +#!/usr/bin/env bash |
| 2 | +# |
| 3 | +# Licensed to the Apache Software Foundation (ASF) under one |
| 4 | +# or more contributor license agreements. See the NOTICE file |
| 5 | +# distributed with this work for additional information |
| 6 | +# regarding copyright ownership. The ASF licenses this file |
| 7 | +# to you under the Apache License, Version 2.0 (the |
| 8 | +# "License"); you may not use this file except in compliance |
| 9 | +# with the License. You may obtain a copy of the License at |
| 10 | +# |
| 11 | +# http://www.apache.org/licenses/LICENSE-2.0 |
| 12 | +# |
| 13 | +# Unless required by applicable law or agreed to in writing, |
| 14 | +# software distributed under the License is distributed on an |
| 15 | +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY |
| 16 | +# KIND, either express or implied. See the License for the |
| 17 | +# specific language governing permissions and limitations |
| 18 | +# under the License. |
| 19 | + |
| 20 | +set -euxo pipefail |
| 21 | + |
| 22 | +OBJECT_STORE_VERSION="${OBJECT_STORE_VERSION:-1.0.0}" |
| 23 | +OBJECT_STORE_IMAGE="${OBJECT_STORE_IMAGE:-rustfs/rustfs:${OBJECT_STORE_VERSION}}" |
| 24 | +OBJECT_STORE_CONTAINER_NAME="${OBJECT_STORE_CONTAINER_NAME:-iceberg-object-store}" |
| 25 | +OBJECT_STORE_ACCESS_KEY="${AWS_ACCESS_KEY_ID:-admin}" |
| 26 | +OBJECT_STORE_SECRET_KEY="${AWS_SECRET_ACCESS_KEY:-password}" |
| 27 | +OBJECT_STORE_PORT="${OBJECT_STORE_PORT:-9000}" |
| 28 | +OBJECT_STORE_BUCKET="${OBJECT_STORE_BUCKET:-iceberg-test}" |
| 29 | +OBJECT_STORE_ENDPOINT="${AWS_ENDPOINT_URL:-http://127.0.0.1:${OBJECT_STORE_PORT}}" |
| 30 | +OBJECT_STORE_DIR="${RUNNER_TEMP:-/tmp}/iceberg-object-store" |
| 31 | +OBJECT_STORE_LOG="" |
| 32 | + |
| 33 | +wait_for_object_store() { |
| 34 | + for ((attempt = 0; attempt < 60; attempt++)); do |
| 35 | + if curl -fs --connect-timeout 1 --max-time 2 "${OBJECT_STORE_ENDPOINT}/health/ready" >/dev/null; then |
| 36 | + return 0 |
| 37 | + fi |
| 38 | + sleep 1 |
| 39 | + done |
| 40 | + echo "Object store did not become ready at ${OBJECT_STORE_ENDPOINT}." >&2 |
| 41 | + if [ -n "${OBJECT_STORE_LOG}" ]; then |
| 42 | + cat "${OBJECT_STORE_LOG}" >&2 |
| 43 | + else |
| 44 | + docker logs "${OBJECT_STORE_CONTAINER_NAME}" >&2 || true |
| 45 | + fi |
| 46 | + return 1 |
| 47 | +} |
| 48 | + |
| 49 | +start_object_store_docker() { |
| 50 | + if docker container inspect "${OBJECT_STORE_CONTAINER_NAME}" >/dev/null 2>&1; then |
| 51 | + docker rm -f "${OBJECT_STORE_CONTAINER_NAME}" |
| 52 | + fi |
| 53 | + |
| 54 | + docker run -d --name "${OBJECT_STORE_CONTAINER_NAME}" \ |
| 55 | + -p "${OBJECT_STORE_PORT}:9000" \ |
| 56 | + -e "RUSTFS_ACCESS_KEY=${OBJECT_STORE_ACCESS_KEY}" \ |
| 57 | + -e "RUSTFS_SECRET_KEY=${OBJECT_STORE_SECRET_KEY}" \ |
| 58 | + -e RUSTFS_CONSOLE_ENABLE=false \ |
| 59 | + -e RUSTFS_OBS_LOG_STDOUT_ENABLED=true \ |
| 60 | + "${OBJECT_STORE_IMAGE}" /data |
| 61 | +} |
| 62 | + |
| 63 | +start_object_store_native() { |
| 64 | + local platform binary |
| 65 | + case "$(uname -s)-$(uname -m)" in |
| 66 | + Darwin-arm64) |
| 67 | + platform=macos-aarch64 |
| 68 | + binary=rustfs |
| 69 | + ;; |
| 70 | + MINGW*-x86_64|MSYS*-x86_64|CYGWIN*-x86_64) |
| 71 | + platform=windows-x86_64 |
| 72 | + binary=rustfs.exe |
| 73 | + ;; |
| 74 | + *) |
| 75 | + echo "Use Docker to run RustFS on $(uname -s)-$(uname -m)." >&2 |
| 76 | + return 1 |
| 77 | + ;; |
| 78 | + esac |
| 79 | + |
| 80 | + local archive="rustfs-${platform}-v${OBJECT_STORE_VERSION}.zip" |
| 81 | + local release_url="https://github.com/rustfs/rustfs/releases/download/${OBJECT_STORE_VERSION}" |
| 82 | + mkdir -p "${OBJECT_STORE_DIR}/data" |
| 83 | + curl -fsSL --retry 3 \ |
| 84 | + "${release_url}/${archive}" -o "${OBJECT_STORE_DIR}/${archive}" |
| 85 | + curl -fsSL --retry 3 "${release_url}/SHA256SUMS" -o "${OBJECT_STORE_DIR}/SHA256SUMS" |
| 86 | + ( |
| 87 | + cd "${OBJECT_STORE_DIR}" |
| 88 | + grep -F " ${archive}" SHA256SUMS > rustfs.sha256 |
| 89 | + if command -v sha256sum >/dev/null 2>&1; then |
| 90 | + sha256sum --check rustfs.sha256 |
| 91 | + else |
| 92 | + shasum -a 256 --check rustfs.sha256 |
| 93 | + fi |
| 94 | + ) |
| 95 | + unzip -o "${OBJECT_STORE_DIR}/${archive}" -d "${OBJECT_STORE_DIR}" |
| 96 | + chmod +x "${OBJECT_STORE_DIR}/${binary}" |
| 97 | + OBJECT_STORE_LOG="${OBJECT_STORE_DIR}/rustfs.log" |
| 98 | + RUSTFS_ACCESS_KEY="${OBJECT_STORE_ACCESS_KEY}" \ |
| 99 | + RUSTFS_SECRET_KEY="${OBJECT_STORE_SECRET_KEY}" \ |
| 100 | + RUSTFS_ADDRESS=":${OBJECT_STORE_PORT}" \ |
| 101 | + RUSTFS_CONSOLE_ENABLE=false \ |
| 102 | + "${OBJECT_STORE_DIR}/${binary}" "${OBJECT_STORE_DIR}/data" \ |
| 103 | + >"${OBJECT_STORE_LOG}" 2>&1 & |
| 104 | +} |
| 105 | + |
| 106 | +create_bucket() { |
| 107 | + export AWS_ACCESS_KEY_ID="${OBJECT_STORE_ACCESS_KEY}" |
| 108 | + export AWS_SECRET_ACCESS_KEY="${OBJECT_STORE_SECRET_KEY}" |
| 109 | + export AWS_DEFAULT_REGION="${AWS_DEFAULT_REGION:-us-east-1}" |
| 110 | + # The health endpoint can pass before S3 requests are accepted. |
| 111 | + for ((attempt = 0; attempt < 5; attempt++)); do |
| 112 | + if aws --endpoint-url "${OBJECT_STORE_ENDPOINT}" s3api head-bucket --bucket "${OBJECT_STORE_BUCKET}" >/dev/null 2>&1 || \ |
| 113 | + aws --endpoint-url "${OBJECT_STORE_ENDPOINT}" s3api create-bucket --bucket "${OBJECT_STORE_BUCKET}"; then |
| 114 | + return 0 |
| 115 | + fi |
| 116 | + sleep 2 |
| 117 | + done |
| 118 | + echo "Failed to create bucket ${OBJECT_STORE_BUCKET} at ${OBJECT_STORE_ENDPOINT}." >&2 |
| 119 | + return 1 |
| 120 | +} |
| 121 | + |
| 122 | +if ! command -v aws >/dev/null 2>&1; then |
| 123 | + echo "AWS CLI is required to create the test bucket." >&2 |
| 124 | + exit 1 |
| 125 | +fi |
| 126 | + |
| 127 | +if command -v docker >/dev/null 2>&1 && [ "$(docker info --format '{{.OSType}}' 2>/dev/null)" = linux ]; then |
| 128 | + start_object_store_docker |
| 129 | +else |
| 130 | + start_object_store_native |
| 131 | +fi |
| 132 | + |
| 133 | +wait_for_object_store |
| 134 | +create_bucket |
0 commit comments