diff --git a/.config/make/tests.mak b/.config/make/tests.mak index 5b3ce1de64..05ea511b5f 100644 --- a/.config/make/tests.mak +++ b/.config/make/tests.mak @@ -15,7 +15,7 @@ test: core-deps test-deps ## Run all tests .PHONY: e2e-server e2e-server: ## Run e2e-server tests - sh $(shell pwd)/scripts/run.sh + bash $(shell pwd)/scripts/run.sh .PHONY: probe-e2e probe-e2e: ## Probe e2e tests diff --git a/.github/actions/cargo-build-jobs/action.yml b/.github/actions/cargo-build-jobs/action.yml new file mode 100644 index 0000000000..e0cecd02da --- /dev/null +++ b/.github/actions/cargo-build-jobs/action.yml @@ -0,0 +1,25 @@ +# Copyright 2024 RustFS Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +name: "Configure Cargo build parallelism" +description: > + Sets CARGO_BUILD_JOBS from CPU and memory (up to half of logical cores, reduced + when RAM per rustc job would be too low). Linux, macOS, and Windows (Git Bash). + +runs: + using: composite + steps: + - name: Compute CARGO_BUILD_JOBS + shell: bash + run: bash "${GITHUB_WORKSPACE}/scripts/ci/compute-cargo-build-jobs.sh" diff --git a/.github/actions/setup/action.yml b/.github/actions/setup/action.yml index 7a2171b976..1018533dab 100644 --- a/.github/actions/setup/action.yml +++ b/.github/actions/setup/action.yml @@ -41,9 +41,18 @@ inputs: required: false default: "" + configure-cargo-jobs: + description: "Set CARGO_BUILD_JOBS from CPU and available memory" + required: false + default: "true" + runs: using: "composite" steps: + - name: Configure Cargo build parallelism + if: inputs.configure-cargo-jobs == 'true' + uses: ./.github/actions/cargo-build-jobs + - name: Install system dependencies (Ubuntu) if: runner.os == 'Linux' shell: bash diff --git a/.github/s3tests/README.md b/.github/s3tests/README.md index af61ed25a0..429b6eb517 100644 --- a/.github/s3tests/README.md +++ b/.github/s3tests/README.md @@ -8,8 +8,8 @@ The `s3tests.conf` file is based on the official `s3tests.conf.SAMPLE` from the ### Key Configuration Points -- **Host**: Set via `${S3_HOST}` environment variable (e.g., `rustfs-single` for single-node, `lb` for multi-node) -- **Port**: 9000 (standard RustFS port) +- **Host**: Set via `${S3_HOST}` environment variable (e.g., `127.0.0.1` on the runner, `rustfs-single` inside `rustfs-net`) +- **Port**: Set via `${S3_PORT}` (host port; use `9000` when publishing `:9000`, or an ephemeral port when avoiding conflicts on shared runners) - **Credentials**: Uses `${S3_ACCESS_KEY}` and `${S3_SECRET_KEY}` from workflow environment - **TLS**: Disabled (`is_secure = False`) @@ -62,6 +62,7 @@ docker run -d --name rustfs-single \ # Generate config export S3_HOST=rustfs-single +export S3_PORT=9000 envsubst < .github/s3tests/s3tests.conf > /tmp/s3tests.conf # Run tests diff --git a/.github/s3tests/s3tests.conf b/.github/s3tests/s3tests.conf index fa2c4d57c9..2e6b1a6640 100644 --- a/.github/s3tests/s3tests.conf +++ b/.github/s3tests/s3tests.conf @@ -2,8 +2,8 @@ # Based on: https://github.com/ceph/s3-tests/blob/master/s3tests.conf.SAMPLE # # Usage: -# Single-node: S3_HOST=rustfs-single envsubst < s3tests.conf > /tmp/s3tests.conf -# Multi-node: S3_HOST=lb envsubst < s3tests.conf > /tmp/s3tests.conf +# export S3_HOST=127.0.0.1 S3_PORT=9000 # S3_PORT must match the published host port +# envsubst < s3tests.conf > /tmp/s3tests.conf [DEFAULT] ## this section is just used for host, port and bucket_prefix @@ -11,8 +11,8 @@ # host set for RustFS - will be substituted via envsubst host = ${S3_HOST} -# port for RustFS -port = 9000 +# port for RustFS (host port; default 9000 — set S3_PORT when mapping e.g. 19001:9000) +port = ${S3_PORT} ## say "False" to disable TLS is_secure = False diff --git a/.github/workflows/audit.yml b/.github/workflows/audit.yml index ade2e6930f..789665e4d7 100644 --- a/.github/workflows/audit.yml +++ b/.github/workflows/audit.yml @@ -46,6 +46,9 @@ jobs: - name: Checkout repository uses: actions/checkout@v6 + - name: Configure Cargo build parallelism + uses: ./.github/actions/cargo-build-jobs + - name: Install cargo-audit uses: taiki-e/install-action@v2 with: diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 9aa65ffb13..c8a36e066b 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -67,7 +67,6 @@ concurrency: env: CARGO_TERM_COLOR: always RUST_BACKTRACE: 1 - CARGO_BUILD_JOBS: 2 jobs: @@ -153,7 +152,7 @@ jobs: - name: Build debug binary run: | touch rustfs/build.rs - cargo build -p rustfs --bins --jobs 2 + cargo build -p rustfs --bins - name: Upload debug binary uses: actions/upload-artifact@v6 @@ -173,6 +172,9 @@ jobs: - name: Checkout repository uses: actions/checkout@v6 + - name: Configure Cargo build parallelism + uses: ./.github/actions/cargo-build-jobs + - name: Clean up previous test run run: | rm -rf /tmp/rustfs @@ -187,6 +189,11 @@ jobs: - name: Make binary executable run: chmod +x ./target/debug/rustfs + - name: Install native build dependencies for s3s-e2e + run: | + sudo apt-get update -qq + sudo apt-get install -y build-essential cmake pkg-config + - name: Setup Rust toolchain for s3s-e2e installation uses: dtolnay/rust-toolchain@stable @@ -231,6 +238,7 @@ jobs: - name: Run implemented s3-tests run: | + export S3_PORT="$(python3 -c "import socket; s=socket.socket(); s.bind(('127.0.0.1',0)); print(s.getsockname()[1]); s.close()")" DEPLOY_MODE=binary \ RUSTFS_BINARY=./target/debug/rustfs \ TEST_MODE=single \ diff --git a/.github/workflows/e2e-s3tests.yml b/.github/workflows/e2e-s3tests.yml index a2a8829ae5..e8144a35a5 100644 --- a/.github/workflows/e2e-s3tests.yml +++ b/.github/workflows/e2e-s3tests.yml @@ -101,9 +101,11 @@ jobs: - name: Start single RustFS run: | + S3_PORT=$(python3 -c "import socket; s=socket.socket(); s.bind(('127.0.0.1',0)); print(s.getsockname()[1]); s.close()") + echo "S3_PORT=${S3_PORT}" >> "$GITHUB_ENV" docker run -d --name rustfs-single \ --network rustfs-net \ - -p 9000:9000 \ + -p "${S3_PORT}:9000" \ -e RUSTFS_ADDRESS=0.0.0.0:9000 \ -e RUSTFS_ACCESS_KEY=$S3_ACCESS_KEY \ -e RUSTFS_SECRET_KEY=$S3_SECRET_KEY \ @@ -114,7 +116,7 @@ jobs: - name: Wait for RustFS ready run: | for i in {1..60}; do - if curl -sf http://127.0.0.1:9000/health >/dev/null 2>&1; then + if curl -sf "http://127.0.0.1:${S3_PORT}/health" >/dev/null 2>&1; then echo "RustFS is ready" exit 0 fi @@ -135,6 +137,7 @@ jobs: - name: Generate s3tests config run: | export S3_HOST=127.0.0.1 + export S3_PORT="${S3_PORT}" envsubst < .github/s3tests/s3tests.conf > s3tests.conf - name: Provision s3-tests alt user (required by suite) @@ -148,7 +151,7 @@ jobs: -X PUT \ -H 'Content-Type: application/json' \ -d '{"secretKey":"'"${S3_ALT_SECRET_KEY}"'","status":"enabled","policy":"readwrite"}' \ - "http://127.0.0.1:9000/rustfs/admin/v3/add-user?accessKey=${S3_ALT_ACCESS_KEY}" + "http://127.0.0.1:${S3_PORT}/rustfs/admin/v3/add-user?accessKey=${S3_ALT_ACCESS_KEY}" # Explicitly attach built-in policy via policy mapping. # s3-tests relies on alt client being able to ListBuckets during setup cleanup. @@ -158,7 +161,7 @@ jobs: --access_key "${S3_ACCESS_KEY}" \ --secret_key "${S3_SECRET_KEY}" \ -X PUT \ - "http://127.0.0.1:9000/rustfs/admin/v3/set-user-or-group-policy?policyName=readwrite&userOrGroup=${S3_ALT_ACCESS_KEY}&isGroup=false" + "http://127.0.0.1:${S3_PORT}/rustfs/admin/v3/set-user-or-group-policy?policyName=readwrite&userOrGroup=${S3_ALT_ACCESS_KEY}&isGroup=false" # Sanity check: alt user can list buckets (should not be AccessDenied). awscurl \ @@ -167,7 +170,7 @@ jobs: --access_key "${S3_ALT_ACCESS_KEY}" \ --secret_key "${S3_ALT_SECRET_KEY}" \ -X GET \ - "http://127.0.0.1:9000/" >/dev/null + "http://127.0.0.1:${S3_PORT}/" >/dev/null - name: Prepare s3-tests run: | @@ -255,9 +258,14 @@ jobs: -t rustfs-ci \ -f Dockerfile.source . + - name: Pick S3 host port for load balancer + run: | + S3_PORT=$(python3 -c "import socket; s=socket.socket(); s.bind(('127.0.0.1',0)); print(s.getsockname()[1]); s.close()") + echo "S3_PORT=${S3_PORT}" >> "$GITHUB_ENV" + - name: Prepare cluster compose run: | - cat > compose.yml <<'EOF' + cat > compose.yml </dev/null 2>&1; then + if curl -sf "http://127.0.0.1:${S3_PORT}/health" >/dev/null 2>&1; then echo "Load balancer is ready" exit 0 fi @@ -359,6 +367,7 @@ jobs: - name: Generate s3tests config run: | export S3_HOST=127.0.0.1 + export S3_PORT="${S3_PORT}" envsubst < .github/s3tests/s3tests.conf > s3tests.conf - name: Provision s3-tests alt user (required by suite) @@ -371,7 +380,7 @@ jobs: -X PUT \ -H 'Content-Type: application/json' \ -d '{"secretKey":"'"${S3_ALT_SECRET_KEY}"'","status":"enabled","policy":"readwrite"}' \ - "http://127.0.0.1:9000/rustfs/admin/v3/add-user?accessKey=${S3_ALT_ACCESS_KEY}" + "http://127.0.0.1:${S3_PORT}/rustfs/admin/v3/add-user?accessKey=${S3_ALT_ACCESS_KEY}" awscurl \ --service s3 \ @@ -379,7 +388,7 @@ jobs: --access_key "${S3_ACCESS_KEY}" \ --secret_key "${S3_SECRET_KEY}" \ -X PUT \ - "http://127.0.0.1:9000/rustfs/admin/v3/set-user-or-group-policy?policyName=readwrite&userOrGroup=${S3_ALT_ACCESS_KEY}&isGroup=false" + "http://127.0.0.1:${S3_PORT}/rustfs/admin/v3/set-user-or-group-policy?policyName=readwrite&userOrGroup=${S3_ALT_ACCESS_KEY}&isGroup=false" awscurl \ --service s3 \ @@ -387,7 +396,7 @@ jobs: --access_key "${S3_ALT_ACCESS_KEY}" \ --secret_key "${S3_ALT_SECRET_KEY}" \ -X GET \ - "http://127.0.0.1:9000/" >/dev/null + "http://127.0.0.1:${S3_PORT}/" >/dev/null - name: Prepare s3-tests run: | diff --git a/.github/workflows/nix.yml b/.github/workflows/nix.yml index ac14ddbae5..d67c996f04 100644 --- a/.github/workflows/nix.yml +++ b/.github/workflows/nix.yml @@ -48,6 +48,9 @@ jobs: - name: Checkout repository uses: actions/checkout@v6 + - name: Configure Cargo / Nix build parallelism + uses: ./.github/actions/cargo-build-jobs + - name: Install Nix uses: cachix/install-nix-action@v31 with: @@ -65,12 +68,12 @@ jobs: run: | echo "Checking flake structure and evaluation..." nix flake show - nix flake check --print-build-logs + nix flake check --max-jobs "$CARGO_BUILD_JOBS" --print-build-logs - name: Build RustFS run: | echo "Building the default package..." - nix build .#default --print-build-logs + nix build .#default --max-jobs "$CARGO_BUILD_JOBS" --print-build-logs - name: Test Binary run: | diff --git a/build-rustfs.sh b/build-rustfs.sh index 0ed3352d6f..c5fa02520a 100755 --- a/build-rustfs.sh +++ b/build-rustfs.sh @@ -5,6 +5,20 @@ set -e +# Default rustc parallelism when not set (e.g. invoking this script without make). +ensure_cargo_build_jobs() { + if [[ -n "${CARGO_BUILD_JOBS:-}" ]]; then + return 0 + fi + local repo_root + repo_root="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" + local compute="${repo_root}/scripts/ci/compute-cargo-build-jobs.sh" + if [[ -f "$compute" ]]; then + CARGO_BUILD_JOBS="$(bash "$compute" --value-only 2>/dev/null || echo 2)" + export CARGO_BUILD_JOBS + fi +} + # Colors for output RED='\033[0;31m' GREEN='\033[0;32m' @@ -588,6 +602,8 @@ main() { exit 1 fi + ensure_cargo_build_jobs + # Override platform if specified if [ -n "$CUSTOM_PLATFORM" ]; then PLATFORM="$CUSTOM_PLATFORM" diff --git a/crates/e2e_test/src/common.rs b/crates/e2e_test/src/common.rs index aab8bf283e..f3b8dcca40 100644 --- a/crates/e2e_test/src/common.rs +++ b/crates/e2e_test/src/common.rs @@ -20,6 +20,15 @@ //! - AWS S3 client creation and configuration //! - Basic health checks and server readiness detection //! - Common test constants and utilities +//! +//! ## Shared self-hosted runners +//! +//! - `RUSTFS_E2E_EXTERNAL_ADDR`: `host:port` for tests that expect a **pre-started** RustFS (policy +//! suite, `scripts/run_e2e_tests.sh`). Default `127.0.0.1:9000`. +//! - `RUSTFS_E2E_EXTERNAL_URL`: optional full base URL (`http://host:port`) for reliant tests; +//! default is `http://` + [`external_rustfs_socket_addr`]. +//! - `RUSTFS_E2E_KILL_EXISTING`: set to `1`/`true`/`yes` to run `pkill` before spawning test +//! servers. **Off by default** so a long-lived RustFS on `:9000` is not killed. use aws_sdk_s3::config::{Credentials, Region}; use aws_sdk_s3::{Client, Config}; @@ -37,6 +46,23 @@ use uuid::Uuid; pub const DEFAULT_ACCESS_KEY: &str = "rustfsadmin"; pub const DEFAULT_SECRET_KEY: &str = "rustfsadmin"; pub const TEST_BUCKET: &str = "e2e-test-bucket"; + +/// `host:port` for tests that attach to a RustFS started outside the test process. +pub fn external_rustfs_socket_addr() -> String { + std::env::var("RUSTFS_E2E_EXTERNAL_ADDR").unwrap_or_else(|_| "127.0.0.1:9000".to_string()) +} + +/// HTTP endpoint for ignored reliant tests (external RustFS). +pub fn external_rustfs_http_url() -> String { + std::env::var("RUSTFS_E2E_EXTERNAL_URL").unwrap_or_else(|_| format!("http://{}", external_rustfs_socket_addr())) +} + +fn kill_existing_rustfs_processes_enabled() -> bool { + matches!( + std::env::var("RUSTFS_E2E_KILL_EXISTING").ok().as_deref(), + Some("1" | "true" | "TRUE" | "yes" | "YES") + ) +} pub fn workspace_root() -> PathBuf { let mut path = PathBuf::from(env!("CARGO_MANIFEST_DIR")); path.pop(); // e2e_test @@ -181,9 +207,14 @@ impl RustFSTestEnvironment { Ok(port) } - /// Kill any existing RustFS processes + /// Kill any existing RustFS processes (only if `RUSTFS_E2E_KILL_EXISTING` is set). pub async fn cleanup_existing_processes(&self) -> Result<(), Box> { - info!("Cleaning up any existing RustFS processes"); + if !kill_existing_rustfs_processes_enabled() { + info!("Skipping pkill of RustFS (set RUSTFS_E2E_KILL_EXISTING=1 to enable)"); + return Ok(()); + } + + info!("Cleaning up any existing RustFS processes (RUSTFS_E2E_KILL_EXISTING is set)"); let binary_path = rustfs_binary_path(); let binary_name = binary_path.to_string_lossy(); let output = Command::new("pkill").args(["-f", &binary_name]).output(); diff --git a/crates/e2e_test/src/policy/policy_variables_test.rs b/crates/e2e_test/src/policy/policy_variables_test.rs index 187f355c73..633b1ee9ae 100644 --- a/crates/e2e_test/src/policy/policy_variables_test.rs +++ b/crates/e2e_test/src/policy/policy_variables_test.rs @@ -14,7 +14,7 @@ //! Tests for AWS IAM policy variables with single-value, multi-value, and nested scenarios -use crate::common::{awscurl_put, init_logging}; +use crate::common::{awscurl_put, external_rustfs_socket_addr, init_logging}; use crate::policy::test_env::PolicyTestEnvironment; use aws_sdk_s3::primitives::ByteStream; use serial_test::serial; @@ -133,7 +133,7 @@ pub async fn test_aws_policy_variables_single_value_impl() -> Result<(), Box Result<(), Box Result<(), Box Result<(), Box Result<(), Box Result<(), Box env, Err(e) => { error!("Failed to create test environment: {}", e); diff --git a/crates/e2e_test/src/reliant/conditional_writes.rs b/crates/e2e_test/src/reliant/conditional_writes.rs index df870f0681..63cdd28ed9 100644 --- a/crates/e2e_test/src/reliant/conditional_writes.rs +++ b/crates/e2e_test/src/reliant/conditional_writes.rs @@ -1,5 +1,6 @@ #![cfg(test)] +use crate::common::external_rustfs_http_url; use aws_config::meta::region::RegionProviderChain; use aws_sdk_s3::Client; use aws_sdk_s3::config::{Credentials, Region}; @@ -8,8 +9,6 @@ use aws_sdk_s3::types::{CompletedMultipartUpload, CompletedPart}; use bytes::Bytes; use serial_test::serial; use std::error::Error; - -const ENDPOINT: &str = "http://localhost:9000"; const ACCESS_KEY: &str = "rustfsadmin"; const SECRET_KEY: &str = "rustfsadmin"; const BUCKET: &str = "api-test"; @@ -19,7 +18,7 @@ async fn create_aws_s3_client() -> Result> { let shared_config = aws_config::defaults(aws_config::BehaviorVersion::latest()) .region(region_provider) .credentials_provider(Credentials::new(ACCESS_KEY, SECRET_KEY, None, None, "static")) - .endpoint_url(ENDPOINT) + .endpoint_url(external_rustfs_http_url()) .load() .await; diff --git a/crates/e2e_test/src/reliant/get_deleted_object_test.rs b/crates/e2e_test/src/reliant/get_deleted_object_test.rs index b34159ec17..a8062e3823 100644 --- a/crates/e2e_test/src/reliant/get_deleted_object_test.rs +++ b/crates/e2e_test/src/reliant/get_deleted_object_test.rs @@ -19,6 +19,7 @@ #![cfg(test)] +use crate::common::external_rustfs_http_url; use aws_config::meta::region::RegionProviderChain; use aws_sdk_s3::Client; use aws_sdk_s3::config::{Credentials, Region}; @@ -27,8 +28,6 @@ use bytes::Bytes; use serial_test::serial; use std::error::Error; use tracing::info; - -const ENDPOINT: &str = "http://localhost:9000"; const ACCESS_KEY: &str = "rustfsadmin"; const SECRET_KEY: &str = "rustfsadmin"; const BUCKET: &str = "test-get-deleted-bucket"; @@ -38,7 +37,7 @@ async fn create_aws_s3_client() -> Result> { let shared_config = aws_config::defaults(aws_config::BehaviorVersion::latest()) .region(region_provider) .credentials_provider(Credentials::new(ACCESS_KEY, SECRET_KEY, None, None, "static")) - .endpoint_url(ENDPOINT) + .endpoint_url(external_rustfs_http_url()) .load() .await; diff --git a/crates/e2e_test/src/reliant/head_deleted_object_versioning_test.rs b/crates/e2e_test/src/reliant/head_deleted_object_versioning_test.rs index a4d471754b..6381ac3a51 100644 --- a/crates/e2e_test/src/reliant/head_deleted_object_versioning_test.rs +++ b/crates/e2e_test/src/reliant/head_deleted_object_versioning_test.rs @@ -19,6 +19,7 @@ #![cfg(test)] +use crate::common::external_rustfs_http_url; use aws_config::meta::region::RegionProviderChain; use aws_sdk_s3::Client; use aws_sdk_s3::config::{Credentials, Region}; @@ -28,8 +29,6 @@ use bytes::Bytes; use serial_test::serial; use std::error::Error; use tracing::info; - -const ENDPOINT: &str = "http://localhost:9000"; const ACCESS_KEY: &str = "rustfsadmin"; const SECRET_KEY: &str = "rustfsadmin"; const BUCKET: &str = "test-head-deleted-versioning-bucket"; @@ -39,7 +38,7 @@ async fn create_aws_s3_client() -> Result> { let shared_config = aws_config::defaults(aws_config::BehaviorVersion::latest()) .region(region_provider) .credentials_provider(Credentials::new(ACCESS_KEY, SECRET_KEY, None, None, "static")) - .endpoint_url(ENDPOINT) + .endpoint_url(external_rustfs_http_url()) .load() .await; diff --git a/crates/e2e_test/src/reliant/lifecycle.rs b/crates/e2e_test/src/reliant/lifecycle.rs index 2302eb3064..6c987856f7 100644 --- a/crates/e2e_test/src/reliant/lifecycle.rs +++ b/crates/e2e_test/src/reliant/lifecycle.rs @@ -13,6 +13,7 @@ // See the License for the specific language governing permissions and // limitations under the License. +use crate::common::external_rustfs_http_url; use aws_config::meta::region::RegionProviderChain; use aws_sdk_s3::Client; use aws_sdk_s3::config::{Credentials, Region}; @@ -20,7 +21,6 @@ use bytes::Bytes; use serial_test::serial; use std::error::Error; -const ENDPOINT: &str = "http://localhost:9000"; const ACCESS_KEY: &str = "rustfsadmin"; const SECRET_KEY: &str = "rustfsadmin"; const BUCKET: &str = "test-basic-bucket"; @@ -30,7 +30,7 @@ async fn create_aws_s3_client() -> Result> { let shared_config = aws_config::defaults(aws_config::BehaviorVersion::latest()) .region(region_provider) .credentials_provider(Credentials::new(ACCESS_KEY, SECRET_KEY, None, None, "static")) - .endpoint_url(ENDPOINT) + .endpoint_url(external_rustfs_http_url()) .load() .await; diff --git a/crates/e2e_test/src/reliant/node_interact_test.rs b/crates/e2e_test/src/reliant/node_interact_test.rs index a25f1db698..2745de6e28 100644 --- a/crates/e2e_test/src/reliant/node_interact_test.rs +++ b/crates/e2e_test/src/reliant/node_interact_test.rs @@ -13,7 +13,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -use crate::common::workspace_root; +use crate::common::{external_rustfs_http_url, workspace_root}; use futures::future::join_all; use rmp_serde::{Deserializer, Serializer}; use rustfs_ecstore::disk::{VolumeInfo, WalkDirOptions}; @@ -34,8 +34,6 @@ use tokio::spawn; use tonic::Request; use tonic::codegen::tokio_stream::StreamExt; -const CLUSTER_ADDR: &str = "http://localhost:9000"; - #[tokio::test] #[ignore = "requires running RustFS server at localhost:9000"] async fn ping() -> Result<(), Box> { @@ -52,10 +50,10 @@ async fn ping() -> Result<(), Box> { let decoded_payload = flatbuffers::root::(finished_data); assert!(decoded_payload.is_ok()); + let cluster_addr = external_rustfs_http_url(); // Create client let mut client = - node_service_time_out_client(&CLUSTER_ADDR.to_string(), TonicInterceptor::Signature(gen_tonic_signature_interceptor())) - .await?; + node_service_time_out_client(&cluster_addr, TonicInterceptor::Signature(gen_tonic_signature_interceptor())).await?; // Construct PingRequest let request = Request::new(PingRequest { @@ -80,9 +78,9 @@ async fn ping() -> Result<(), Box> { #[tokio::test] #[ignore = "requires running RustFS server at localhost:9000"] async fn make_volume() -> Result<(), Box> { + let cluster_addr = external_rustfs_http_url(); let mut client = - node_service_time_out_client(&CLUSTER_ADDR.to_string(), TonicInterceptor::Signature(gen_tonic_signature_interceptor())) - .await?; + node_service_time_out_client(&cluster_addr, TonicInterceptor::Signature(gen_tonic_signature_interceptor())).await?; let request = Request::new(MakeVolumeRequest { disk: "data".to_string(), volume: "dandan".to_string(), @@ -100,9 +98,9 @@ async fn make_volume() -> Result<(), Box> { #[tokio::test] #[ignore = "requires running RustFS server at localhost:9000"] async fn list_volumes() -> Result<(), Box> { + let cluster_addr = external_rustfs_http_url(); let mut client = - node_service_time_out_client(&CLUSTER_ADDR.to_string(), TonicInterceptor::Signature(gen_tonic_signature_interceptor())) - .await?; + node_service_time_out_client(&cluster_addr, TonicInterceptor::Signature(gen_tonic_signature_interceptor())).await?; let request = Request::new(ListVolumesRequest { disk: "data".to_string(), }); @@ -132,9 +130,9 @@ async fn walk_dir() -> Result<(), Box> { let (rd, mut wr) = tokio::io::duplex(1024); let mut buf = Vec::new(); opts.serialize(&mut Serializer::new(&mut buf))?; + let cluster_addr = external_rustfs_http_url(); let mut client = - node_service_time_out_client(&CLUSTER_ADDR.to_string(), TonicInterceptor::Signature(gen_tonic_signature_interceptor())) - .await?; + node_service_time_out_client(&cluster_addr, TonicInterceptor::Signature(gen_tonic_signature_interceptor())).await?; let disk_path = std::env::var_os("RUSTFS_DISK_PATH").map(PathBuf::from).unwrap_or_else(|| { let mut path = workspace_root(); path.push("target"); @@ -187,9 +185,9 @@ async fn walk_dir() -> Result<(), Box> { #[tokio::test] #[ignore = "requires running RustFS server at localhost:9000"] async fn read_all() -> Result<(), Box> { + let cluster_addr = external_rustfs_http_url(); let mut client = - node_service_time_out_client(&CLUSTER_ADDR.to_string(), TonicInterceptor::Signature(gen_tonic_signature_interceptor())) - .await?; + node_service_time_out_client(&cluster_addr, TonicInterceptor::Signature(gen_tonic_signature_interceptor())).await?; let request = Request::new(ReadAllRequest { disk: "data".to_string(), volume: "ff".to_string(), @@ -207,9 +205,9 @@ async fn read_all() -> Result<(), Box> { #[tokio::test] #[ignore = "requires running RustFS server at localhost:9000"] async fn storage_info() -> Result<(), Box> { + let cluster_addr = external_rustfs_http_url(); let mut client = - node_service_time_out_client(&CLUSTER_ADDR.to_string(), TonicInterceptor::Signature(gen_tonic_signature_interceptor())) - .await?; + node_service_time_out_client(&cluster_addr, TonicInterceptor::Signature(gen_tonic_signature_interceptor())).await?; let request = Request::new(LocalStorageInfoRequest { metrics: true }); let response = client.local_storage_info(request).await?.into_inner(); diff --git a/crates/e2e_test/src/reliant/sql.rs b/crates/e2e_test/src/reliant/sql.rs index 4ca02d6828..7497b8caee 100644 --- a/crates/e2e_test/src/reliant/sql.rs +++ b/crates/e2e_test/src/reliant/sql.rs @@ -13,6 +13,7 @@ // See the License for the specific language governing permissions and // limitations under the License. +use crate::common::external_rustfs_http_url; use aws_config::meta::region::RegionProviderChain; use aws_sdk_s3::Client; use aws_sdk_s3::config::{Credentials, Region}; @@ -23,7 +24,6 @@ use bytes::Bytes; use serial_test::serial; use std::error::Error; -const ENDPOINT: &str = "http://localhost:9000"; const ACCESS_KEY: &str = "rustfsadmin"; const SECRET_KEY: &str = "rustfsadmin"; const BUCKET: &str = "test-sql-bucket"; @@ -35,7 +35,7 @@ async fn create_aws_s3_client() -> Result> { let shared_config = aws_config::defaults(aws_config::BehaviorVersion::latest()) .region(region_provider) .credentials_provider(Credentials::new(ACCESS_KEY, SECRET_KEY, None, None, "static")) - .endpoint_url(ENDPOINT) + .endpoint_url(external_rustfs_http_url()) .load() .await; diff --git a/scripts/ci/compute-cargo-build-jobs.sh b/scripts/ci/compute-cargo-build-jobs.sh new file mode 100755 index 0000000000..b0f3c6f4ba --- /dev/null +++ b/scripts/ci/compute-cargo-build-jobs.sh @@ -0,0 +1,100 @@ +#!/usr/bin/env bash +# Set CARGO_BUILD_JOBS from CPU count and memory: up to half of logical cores, +# capped when RAM per concurrent rustc job would be too low. +# +# Usage: +# compute-cargo-build-jobs.sh # log to stderr; append to GITHUB_ENV if set +# compute-cargo-build-jobs.sh --value-only # print job count on stdout only (for Make) +# +# Override knobs (optional): +# CARGO_JOBS_RESERVE_MB — MiB reserved for OS / link peaks (default 4096) +# CARGO_JOBS_PER_RUSTC_MB — budget MiB per parallel rustc job (default 2048) + +set -euo pipefail + +VALUE_ONLY=0 +if [[ "${1:-}" == "--value-only" ]]; then + VALUE_ONLY=1 +fi + +readonly RESERVE_MB="${CARGO_JOBS_RESERVE_MB:-4096}" +readonly PER_JOB_MB="${CARGO_JOBS_PER_RUSTC_MB:-2048}" + +detect_resources() { + local os + os="$(uname -s 2>/dev/null || echo Unknown)" + + case "$os" in + Linux) + cores="$(nproc 2>/dev/null || echo 2)" + if [[ -r /proc/meminfo ]]; then + local mem_kb + mem_kb="$(grep -E '^MemAvailable:' /proc/meminfo 2>/dev/null | awk '{print $2}' || true)" + if [[ -z "${mem_kb:-}" || "${mem_kb:-0}" -eq 0 ]]; then + mem_kb="$(grep -E '^MemTotal:' /proc/meminfo 2>/dev/null | awk '{print $2}' || echo 8388608)" + fi + mem_mb=$((mem_kb / 1024)) + else + mem_mb=8192 + fi + ;; + Darwin) + cores="$(sysctl -n hw.ncpu 2>/dev/null || echo 2)" + local mem_bytes + mem_bytes="$(sysctl -n hw.memsize 2>/dev/null || echo 8589934592)" + mem_mb=$((mem_bytes / 1024 / 1024)) + # No MemAvailable equivalent; assume a large fraction is usable on CI runners. + mem_mb=$((mem_mb * 8 / 10)) + ;; + MINGW* | MSYS* | CYGWIN* | Windows_NT) + cores="${NUMBER_OF_PROCESSORS:-2}" + if command -v powershell.exe >/dev/null 2>&1; then + # FreePhysicalMemory is KiB per WMI. + local free_kb + free_kb="$(powershell.exe -NoProfile -Command \ + "[int]((Get-CimInstance Win32_OperatingSystem).FreePhysicalMemory)" 2>/dev/null || echo 8388608)" + mem_mb=$((free_kb / 1024)) + else + mem_mb=8192 + fi + ;; + *) + cores=2 + mem_mb=8192 + ;; + esac +} + +detect_resources + +half=$((cores / 2)) +[[ "$half" -lt 1 ]] && half=1 + +if ((mem_mb > RESERVE_MB)); then + usable_mb=$((mem_mb - RESERVE_MB)) +else + usable_mb=$((mem_mb * 3 / 4)) +fi +[[ "$usable_mb" -lt 256 ]] && usable_mb=256 + +max_by_mem=$((usable_mb / PER_JOB_MB)) +[[ "$max_by_mem" -lt 1 ]] && max_by_mem=1 + +if [[ "$half" -le "$max_by_mem" ]]; then + jobs=$half + detail="half of ${cores} cores (memory sufficient)" +else + jobs=$max_by_mem + detail="capped by memory (~${mem_mb} MiB reported, ~${usable_mb} MiB budget, ${PER_JOB_MB} MiB/job)" +fi + +if [[ "$VALUE_ONLY" -eq 1 ]]; then + printf '%s\n' "$jobs" +else + echo "Cargo parallelism: CARGO_BUILD_JOBS=${jobs} (${detail})" >&2 + if [[ -n "${GITHUB_ENV:-}" ]]; then + { + echo "CARGO_BUILD_JOBS=${jobs}" + } >>"$GITHUB_ENV" + fi +fi diff --git a/scripts/e2e-run.sh b/scripts/e2e-run.sh index b518c59834..8efbc9c692 100755 --- a/scripts/e2e-run.sh +++ b/scripts/e2e-run.sh @@ -17,21 +17,55 @@ set -ex BIN=$1 VOLUME=$2 -chmod +x $BIN -mkdir -p $VOLUME +chmod +x "$BIN" +mkdir -p "$VOLUME" + +# Avoid shared runners where another RustFS already listens on :9000. +pick_port() { + if [[ -n "${E2E_RUSTFS_PORT:-}" ]]; then + echo "$E2E_RUSTFS_PORT" + return + fi + if command -v python3 >/dev/null 2>&1; then + python3 -c "import socket; s=socket.socket(); s.bind(('127.0.0.1',0)); print(s.getsockname()[1]); s.close()" + else + echo "19080" + fi +} + +PORT="$(pick_port)" +export RUSTFS_E2E_EXTERNAL_ADDR="127.0.0.1:${PORT}" export RUST_LOG="rustfs=debug,ecstore=debug,s3s=debug,iam=debug" export RUST_BACKTRACE=full -$BIN $VOLUME > /tmp/rustfs.log 2>&1 & +"$BIN" \ + --address "127.0.0.1:${PORT}" \ + --access-key rustfsadmin \ + --secret-key rustfsadmin \ + "$VOLUME" >/tmp/rustfs.log 2>&1 & +RUSTFS_PID=$! + +cleanup_rustfs() { + kill "${RUSTFS_PID}" 2>/dev/null || true + wait "${RUSTFS_PID}" 2>/dev/null || true +} +trap cleanup_rustfs EXIT -sleep 10 +# Wait for listener (fixed sleep is unreliable on slow hosts) +for _ in $(seq 1 30); do + if nc -z 127.0.0.1 "${PORT}" 2>/dev/null || timeout 1 bash -c "cat < /dev/null > /dev/tcp/127.0.0.1/${PORT}" 2>/dev/null; then + break + fi + sleep 1 +done export AWS_ACCESS_KEY_ID=rustfsadmin export AWS_SECRET_ACCESS_KEY=rustfsadmin export AWS_REGION=us-east-1 -export AWS_ENDPOINT_URL=http://localhost:9000 +export AWS_ENDPOINT_URL="http://127.0.0.1:${PORT}" export RUST_LOG="s3s_e2e=debug,s3s_test=info,s3s=debug" export RUST_BACKTRACE=full s3s-e2e -killall $BIN +trap - EXIT +cleanup_rustfs diff --git a/scripts/install-github-runner.sh b/scripts/install-github-runner.sh new file mode 100755 index 0000000000..c14d21f775 --- /dev/null +++ b/scripts/install-github-runner.sh @@ -0,0 +1,83 @@ +#!/usr/bin/env bash +# Install and configure a GitHub Actions self-hosted runner on Ubuntu. +# Repo: https://github.com/wasabi/rustfs +# Prompts for runner token and runner name; adds labels 'ubicloud-standard-2' and 'ubicloud-standard-4 '; uses default group. + +set -e + +RUNNER_VERSION="2.332.0" +RUNNER_TAR="actions-runner-linux-x64-${RUNNER_VERSION}.tar.gz" +RUNNER_SHA256="f2094522a6b9afeab07ffb586d1eb3f190b6457074282796c497ce7dce9e0f2a" +REPO_URL="https://github.com/wasabi/rustfs" +RUNNER_LABELS="ubicloud-standard-2,ubicloud-standard-4" +INSTALL_DIR="${INSTALL_DIR:-$HOME/actions-runner}" + +echo "GitHub self-hosted runner installer for ${REPO_URL}" +echo "Runner version: ${RUNNER_VERSION}" +echo "Install directory: ${INSTALL_DIR}" +echo "" + +# System packages and pipx CLI tools used by CI workflows (incl. native deps for +# cargo install of s3s-e2e / aws-lc-sys on self-hosted runners). +echo "Installing dependencies (apt: unzip, pipx, build toolchain, cmake)..." +export DEBIAN_FRONTEND=noninteractive +sudo apt-get update -qq +sudo apt-get install -y unzip pipx build-essential cmake pkg-config +echo "Installing awscurl via pipx..." +pipx install awscurl +pipx ensurepath + +# Prompt for token +read -r -p "Runner token: " RUNNER_TOKEN +if [ -z "$RUNNER_TOKEN" ]; then + echo "Error: Runner token is required." + exit 1 +fi + +# Prompt for runner name +read -r -p "Runner name: " RUNNER_NAME +if [ -z "$RUNNER_NAME" ]; then + echo "Error: Runner name is required." + exit 1 +fi + +# Create install directory +mkdir -p "$INSTALL_DIR" +cd "$INSTALL_DIR" + +# Download +echo "Downloading runner package..." +curl -o "$RUNNER_TAR" -L "https://github.com/actions/runner/releases/download/v${RUNNER_VERSION}/${RUNNER_TAR}" + +# Validate hash +echo "Validating checksum..." +echo "${RUNNER_SHA256} ${RUNNER_TAR}" | shasum -a 256 -c + +# Extract +echo "Extracting..." +tar xzf "$RUNNER_TAR" +rm -f "$RUNNER_TAR" + +# Configure: default group (omit --runnergroup), labels x,y +echo "Configuring runner..." +./config.sh \ + --url "$REPO_URL" \ + --token "$RUNNER_TOKEN" \ + --name "$RUNNER_NAME" \ + --labels "$RUNNER_LABELS" \ + --unattended + +# Install and start service (runs in background, survives reboot) +echo "Installing runner as a systemd service..." +sudo ./svc.sh install +sudo ./svc.sh start + +echo "" +echo "Runner installed and running. To use in workflows:" +echo " runs-on: self-hosted" +echo "" +echo "Useful commands (from ${INSTALL_DIR}):" +echo " sudo ./svc.sh status # check status" +echo " sudo ./svc.sh stop # stop" +echo " sudo ./svc.sh start # start" +echo " sudo ./svc.sh uninstall # remove service (then delete directory to fully remove)" diff --git a/scripts/run_e2e_tests.sh b/scripts/run_e2e_tests.sh index 754782f158..1f38746aac 100755 --- a/scripts/run_e2e_tests.sh +++ b/scripts/run_e2e_tests.sh @@ -21,6 +21,20 @@ RUSTFS_PID="" TEST_FILTER="" TEST_TYPE="all" +# Free TCP port for this script's RustFS (avoids clashing with another instance on :9000). +pick_rustfs_http_port() { + if [ -n "${RUSTFS_E2E_HTTP_PORT:-}" ]; then + echo "$RUSTFS_E2E_HTTP_PORT" + return + fi + if command -v python3 >/dev/null 2>&1; then + python3 -c "import socket; s=socket.socket(); s.bind(('127.0.0.1',0)); print(s.getsockname()[1]); s.close()" + else + echo "[WARNING] python3 not found; using port 19001 (set RUSTFS_E2E_HTTP_PORT to pick explicitly)" >&2 + echo "19001" + fi +} + # Function to print colored output print_info() { echo -e "${BLUE}[INFO]${NC} $1" @@ -48,7 +62,11 @@ Options: -t, --test Run specific test(s) matching pattern -f, --file Run all tests in specific file (e.g., sql, basic) -a, --all Run all e2e tests (default) - + +Environment: + RUSTFS_E2E_HTTP_PORT Listen port for RustFS (default: ephemeral via python3, else 19001) + RUSTFS_E2E_EXTERNAL_ADDR host:port for policy / external tests (set automatically from the port above) + Examples: $0 # Run all e2e tests $0 -t test_select_object_content_csv_basic # Run specific test @@ -136,7 +154,7 @@ start_rustfs() { cd "$TARGET_DIR" RUSTFS_ACCESS_KEY=rustfsadmin RUSTFS_SECRET_KEY=rustfsadmin \ RUSTFS_OBS_LOG_DIRECTORY="$TARGET_DIR/logs" \ - ./rustfs --address :9000 "$DATA_DIR" > rustfs.log 2>&1 & + ./rustfs --address ":${RUSTFS_HTTP_PORT}" "$DATA_DIR" > rustfs.log 2>&1 & RUSTFS_PID=$! print_info "RustFS started with PID: $RUSTFS_PID" @@ -159,21 +177,21 @@ start_rustfs() { fi # Try simple HTTP connection first (most reliable) - if curl -s --noproxy localhost --connect-timeout 2 --max-time 3 "http://localhost:9000/" >/dev/null 2>&1; then + if curl -s --noproxy localhost --connect-timeout 2 --max-time 3 "http://localhost:${RUSTFS_HTTP_PORT}/" >/dev/null 2>&1; then print_success "RustFS is ready!" return 0 fi # Try health endpoint if available - if curl -s --noproxy localhost --connect-timeout 2 --max-time 3 "http://localhost:9000/health" >/dev/null 2>&1; then + if curl -s --noproxy localhost --connect-timeout 2 --max-time 3 "http://localhost:${RUSTFS_HTTP_PORT}/health" >/dev/null 2>&1; then print_success "RustFS is ready!" return 0 fi # Try port connectivity check (faster than HTTP) - if nc -z localhost 9000 2>/dev/null; then - print_info "Port 9000 is open, verifying HTTP response..." - if curl -s --noproxy localhost --connect-timeout 1 --max-time 2 "http://localhost:9000/" >/dev/null 2>&1; then + if nc -z localhost "${RUSTFS_HTTP_PORT}" 2>/dev/null; then + print_info "Port ${RUSTFS_HTTP_PORT} is open, verifying HTTP response..." + if curl -s --noproxy localhost --connect-timeout 1 --max-time 2 "http://localhost:${RUSTFS_HTTP_PORT}/" >/dev/null 2>&1; then print_success "RustFS is ready!" return 0 fi @@ -193,12 +211,12 @@ start_rustfs() { # Quick final attempts with shorter timeouts for i in 1 2 3; do - if curl -s --noproxy localhost --connect-timeout 1 --max-time 2 "http://localhost:9000/" >/dev/null 2>&1; then + if curl -s --noproxy localhost --connect-timeout 1 --max-time 2 "http://localhost:${RUSTFS_HTTP_PORT}/" >/dev/null 2>&1; then print_success "RustFS is now ready!" return 0 fi - if nc -z localhost 9000 2>/dev/null; then - print_info "Port 9000 is accessible, continuing with tests..." + if nc -z localhost "${RUSTFS_HTTP_PORT}" 2>/dev/null; then + print_info "Port ${RUSTFS_HTTP_PORT} is accessible, continuing with tests..." return 0 fi sleep 1 @@ -288,6 +306,11 @@ main() { # Build RustFS build_rustfs + + RUSTFS_HTTP_PORT="$(pick_rustfs_http_port)" + export RUSTFS_HTTP_PORT + export RUSTFS_E2E_EXTERNAL_ADDR="127.0.0.1:${RUSTFS_HTTP_PORT}" + print_info "RustFS test port: ${RUSTFS_HTTP_PORT} (RUSTFS_E2E_EXTERNAL_ADDR=${RUSTFS_E2E_EXTERNAL_ADDR})" # Start RustFS if ! start_rustfs; then diff --git a/scripts/s3-tests/run.sh b/scripts/s3-tests/run.sh index ced0ecbb62..728a1d1c1b 100755 --- a/scripts/s3-tests/run.sh +++ b/scripts/s3-tests/run.sh @@ -708,7 +708,7 @@ log_info "Using template: ${TEMPLATE_PATH}" log_info "Generating config: ${CONF_OUTPUT_PATH}" # Export all required variables for envsubst -export S3_HOST S3_ACCESS_KEY S3_SECRET_KEY S3_ALT_ACCESS_KEY S3_ALT_SECRET_KEY +export S3_HOST S3_PORT S3_ACCESS_KEY S3_SECRET_KEY S3_ALT_ACCESS_KEY S3_ALT_SECRET_KEY envsubst < "${TEMPLATE_PATH}" > "${CONF_OUTPUT_PATH}" || { log_error "Failed to generate s3tests config" exit 1