diff --git a/.github/workflows/e2e-tests.yml b/.github/workflows/e2e-tests.yml index 94b8ad0..fed154f 100644 --- a/.github/workflows/e2e-tests.yml +++ b/.github/workflows/e2e-tests.yml @@ -69,7 +69,7 @@ jobs: run: docker compose -f docker-compose.prod.yml build - name: Create network - run: docker network create process_api_net + run: docker network create sepex_net - name: Run the docker compose stack run: docker compose -f docker-compose.prod.yml up -d @@ -77,7 +77,7 @@ jobs: - name: Create bucket run: > docker run - --network process_api_net + --network sepex_net -e MINIO_ROOT_USER=user -e MINIO_ROOT_PASSWORD=password -e STORAGE_BUCKET=sepex-storage diff --git a/README.md b/README.md index 8505120..87d5bd2 100644 --- a/README.md +++ b/README.md @@ -24,7 +24,7 @@ https://developer.ogc.org/api/processes/index.html ### Linux using Docker -1. Create docker network `docker network create process_api_net` +1. Create docker network `docker network create sepex_net` 1. Build docker images for example plugins ```sh diff --git a/api/controllers/docker.go b/api/controllers/docker.go index 1267144..b794caf 100644 --- a/api/controllers/docker.go +++ b/api/controllers/docker.go @@ -17,7 +17,7 @@ import ( "github.com/labstack/gommon/log" ) -const DOCKER_NETWORK = "process_api_net" +const DOCKER_NETWORK = "sepex_net" type ContainerInfo struct { Exists bool diff --git a/docker-compose.yml b/docker-compose.base.yml similarity index 86% rename from docker-compose.yml rename to docker-compose.base.yml index 853afa2..7ed614e 100644 --- a/docker-compose.yml +++ b/docker-compose.base.yml @@ -3,17 +3,13 @@ services: container_name: api build: context: ./api - target: dev - ports: - - '5050:5050' env_file: - .env volumes: - - ./api:/app - - ./.data/api:/.data - /var/run/docker.sock:/var/run/docker.sock + - ./.data/api:/.data networks: - - process_api_net + - sepex_net depends_on: minio: condition: service_started @@ -32,25 +28,25 @@ services: volumes: - ./.data/minio:/data networks: - - process_api_net + - sepex_net postgres: container_name: postgres image: postgres:17.2-alpine3.20 env_file: - .env - ports: - - '5432:5432' healthcheck: test: ['CMD-SHELL', 'pg_isready -U $$POSTGRES_USER -d $$POSTGRES_DB'] interval: 10s timeout: 5s retries: 5 + ports: + - '5432:5432' volumes: - ./.data/postgres:/var/lib/postgresql/data networks: - - process_api_net + - sepex_net networks: - process_api_net: + sepex_net: external: true diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml new file mode 100644 index 0000000..264d3d2 --- /dev/null +++ b/docker-compose.dev.yml @@ -0,0 +1,11 @@ +include: + - docker-compose.base.yml + +services: + api: + build: + target: dev + ports: + - '5050:5050' + volumes: + - ./api:/app diff --git a/docker-compose.prod.yml b/docker-compose.prod.yml index b8e60cd..c879f63 100644 --- a/docker-compose.prod.yml +++ b/docker-compose.prod.yml @@ -1,55 +1,9 @@ +include: + - docker-compose.base.yml + services: api: - container_name: api - build: - context: ./api ports: - '80:5050' - env_file: - - .env volumes: - ./api/plugins:/app/plugins - - ./.data/api:/.data - - /var/run/docker.sock:/var/run/docker.sock - networks: - - process_api_net - depends_on: - minio: - condition: service_started - postgres: - condition: service_healthy - - minio: - container_name: minio - image: minio/minio:RELEASE.2024-11-07T00-52-20Z.fips - command: server /data --console-address ":9001" - env_file: - - .env - ports: - - '9000:9000' - - '9001:9001' - volumes: - - ./.data/minio:/data - networks: - - process_api_net - - postgres: - container_name: postgres - image: postgres:17.2-alpine3.20 - healthcheck: - test: ['CMD-SHELL', 'pg_isready -U $$POSTGRES_USER -d $$POSTGRES_DB'] - interval: 10s - timeout: 5s - retries: 5 - env_file: - - .env - ports: - - '5432:5432' - volumes: - - ./.data/postgres:/var/lib/postgresql/data - networks: - - process_api_net - -networks: - process_api_net: - external: true diff --git a/run_e2e_local.sh b/run_e2e_local.sh deleted file mode 100755 index 82675a0..0000000 --- a/run_e2e_local.sh +++ /dev/null @@ -1,101 +0,0 @@ -#!/usr/bin/env bash -set -euo pipefail - -# --- Create env file (same as GH Actions) --- -cat > .env <<'EOF' -API_NAME='github-testing-sepex' -STORAGE_SERVICE='minio' -STORAGE_BUCKET='sepex-storage' -STORAGE_METADATA_PREFIX='metadata' -STORAGE_RESULTS_PREFIX='results' -STORAGE_LOGS_PREFIX='logs' - -PLUGINS_LOAD_DIR='plugins' -PLUGINS_DIR='/.data/plugins' -TMP_JOB_LOGS_DIR='/.data/tmp/logs' - -MAX_LOCAL_CPUS=4 -MAX_LOCAL_MEMORY=4096 - -MINIO_ACCESS_KEY_ID=user -MINIO_SECRET_ACCESS_KEY=password -MINIO_S3_ENDPOINT=http://minio:9000 -MINIO_S3_REGION='us-east-1' -MINIO_ROOT_USER=user -MINIO_ROOT_PASSWORD=password - -DB_SERVICE='postgres' -POSTGRES_CONN_STRING='postgres://user:password@postgres:5432/db?sslmode=disable' -POSTGRES_PASSWORD='password' -POSTGRES_USER='user' -POSTGRES_DB='db' -PG_LOG_CHECKPOINTS='off' - -# Optional (only needed if your tests hit aws-batch paths) -# AWS_ACCESS_KEY_ID=... -# AWS_SECRET_ACCESS_KEY=... -# AWS_REGION=... - -BATCH_LOG_STREAM_GROUP='/aws/batch/job' - -PYWRITE_MINIO_ACCESS_KEY_ID='user' -PYWRITE_MINIO_SECRET_ACCESS_KEY='password' -PYWRITE_MINIO_S3_ENDPOINT='http://minio:9000' -PYWRITE_MINIO_S3_REGION='us-east-1' -PYWRITE_MINIO_S3_BUCKET='sepex-storage' -EOF - -# --- Build plugin example images (same as GH Actions) --- -( cd plugin-examples && chmod +x build.sh && ./build.sh ) & - -# --- Build compose stack --- -docker compose -f docker-compose.prod.yml build - -# --- Network (ignore error if already exists) --- -docker network create process_api_net >/dev/null 2>&1 || true - -# --- Run stack --- -docker compose -f docker-compose.prod.yml up -d - -# --- Create bucket in minio --- -docker run \ - --network process_api_net \ - -e MINIO_ROOT_USER=user \ - -e MINIO_ROOT_PASSWORD=password \ - -e STORAGE_BUCKET=sepex-storage \ - --entrypoint /bin/sh \ - minio/mc:RELEASE.2023-08-18T21-57-55Z \ - -c "mc alias set myminio http://minio:9000 \$MINIO_ROOT_USER \$MINIO_ROOT_PASSWORD && mc mb -p myminio/\${STORAGE_BUCKET} || true" - -# --- Wait for API --- -attempts=0 -max_attempts=12 -until curl -fsS http://localhost:80 >/dev/null 2>&1; do - attempts=$((attempts+1)) - if [ "$attempts" -ge "$max_attempts" ]; then - echo "API not ready after $max_attempts attempts" - docker compose logs - exit 1 - fi - echo "Waiting for API server... attempt $attempts" - sleep 10 -done -echo "API server is ready!" - -# --- Run newman tests (use host networking like GH Actions) --- -docker run --network="host" \ - -v "$PWD/tests/e2e:/etc/newman" \ - postman/newman:5.3.1-alpine \ - run tests.postman_collection.json \ - --env-var "url=localhost:80" \ - --reporters cli \ - --bail \ - --color on - -# --- Logs (helpful locally too) --- -echo "---- docker compose logs ----" -docker compose logs || true -if [ -f .data/api/logs/api.jsonl ]; then - echo "---- .data/api/logs/api.jsonl ----" - cat .data/api/logs/api.jsonl || true -fi diff --git a/run_e2e_tests_local.sh b/run_e2e_tests_local.sh new file mode 100755 index 0000000..35cea5a --- /dev/null +++ b/run_e2e_tests_local.sh @@ -0,0 +1,89 @@ +#!/usr/bin/env bash +set -euo pipefail + +COMPOSE="docker compose -f docker-compose.prod.yml" + +cleanup() { + echo "---- Tearing down ----" + $COMPOSE down --volumes --remove-orphans 2>/dev/null || true + if [ -d ".data" ]; then +# cleaning up .data/ with docker to avoid permission issues + docker run --rm -v "$PWD/.data:/cleanup" alpine sh -c "rm -rf /cleanup/*" + fi +} +if [ "${1:-}" != "--no-cleanup" ]; then + trap cleanup EXIT +fi + +# --- Check .env exists --- +if [ ! -f ".env" ]; then + echo "ERROR: .env file not found. Copy example.env to .env and configure it." + exit 1 +fi + +# --- Check if .data exists and is non-empty --- +if [ -d ".data" ] && [ -n "$(ls -A .data 2>/dev/null)" ]; then + echo "WARNING: .data/ directory is not empty. E2E tests will overwrite its contents." + read -rp "Delete .data/ and continue? [y/N] " answer + if [[ "$answer" =~ ^[Yy]$ ]]; then + docker run --rm -v "$PWD/.data:/cleanup" alpine sh -c "rm -rf /cleanup/*" + else + echo "Aborted." + exit 1 + fi +fi + +# --- Build plugin example images (same as GH Actions) --- +( cd plugin-examples && chmod +x build.sh && ./build.sh ) & + +# --- Build compose stack --- +$COMPOSE build + +# --- Network (ignore error if already exists) --- +docker network create sepex_net >/dev/null 2>&1 || true + +# --- Run stack --- +$COMPOSE up -d + +# --- Create bucket in minio --- +docker run --rm \ + --network sepex_net \ + -e MINIO_ROOT_USER=user \ + -e MINIO_ROOT_PASSWORD=password \ + -e STORAGE_BUCKET=sepex-storage \ + --entrypoint /bin/sh \ + minio/mc:RELEASE.2023-08-18T21-57-55Z \ + -c "mc alias set myminio http://minio:9000 \$MINIO_ROOT_USER \$MINIO_ROOT_PASSWORD && mc mb -p myminio/\${STORAGE_BUCKET} || true" + +# --- Wait for API --- +attempts=0 +max_attempts=12 +until curl -fsS http://localhost:80 >/dev/null 2>&1; do + attempts=$((attempts+1)) + if [ "$attempts" -ge "$max_attempts" ]; then + echo "API not ready after $max_attempts attempts" + $COMPOSE logs + exit 1 + fi + echo "Waiting for API server... attempt $attempts" + sleep 10 +done +echo "API server is ready!" + +# --- Run newman tests --- +docker run --rm --network="host" \ + -v "$PWD/tests/e2e:/etc/newman" \ + postman/newman:5.3.1-alpine \ + run tests.postman_collection.json \ + --env-var "url=localhost:80" \ + --reporters cli \ + --bail \ + --color on + +# --- Logs --- +echo "---- docker compose logs ----" +$COMPOSE logs || true +if [ -f .data/api/logs/api.jsonl ]; then + echo "---- .data/api/logs/api.jsonl ----" + cat .data/api/logs/api.jsonl || true +fi