diff --git a/.github/workflows/build-images.yml b/.github/workflows/build-images.yml index 89c25bd02..baf95a9d3 100644 --- a/.github/workflows/build-images.yml +++ b/.github/workflows/build-images.yml @@ -17,6 +17,14 @@ on: - all - api - worker + deployment_target: + description: Staging deployment target + required: false + type: choice + default: ecs-staging + options: + - ecs-staging + - eks-staging-rollback env: ACR_REGISTRY: ${{ secrets.ALIYUN_ACR_REGISTRY }} @@ -25,6 +33,14 @@ env: ECR_REPOSITORY: knowhere AWS_EKS_PROD_CLUSTER_NAME: ${{ secrets.AWS_EKS_PROD_CLUSTER_NAME }} AWS_EKS_PROD_REGION: ${{ secrets.AWS_EKS_PROD_REGION }} + AWS_ECS_STAGING_CLUSTER_NAME: knowhere-fargate + AWS_ECS_STAGING_REGION: us-east-1 + AWS_ECS_STAGING_EXECUTION_ROLE_ARN: arn:aws:iam::107424103509:role/knowhere-fargate-staging-execution-role + AWS_ECS_STAGING_API_TASK_ROLE_ARN: arn:aws:iam::107424103509:role/knowhere-api-staging-task-role + AWS_ECS_STAGING_WORKER_TASK_ROLE_ARN: arn:aws:iam::107424103509:role/knowhere-worker-staging-task-role + AWS_ECS_STAGING_SECRETS_ARN: ${{ secrets.AWS_ECS_STAGING_SECRETS_ARN }} + AWS_ECS_STAGING_API_SERVICE_NAME: knowhere-api-staging + AWS_ECS_STAGING_WORKER_SERVICE_NAME: knowhere-worker-staging jobs: build-and-publish: @@ -79,7 +95,7 @@ jobs: should_push="true" - if [ "${{ github.event_name }}" = "workflow_dispatch" ] && [ -n "${{ github.event.inputs.service }}" ] && [ "${{ github.event.inputs.service }}" != "all" ] && [ "${{ github.event.inputs.service }}" != "${{ matrix.service }}" ]; then + if [ "${{ github.event_name }}" = "workflow_dispatch" ] && [ "${{ github.event.inputs.deployment_target }}" != "ecs-staging" ] && [ -n "${{ github.event.inputs.service }}" ] && [ "${{ github.event.inputs.service }}" != "all" ] && [ "${{ github.event.inputs.service }}" != "${{ matrix.service }}" ]; then should_build="false" else should_build="true" @@ -218,14 +234,16 @@ jobs: echo "Push enabled: ${{ steps.context.outputs.should_push }}" echo "Primary tag: ${{ steps.tags.outputs.image_tag }}" - migrate: - name: Run API database migration + migrate-ecs-staging: + name: Run staging Neon database migration runs-on: ubuntu-latest needs: build-and-publish if: >- ${{ (github.ref == 'refs/heads/staging' || github.event_name == 'workflow_dispatch') && (github.event_name != 'workflow_dispatch' || + github.event.inputs.deployment_target == 'ecs-staging') && + (github.event_name != 'workflow_dispatch' || github.event.inputs.service == '' || github.event.inputs.service == 'all' || github.event.inputs.service == 'api') }} @@ -233,36 +251,70 @@ jobs: contents: read steps: - - name: Decide migration context - id: context + - name: Validate migration configuration shell: bash run: | set -euo pipefail - - if [ "${{ github.event_name }}" = "release" ] || [[ "${{ github.ref }}" == refs/tags/* ]]; then - environment="prod" - else - environment="staging" + if [ -z "${{ secrets.AWS_ACCESS_KEY_ID }}" ] || [ -z "${{ secrets.AWS_SECRET_ACCESS_KEY }}" ]; then + echo "::error::Staging Neon migration requires AWS ECR credentials." + exit 1 fi - - if [ "$environment" = "prod" ]; then - namespace="knowhere-prod" - else - namespace="knowhere-staging" + if [ -z "${{ secrets.STAGING_MIGRATION_DATABASE_URL }}" ]; then + echo "::error::STAGING_MIGRATION_DATABASE_URL is not configured. Add the direct Neon migration-role URL before the ECS staging deployment." + exit 1 fi + - name: Configure AWS credentials + uses: aws-actions/configure-aws-credentials@v4 + with: + aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} + aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + aws-region: ${{ env.AWS_ECS_STAGING_REGION }} + + - name: Login to ECR + shell: bash + run: | + set -euo pipefail + aws ecr get-login-password --region "${AWS_ECS_STAGING_REGION}" \ + | docker login --username AWS --password-stdin "${ECR_REGISTRY}" + + - name: Run migration against Neon + shell: bash + env: + DATABASE_URL: ${{ secrets.STAGING_MIGRATION_DATABASE_URL }} + run: | + set -euo pipefail short_sha="${GITHUB_SHA::8}" - if [ "${{ github.event_name }}" = "release" ]; then - image_tag="${{ github.event.release.tag_name }}-${environment}" - elif [[ "${{ github.ref }}" == refs/tags/* ]]; then - image_tag="${GITHUB_REF#refs/tags/}-${environment}" - else - image_tag="${environment}-${short_sha}" - fi + image_uri="${ECR_REGISTRY}/${ECR_REPOSITORY}/knowhere-backend:staging-${short_sha}" + docker run --rm \ + --env DATABASE_URL \ + --env DB_SSL_MODE=require \ + --entrypoint python \ + "$image_uri" \ + -m alembic upgrade heads + + migrate-eks-staging: + name: Run EKS rollback database migration + runs-on: ubuntu-latest + needs: build-and-publish + if: >- + ${{ github.event_name == 'workflow_dispatch' && + github.event.inputs.deployment_target == 'eks-staging-rollback' && + (github.event.inputs.service == '' || + github.event.inputs.service == 'all' || + github.event.inputs.service == 'api') }} + permissions: + contents: read - echo "environment=$environment" >> "$GITHUB_OUTPUT" - echo "namespace=$namespace" >> "$GITHUB_OUTPUT" - echo "image_uri=${{ env.ECR_REGISTRY }}/${{ env.ECR_REPOSITORY }}/knowhere-backend:${image_tag}" >> "$GITHUB_OUTPUT" + steps: + - name: Decide migration context + id: context + shell: bash + run: | + set -euo pipefail + short_sha="${GITHUB_SHA::8}" + echo "namespace=knowhere-staging" >> "$GITHUB_OUTPUT" + echo "image_uri=${{ env.ECR_REGISTRY }}/${{ env.ECR_REPOSITORY }}/knowhere-backend:staging-${short_sha}" >> "$GITHUB_OUTPUT" - name: Validate deployment configuration shell: bash @@ -272,7 +324,7 @@ jobs: [ -z "${{ secrets.AWS_SECRET_ACCESS_KEY }}" ] || \ [ -z "${{ env.AWS_EKS_PROD_CLUSTER_NAME }}" ] || \ [ -z "${{ env.AWS_EKS_PROD_REGION }}" ]; then - echo "::error::Migration requires AWS deployment credentials and cluster configuration." + echo "::error::EKS rollback migration requires AWS deployment credentials and cluster configuration." exit 1 fi @@ -302,7 +354,6 @@ jobs: set -euo pipefail job_name="knowhere-api-migrate-${GITHUB_RUN_ID}" - migration_manifest="$(kubectl get deployment/knowhere-api --namespace "$NAMESPACE" -o json \ | jq --arg job_name "$job_name" --arg image_uri "$IMAGE_URI" ' if ((.spec.template.spec.containers // []) | length) == 0 then @@ -331,23 +382,256 @@ jobs: ')" printf '%s\n' "$migration_manifest" | kubectl apply -f - - - if ! kubectl wait --for=condition=complete "job/$job_name" \ - --namespace "$NAMESPACE" --timeout=900s; then + if ! kubectl wait --for=condition=complete "job/$job_name" --namespace "$NAMESPACE" --timeout=900s; then kubectl describe job "$job_name" --namespace "$NAMESPACE" || true kubectl logs "job/$job_name" --namespace "$NAMESPACE" --all-containers=true || true exit 1 fi - kubectl delete job "$job_name" --namespace "$NAMESPACE" --ignore-not-found - deploy: + deploy-ecs-staging: + name: Deploy staging services to ECS + runs-on: ubuntu-latest + needs: [build-and-publish, migrate-ecs-staging] + if: >- + ${{ always() && needs.build-and-publish.result == 'success' && + (needs['migrate-ecs-staging'].result == 'success' || needs['migrate-ecs-staging'].result == 'skipped') && + ((github.ref == 'refs/heads/staging' && github.event_name == 'push') || + (github.event_name == 'workflow_dispatch' && + github.event.inputs.deployment_target == 'ecs-staging')) }} + permissions: + contents: read + + steps: + - name: Checkout deployed source + uses: actions/checkout@v4 + with: + persist-credentials: false + + - name: Decide ECS services to update + id: deployment + shell: bash + run: | + set -euo pipefail + selected_service="${{ github.event.inputs.service }}" + if [ "${{ github.event_name }}" = "push" ] || [ -z "$selected_service" ] || [ "$selected_service" = "all" ]; then + deploy_api="true" + deploy_worker="true" + elif [ "$selected_service" = "api" ]; then + deploy_api="true" + deploy_worker="false" + elif [ "$selected_service" = "worker" ]; then + deploy_api="false" + deploy_worker="true" + else + echo "::error::Unsupported ECS staging service selection: ${selected_service}" + exit 1 + fi + echo "deploy_api=$deploy_api" >> "$GITHUB_OUTPUT" + echo "deploy_worker=$deploy_worker" >> "$GITHUB_OUTPUT" + + - name: Configure AWS credentials + uses: aws-actions/configure-aws-credentials@v4 + with: + aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} + aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + aws-region: ${{ env.AWS_ECS_STAGING_REGION }} + + - name: Validate ECS staging prerequisites + shell: bash + env: + STAGING_SECRETS_ARN: ${{ env.AWS_ECS_STAGING_SECRETS_ARN }} + run: | + set -euo pipefail + + if [ -z "${{ secrets.AWS_ACCESS_KEY_ID }}" ] || [ -z "${{ secrets.AWS_SECRET_ACCESS_KEY }}" ]; then + echo "::error::ECS staging deployment requires AWS deployment credentials." + exit 1 + fi + if [ -z "$STAGING_SECRETS_ARN" ]; then + echo "::error::AWS_ECS_STAGING_SECRETS_ARN is not configured. Create and approve the staging runtime secret before deploying ECS." + exit 1 + fi + + cluster_status="$(aws ecs describe-clusters \ + --clusters "${AWS_ECS_STAGING_CLUSTER_NAME}" \ + --query 'clusters[0].status' --output text)" + if [ "$cluster_status" != "ACTIVE" ]; then + echo "::error::ECS cluster ${AWS_ECS_STAGING_CLUSTER_NAME} is not ACTIVE (status: ${cluster_status})." + exit 1 + fi + + for role_arn in \ + "${AWS_ECS_STAGING_EXECUTION_ROLE_ARN}" \ + "${AWS_ECS_STAGING_API_TASK_ROLE_ARN}" \ + "${AWS_ECS_STAGING_WORKER_TASK_ROLE_ARN}"; do + aws iam get-role --role-name "${role_arn##*/}" --query 'Role.Arn' --output text >/dev/null + done + + aws secretsmanager describe-secret \ + --secret-id "$STAGING_SECRETS_ARN" \ + --query 'ARN' --output text >/dev/null + + for log_group in /ecs/knowhere-api-staging /ecs/knowhere-worker-staging; do + found="$(aws logs describe-log-groups \ + --log-group-name "$log_group" \ + --query 'logGroups[0].logGroupName' --output text)" + if [ "$found" != "$log_group" ]; then + echo "::error::Required CloudWatch log group ${log_group} does not exist." + exit 1 + fi + done + + services="" + if [ "${{ steps.deployment.outputs.deploy_api }}" = "true" ]; then + services="${AWS_ECS_STAGING_API_SERVICE_NAME}" + fi + if [ "${{ steps.deployment.outputs.deploy_worker }}" = "true" ]; then + services="${services} ${AWS_ECS_STAGING_WORKER_SERVICE_NAME}" + fi + for service in $services; do + service_json="$(aws ecs describe-services \ + --cluster "${AWS_ECS_STAGING_CLUSTER_NAME}" \ + --services "$service" --output json)" + status="$(jq -r '.services[0].status // "MISSING"' <<<"$service_json")" + if [ "$status" != "ACTIVE" ]; then + echo "::error::Required ECS service ${service} is not ACTIVE (status: ${status}). Create it before deploying." + exit 1 + fi + network_count="$(jq '[.services[0].networkConfiguration.awsvpcConfiguration.subnets // [] | length] | add' <<<"$service_json")" + security_group_count="$(jq '[.services[0].networkConfiguration.awsvpcConfiguration.securityGroups // [] | length] | add' <<<"$service_json")" + if [ "$network_count" -lt 1 ] || [ "$security_group_count" -lt 1 ]; then + echo "::error::ECS service ${service} has no usable awsvpc subnet/security-group configuration." + exit 1 + fi + done + + if [ "${{ steps.deployment.outputs.deploy_api }}" = "true" ]; then + api_load_balancer_count="$(aws ecs describe-services \ + --cluster "${AWS_ECS_STAGING_CLUSTER_NAME}" \ + --services "${AWS_ECS_STAGING_API_SERVICE_NAME}" \ + --query 'length(services[0].loadBalancers)' --output text)" + if [ "$api_load_balancer_count" -lt 1 ]; then + echo "::error::API ECS service has no load balancer target configured." + exit 1 + fi + fi + + - name: Resolve immutable ECR image digests + id: images + shell: bash + run: | + set -euo pipefail + short_sha="${GITHUB_SHA::8}" + image_tag="staging-${short_sha}" + + resolve_image() { + local repository="$1" + local output_name="$2" + local digest + digest="$(aws ecr describe-images \ + --repository-name "${ECR_REPOSITORY}/${repository}" \ + --image-ids "imageTag=${image_tag}" \ + --query 'imageDetails[0].imageDigest' --output text)" + if [ -z "$digest" ] || [ "$digest" = "None" ]; then + echo "::error::No ECR image found for ${repository}:${image_tag}." + exit 1 + fi + echo "${output_name}=${ECR_REGISTRY}/${ECR_REPOSITORY}/${repository}@${digest}" >> "$GITHUB_OUTPUT" + } + + resolve_image knowhere-backend api_image + resolve_image knowhere-worker worker_image + + - name: Render ECS task definitions + env: + API_IMAGE: ${{ steps.images.outputs.api_image }} + WORKER_IMAGE: ${{ steps.images.outputs.worker_image }} + EXECUTION_ROLE_ARN: ${{ env.AWS_ECS_STAGING_EXECUTION_ROLE_ARN }} + API_TASK_ROLE_ARN: ${{ env.AWS_ECS_STAGING_API_TASK_ROLE_ARN }} + WORKER_TASK_ROLE_ARN: ${{ env.AWS_ECS_STAGING_WORKER_TASK_ROLE_ARN }} + STAGING_SECRETS_ARN: ${{ env.AWS_ECS_STAGING_SECRETS_ARN }} + shell: bash + run: | + set -euo pipefail + python3 deploy/ecs/render_task_definitions.py --output-dir "$RUNNER_TEMP/ecs-task-definitions" + + - name: Register ECS task definitions + id: task-definitions + shell: bash + run: | + set -euo pipefail + if [ "${{ steps.deployment.outputs.deploy_api }}" = "true" ]; then + api_task_definition_arn="$(aws ecs register-task-definition \ + --cli-input-json "file://${RUNNER_TEMP}/ecs-task-definitions/knowhere-api-staging.json" \ + --query 'taskDefinition.taskDefinitionArn' --output text)" + echo "api_task_definition_arn=${api_task_definition_arn}" >> "$GITHUB_OUTPUT" + fi + if [ "${{ steps.deployment.outputs.deploy_worker }}" = "true" ]; then + worker_task_definition_arn="$(aws ecs register-task-definition \ + --cli-input-json "file://${RUNNER_TEMP}/ecs-task-definitions/knowhere-worker-staging.json" \ + --query 'taskDefinition.taskDefinitionArn' --output text)" + echo "worker_task_definition_arn=${worker_task_definition_arn}" >> "$GITHUB_OUTPUT" + fi + + - name: Update API ECS service on on-demand Fargate + if: steps.deployment.outputs.deploy_api == 'true' + shell: bash + run: | + set -euo pipefail + aws ecs update-service \ + --cluster "${AWS_ECS_STAGING_CLUSTER_NAME}" \ + --service "${AWS_ECS_STAGING_API_SERVICE_NAME}" \ + --task-definition "${{ steps.task-definitions.outputs.api_task_definition_arn }}" \ + --capacity-provider-strategy capacityProvider=FARGATE,weight=1 + + - name: Update worker ECS service on Fargate Spot + if: steps.deployment.outputs.deploy_worker == 'true' + shell: bash + run: | + set -euo pipefail + aws ecs update-service \ + --cluster "${AWS_ECS_STAGING_CLUSTER_NAME}" \ + --service "${AWS_ECS_STAGING_WORKER_SERVICE_NAME}" \ + --task-definition "${{ steps.task-definitions.outputs.worker_task_definition_arn }}" \ + --capacity-provider-strategy capacityProvider=FARGATE_SPOT,weight=1 + + - name: Wait for API ECS service to stabilize + if: steps.deployment.outputs.deploy_api == 'true' + shell: bash + run: | + set -euo pipefail + aws ecs wait services-stable \ + --cluster "${AWS_ECS_STAGING_CLUSTER_NAME}" \ + --services "${AWS_ECS_STAGING_API_SERVICE_NAME}" + + - name: Wait for worker ECS service to stabilize + if: steps.deployment.outputs.deploy_worker == 'true' + shell: bash + run: | + set -euo pipefail + aws ecs wait services-stable \ + --cluster "${AWS_ECS_STAGING_CLUSTER_NAME}" \ + --services "${AWS_ECS_STAGING_WORKER_SERVICE_NAME}" + + - name: Summarize ECS deployment + shell: bash + run: | + echo "Environment: staging" + echo "Cluster: ${AWS_ECS_STAGING_CLUSTER_NAME}" + echo "API image: ${{ steps.images.outputs.api_image }}" + echo "Worker image: ${{ steps.images.outputs.worker_image }}" + + deploy-eks: runs-on: ubuntu-latest - needs: [build-and-publish, migrate] + needs: [build-and-publish, migrate-eks-staging] if: >- ${{ always() && github.event_name != 'pull_request' && needs.build-and-publish.result == 'success' && - (needs.migrate.result == 'success' || needs.migrate.result == 'skipped') }} + (needs['migrate-eks-staging'].result == 'success' || needs['migrate-eks-staging'].result == 'skipped') && + (github.event_name == 'release' || + (github.event_name == 'workflow_dispatch' && + github.event.inputs.deployment_target == 'eks-staging-rollback')) }} permissions: contents: read @@ -471,7 +755,7 @@ jobs: release: name: Attach deployment release assets runs-on: ubuntu-latest - needs: deploy + needs: deploy-eks if: >- ${{ github.event_name == 'release' && github.event.action == 'published' }} diff --git a/apps/api/pyproject.toml b/apps/api/pyproject.toml index 0ff6234ee..cb8aaf85b 100644 --- a/apps/api/pyproject.toml +++ b/apps/api/pyproject.toml @@ -11,7 +11,7 @@ dependencies = [ "sqlalchemy==2.0.42", "alembic==1.13.1", "redis==5.3.1", - "celery==5.4.0", + "celery==5.5.3", "stripe==13.0.1", "pydantic==2.13.4", "PyJWT==2.12.0", diff --git a/apps/worker/requirements.txt b/apps/worker/requirements.txt index b777a1580..2e22a69a0 100644 --- a/apps/worker/requirements.txt +++ b/apps/worker/requirements.txt @@ -50,7 +50,7 @@ beautifulsoup4==4.13.4 # knowhere-worker-app # markdownify # markitdown -billiard==4.2.0 +billiard==4.2.1 # via # celery # knowhere-shared @@ -63,7 +63,7 @@ botocore==1.38.46 # boto3 # knowhere-shared # s3transfer -celery==5.4.0 +celery==5.5.3 # via # celery-redbeat # knowhere-shared @@ -185,7 +185,7 @@ jmespath==0.10.0 # aliyun-python-sdk-core # boto3 # botocore -kombu==5.4.0 +kombu==5.5.4 # via # celery # knowhere-shared @@ -207,7 +207,9 @@ mako==1.3.11 markdown-it-py==4.0.0 # via rich markdownify==1.2.2 - # via markitdown + # via + # knowhere-worker-app + # markitdown markitdown==0.1.2 # via knowhere-worker-app markupsafe==3.0.3 @@ -301,6 +303,7 @@ oss2==2.19.1 # via knowhere-worker-app packaging==26.1 # via + # kombu # onnxruntime # opentelemetry-instrumentation # opentelemetry-instrumentation-sqlalchemy @@ -379,7 +382,9 @@ pymupdf-layout==1.27.2 pymupdf4llm==1.27.2.1 # via knowhere-worker-app pypdf==6.10.2 - # via knowhere-worker-app + # via + # knowhere-shared + # knowhere-worker-app pytest==9.0.3 # via # knowhere-shared @@ -503,7 +508,7 @@ typing-inspection==0.4.2 # pydantic-settings tzdata==2026.1 # via - # celery + # kombu # pandas urllib3==2.6.3 # via diff --git a/apps/worker/tests/contract/test_worker_shutdown_contract.py b/apps/worker/tests/contract/test_worker_shutdown_contract.py new file mode 100644 index 000000000..cc7c5c65a --- /dev/null +++ b/apps/worker/tests/contract/test_worker_shutdown_contract.py @@ -0,0 +1,35 @@ +from __future__ import annotations + +import json +from pathlib import Path + + +def test_should_preserve_fargate_worker_soft_shutdown_contract( + worker_contract_environment: None, +) -> None: + from shared.core.celery_app import celery_app + + repository_root: Path = Path(__file__).resolve().parents[4] + task_definition_path: Path = ( + repository_root / "deploy/ecs/task-definition-worker.staging.json" + ) + task_definition: dict[str, object] = json.loads( + task_definition_path.read_text(encoding="utf-8") + ) + container_definitions: list[dict[str, object]] = task_definition[ + "containerDefinitions" + ] + worker_container: dict[str, object] = next( + container + for container in container_definitions + if container.get("name") == "worker" + ) + environment: list[dict[str, str]] = worker_container["environment"] + environment_values: dict[str, str] = { + item["name"]: item["value"] for item in environment + } + + assert celery_app.conf.worker_soft_shutdown_timeout == 90 + assert celery_app.conf.worker_enable_soft_shutdown_on_idle is True + assert environment_values["REMAP_SIGTERM"] == "SIGQUIT" + assert worker_container["stopTimeout"] == 120 diff --git a/deploy/ecs/README.md b/deploy/ecs/README.md new file mode 100644 index 000000000..b02ebd3a2 --- /dev/null +++ b/deploy/ecs/README.md @@ -0,0 +1,44 @@ +# ECS task definitions + +These files are deployment templates for the shared `knowhere-fargate` cluster. They do not contain secret values and are not registered automatically. + +The staging templates intentionally omit `S3_ACCESS_KEY_ID` and `S3_SECRET_ACCESS_KEY`. With `S3_TYPE=s3`, boto3 obtains temporary authenticated credentials from the ECS task role. + +The single staging Secrets Manager secret supplied to the renderer must be a JSON secret with these keys: + +- API: `DATABASE_URL`, `REDIS_HOST`, `REDIS_PORT`, `REDIS_PASSWORD`, `CELERY_REDIS_URL`, `SECRET_KEY`, `DS_KEY`, `ALI_API_KEYS`, `ARK_API_KEY`, `GPT_API_KEY`, `MINERU_API_KEYS`, `STRIPE_SECRET_KEY`, `STRIPE_WEBHOOK_SECRET`, `WEBHOOK_MASTER_KEY`, `LOGFIRE_TOKEN`, `QSTASH_TOKEN`, `QSTASH_CURRENT_SIGNING_KEY`, `QSTASH_NEXT_SIGNING_KEY` +- Worker: the API keys above plus `CELERY_REDIS_PASSWORD` and `ILOVEAPI_KEYS` + +Render only after the secret and log groups exist, substituting the exact immutable ECR image digests and IAM role/secret ARNs: + +```bash +API_IMAGE=107424103509.dkr.ecr.us-east-1.amazonaws.com/knowhere/knowhere-backend@sha256:... \ +WORKER_IMAGE=107424103509.dkr.ecr.us-east-1.amazonaws.com/knowhere/knowhere-worker@sha256:... \ +EXECUTION_ROLE_ARN=arn:aws:iam::107424103509:role/knowhere-fargate-staging-execution-role \ +API_TASK_ROLE_ARN=arn:aws:iam::107424103509:role/knowhere-api-staging-task-role \ +WORKER_TASK_ROLE_ARN=arn:aws:iam::107424103509:role/knowhere-worker-staging-task-role \ +STAGING_SECRETS_ARN=arn:aws:secretsmanager:us-east-1:107424103509:secret:knowhere/staging/runtime-... \ +python deploy/ecs/render_task_definitions.py --output-dir /tmp/knowhere-ecs-rendered +``` + +The output directory is deployment-only and must not be committed. The renderer fails on missing inputs, unresolved placeholders, or either long-lived S3 credential variable. + +## Staging workflow prerequisites + +The staging workflow in `.github/workflows/build-images.yml` expects these GitHub Actions secrets: + +- `AWS_ACCESS_KEY_ID` and `AWS_SECRET_ACCESS_KEY`: the deployment credential used by the existing release workflow. +- `AWS_ECS_STAGING_SECRETS_ARN`: the ARN of the approved JSON runtime secret described above. +- `STAGING_MIGRATION_DATABASE_URL`: the direct Neon URL for the staging migration role. It is used only by the migration container and is never passed to the ECS runtime tasks. + +Before an ECS staging deployment, an operator must create and verify the ECS services, network configuration, API load-balancer target, CloudWatch log groups, and runtime secret. The workflow validates those resources and fails without registering or updating a service when any prerequisite is missing. It does not create or delete AWS resources. + +## Local Docker smoke test + +LocalStack Community does not implement the ECS API used by this deployment, so it cannot validate Fargate orchestration. The runtime can still be checked locally with Docker: + +```bash +./deploy/ecs/docker-runtime-smoke.sh +``` + +The script creates an isolated Docker network, starts disposable PostgreSQL and Redis containers, builds the API and worker images, runs Alembic, and verifies the API and worker container health checks. It uses filesystem object storage and mock LLM responses, calls no AWS API, and removes its containers and network on exit. Override `POSTGRES_IMAGE` or `REDIS_IMAGE` when the official images are not available locally. When `http_proxy` or `https_proxy` is set, the script passes those values only to the Docker build process. diff --git a/deploy/ecs/docker-runtime-smoke.sh b/deploy/ecs/docker-runtime-smoke.sh new file mode 100755 index 000000000..53ff85bf6 --- /dev/null +++ b/deploy/ecs/docker-runtime-smoke.sh @@ -0,0 +1,151 @@ +#!/usr/bin/env bash + +set -euo pipefail + +NETWORK_NAME="knowhere-runtime-smoke" +POSTGRES_CONTAINER="knowhere-runtime-smoke-postgres" +REDIS_CONTAINER="knowhere-runtime-smoke-redis" +API_CONTAINER="knowhere-runtime-smoke-api" +WORKER_CONTAINER="knowhere-runtime-smoke-worker" +API_IMAGE="${API_IMAGE:-knowhere-runtime-smoke-api:local}" +WORKER_IMAGE="${WORKER_IMAGE:-knowhere-runtime-smoke-worker:local}" +POSTGRES_IMAGE="${POSTGRES_IMAGE:-postgres:15-alpine}" +REDIS_IMAGE="${REDIS_IMAGE:-redis:7-alpine}" + +cleanup() { + docker rm --force \ + "$API_CONTAINER" \ + "$WORKER_CONTAINER" \ + "$POSTGRES_CONTAINER" \ + "$REDIS_CONTAINER" >/dev/null 2>&1 || true + docker network rm "$NETWORK_NAME" >/dev/null 2>&1 || true +} + +trap cleanup EXIT + +wait_for_postgres() { + for _attempt in $(seq 1 60); do + if docker exec "$POSTGRES_CONTAINER" pg_isready -U root -d Knowhere >/dev/null 2>&1; then + return 0 + fi + sleep 2 + done + docker logs "$POSTGRES_CONTAINER" || true + return 1 +} + +wait_for_redis() { + for _attempt in $(seq 1 60); do + if docker exec "$REDIS_CONTAINER" redis-cli ping >/dev/null 2>&1; then + return 0 + fi + sleep 2 + done + docker logs "$REDIS_CONTAINER" || true + return 1 +} + +wait_for_api() { + for _attempt in $(seq 1 90); do + if docker exec "$API_CONTAINER" curl --fail --silent http://localhost:5005/health >/dev/null 2>&1; then + return 0 + fi + if ! docker inspect --format '{{.State.Running}}' "$API_CONTAINER" 2>/dev/null | grep --quiet true; then + docker logs "$API_CONTAINER" || true + return 1 + fi + sleep 2 + done + docker logs "$API_CONTAINER" || true + return 1 +} + +wait_for_worker() { + for _attempt in $(seq 1 90); do + if docker exec "$WORKER_CONTAINER" \ + python -c 'from shared.services.worker_health import assert_worker_healthy; assert_worker_healthy()' >/dev/null 2>&1; then + return 0 + fi + if ! docker inspect --format '{{.State.Running}}' "$WORKER_CONTAINER" 2>/dev/null | grep --quiet true; then + docker logs "$WORKER_CONTAINER" || true + return 1 + fi + sleep 2 + done + docker logs "$WORKER_CONTAINER" || true + return 1 +} + +cleanup +docker network create "$NETWORK_NAME" >/dev/null + +docker run --detach --name "$POSTGRES_CONTAINER" --network "$NETWORK_NAME" \ + --env POSTGRES_DB=Knowhere \ + --env POSTGRES_USER=root \ + --env POSTGRES_PASSWORD=root123 \ + "$POSTGRES_IMAGE" >/dev/null +docker run --detach --name "$REDIS_CONTAINER" --network "$NETWORK_NAME" \ + "$REDIS_IMAGE" >/dev/null + +wait_for_postgres +wait_for_redis + +build_arguments=() +if [[ -n "${http_proxy:-}" ]]; then + build_arguments+=(--build-arg "http_proxy=${http_proxy}") +fi +if [[ -n "${https_proxy:-}" ]]; then + build_arguments+=(--build-arg "https_proxy=${https_proxy}") +fi + +docker build "${build_arguments[@]}" --file deploy/docker/Dockerfile.api --tag "$API_IMAGE" . +docker build "${build_arguments[@]}" --file deploy/docker/Dockerfile.worker --tag "$WORKER_IMAGE" . + +common_environment=( + --env ENVIRONMENT=development + --env APP_ENV=development + --env TMP_PATH=/tmp/knowhere + --env DATABASE_URL=postgresql+asyncpg://root:root123@${POSTGRES_CONTAINER}:5432/Knowhere + --env DB_SSL_MODE=disable + --env DB_POOL_SIZE=2 + --env DB_MAX_OVERFLOW=2 + --env DB_SYNC_POOL_SIZE=2 + --env DB_SYNC_MAX_OVERFLOW=2 + --env REDIS_HOST="$REDIS_CONTAINER" + --env REDIS_PORT=6379 + --env REDIS_DATABASE=0 + --env REDIS_SSL=false + --env CELERY_REDIS_URL=redis://${REDIS_CONTAINER}:6379/0 + --env S3_TYPE=filesystem + --env S3_BUCKET_NAME=knowhere-runtime-smoke + --env S3_RESULTS_BUCKET=knowhere-runtime-smoke + --env S3_TEMP_PATH=/tmp + --env OBJECT_STORAGE_LOCAL_ROOT=/tmp/object-storage + --env TELEMETRY_ENABLED=false + --env LOGFIRE_TOKEN= + --env LLM_MOCK_ENABLED=true + --env BILLING_ENABLED=false + --env API_STANDALONE_MODE_ENABLED=true + --env RATE_LIMIT_ENABLED=false +) + +docker run --rm --network "$NETWORK_NAME" \ + "${common_environment[@]}" \ + --entrypoint python "$API_IMAGE" -m alembic upgrade heads + +docker run --detach --name "$API_CONTAINER" --network "$NETWORK_NAME" \ + "${common_environment[@]}" \ + --env INTERNAL_DASHBOARD_ENDPOINT=http://localhost:3000 \ + "$API_IMAGE" >/dev/null + +docker run --detach --name "$WORKER_CONTAINER" --network "$NETWORK_NAME" \ + "${common_environment[@]}" \ + --env WORKER_CONCURRENCY=1 \ + "$WORKER_IMAGE" >/dev/null + +wait_for_api +wait_for_worker + +printf 'Docker runtime smoke test passed.\n' +printf ' API image: %s\n' "$API_IMAGE" +printf ' Worker image: %s\n' "$WORKER_IMAGE" diff --git a/deploy/ecs/render_task_definitions.py b/deploy/ecs/render_task_definitions.py new file mode 100644 index 000000000..4312533ce --- /dev/null +++ b/deploy/ecs/render_task_definitions.py @@ -0,0 +1,119 @@ +"""Render staging ECS task-definition templates without storing secrets.""" + +from __future__ import annotations + +import argparse +import json +import os +import re +from pathlib import Path +from typing import Final + + +PLACEHOLDER_PATTERN: Final[re.Pattern[str]] = re.compile(r"\$\{([A-Z0-9_]+)\}") +REQUIRED_VARIABLES: Final[tuple[str, ...]] = ( + "API_IMAGE", + "WORKER_IMAGE", + "EXECUTION_ROLE_ARN", + "API_TASK_ROLE_ARN", + "WORKER_TASK_ROLE_ARN", + "STAGING_SECRETS_ARN", +) +FORBIDDEN_ENVIRONMENT_NAMES: Final[frozenset[str]] = frozenset( + {"S3_ACCESS_KEY_ID", "S3_SECRET_ACCESS_KEY"} +) + + +def replace_placeholders(value: object, variables: dict[str, str]) -> object: + """Recursively replace required placeholders in a JSON-compatible value.""" + if isinstance(value, str): + return PLACEHOLDER_PATTERN.sub( + lambda match: variables.get(match.group(1), match.group(0)), value + ) + if isinstance(value, list): + return [replace_placeholders(item, variables) for item in value] + if isinstance(value, dict): + return { + str(key): replace_placeholders(item, variables) + for key, item in value.items() + } + return value + + +def collect_strings(value: object) -> list[str]: + """Collect all string values for unresolved-placeholder validation.""" + if isinstance(value, str): + return [value] + if isinstance(value, list): + return [item for child in value for item in collect_strings(child)] + if isinstance(value, dict): + return [item for child in value.values() for item in collect_strings(child)] + return [] + + +def validate_rendered_definition(definition: object) -> None: + """Reject unresolved placeholders and long-lived S3 credential variables.""" + strings = collect_strings(definition) + unresolved = [value for value in strings if PLACEHOLDER_PATTERN.search(value)] + if unresolved: + raise ValueError(f"Unresolved task-definition placeholders: {unresolved}") + + containers = definition.get("containerDefinitions") if isinstance(definition, dict) else None + if not isinstance(containers, list): + raise ValueError("Task definition must contain containerDefinitions") + environment_names = { + str(item.get("name")) + for container in containers + if isinstance(container, dict) + for item in [*(container.get("environment") or []), *(container.get("secrets") or [])] + if isinstance(item, dict) + } + forbidden = environment_names & FORBIDDEN_ENVIRONMENT_NAMES + if forbidden: + raise ValueError(f"Task definition contains forbidden S3 credentials: {sorted(forbidden)}") + + +def render_template(template_path: Path, output_path: Path, variables: dict[str, str]) -> None: + """Render one JSON template to a deployment-only output path.""" + definition = json.loads(template_path.read_text(encoding="utf-8")) + rendered = replace_placeholders(definition, variables) + validate_rendered_definition(rendered) + output_path.parent.mkdir(parents=True, exist_ok=True) + output_path.write_text(json.dumps(rendered, indent=2) + "\n", encoding="utf-8") + + +def parse_arguments() -> argparse.Namespace: + """Parse renderer CLI arguments.""" + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--output-dir", type=Path, required=True) + return parser.parse_args() + + +def load_variables() -> dict[str, str]: + """Load and validate deployment inputs from the environment.""" + variables = {name: os.environ.get(name, "") for name in REQUIRED_VARIABLES} + missing = [name for name, value in variables.items() if not value] + if missing: + raise ValueError(f"Missing required renderer variables: {', '.join(missing)}") + return variables + + +def main() -> None: + """Render both staging task definitions.""" + arguments = parse_arguments() + variables = load_variables() + template_directory = Path(__file__).parent + render_template( + template_directory / "task-definition-api.staging.json", + arguments.output_dir / "knowhere-api-staging.json", + variables, + ) + render_template( + template_directory / "task-definition-worker.staging.json", + arguments.output_dir / "knowhere-worker-staging.json", + variables, + ) + + +if __name__ == "__main__": + main() diff --git a/deploy/ecs/task-definition-api.staging.json b/deploy/ecs/task-definition-api.staging.json new file mode 100644 index 000000000..71025dd3d --- /dev/null +++ b/deploy/ecs/task-definition-api.staging.json @@ -0,0 +1,101 @@ +{ + "family": "knowhere-api-staging", + "taskRoleArn": "${API_TASK_ROLE_ARN}", + "executionRoleArn": "${EXECUTION_ROLE_ARN}", + "networkMode": "awsvpc", + "requiresCompatibilities": ["FARGATE"], + "cpu": "256", + "memory": "1024", + "runtimePlatform": { + "cpuArchitecture": "X86_64", + "operatingSystemFamily": "LINUX" + }, + "containerDefinitions": [ + { + "name": "api", + "image": "${API_IMAGE}", + "essential": true, + "portMappings": [ + { + "name": "http", + "containerPort": 5005, + "hostPort": 5005, + "protocol": "tcp" + } + ], + "environment": [ + {"name": "ENVIRONMENT", "value": "staging"}, + {"name": "APP_ENV", "value": "staging"}, + {"name": "DB_SSL_MODE", "value": "require"}, + {"name": "DB_POOL_SIZE", "value": "5"}, + {"name": "DB_MAX_OVERFLOW", "value": "5"}, + {"name": "TMP_PATH", "value": "/tmp/aismart_bid"}, + {"name": "S3_TYPE", "value": "s3"}, + {"name": "S3_BUCKET_NAME", "value": "knowhere-storage-staging"}, + {"name": "S3_RESULTS_BUCKET", "value": "knowhere-storage-staging"}, + {"name": "S3_TEMP_PATH", "value": "/tmp"}, + {"name": "S3_REGION", "value": "us-east-1"}, + {"name": "S3_USE_SSL", "value": "true"}, + {"name": "S3_ADDRESSING_STYLE", "value": "virtual"}, + {"name": "SUPPORTED_EXTENSIONS", "value": ".doc,.docx,.pdf,.txt,.xls,.xlsx,.csv,.jpg,.jpeg,.png,.pptx,.md"}, + {"name": "REDIS_DATABASE", "value": "0"}, + {"name": "REDIS_SSL", "value": "true"}, + {"name": "BROKER_POOL_LIMIT", "value": "5"}, + {"name": "DS_URL", "value": "https://api.deepseek.com/v1/chat/completions"}, + {"name": "ALI_URL", "value": "https://dashscope-intl.aliyuncs.com/compatible-mode/v1/chat/completions"}, + {"name": "ARK_URL", "value": "https://ark.cn-beijing.volces.com/api/v3/chat/completions"}, + {"name": "EMBEDDING_MODEL", "value": "text-embedding-v4"}, + {"name": "RATE_LIMIT_ENABLED", "value": "true"}, + {"name": "TELEMETRY_ENABLED", "value": "false"}, + {"name": "API_STANDALONE_MODE_ENABLED", "value": "true"}, + {"name": "INTERNAL_DASHBOARD_ENDPOINT", "value": "https://staging.knowhereto.ai"}, + {"name": "FRONTEND_URL", "value": "https://staging.knowhereto.ai"}, + {"name": "BILLING_ENABLED", "value": "true"}, + {"name": "QSTASH_CALLBACK_BASE_URL", "value": "https://api-staging.knowhereto.ai/api/v1"}, + {"name": "AWS_REGION", "value": "us-east-1"}, + {"name": "AWS_ACCOUNT_ID", "value": "107424103509"} + ], + "secrets": [ + {"name": "DATABASE_URL", "valueFrom": "${STAGING_SECRETS_ARN}:DATABASE_URL::"}, + {"name": "REDIS_HOST", "valueFrom": "${STAGING_SECRETS_ARN}:REDIS_HOST::"}, + {"name": "REDIS_PORT", "valueFrom": "${STAGING_SECRETS_ARN}:REDIS_PORT::"}, + {"name": "REDIS_PASSWORD", "valueFrom": "${STAGING_SECRETS_ARN}:REDIS_PASSWORD::"}, + {"name": "CELERY_REDIS_URL", "valueFrom": "${STAGING_SECRETS_ARN}:CELERY_REDIS_URL::"}, + {"name": "SECRET_KEY", "valueFrom": "${STAGING_SECRETS_ARN}:SECRET_KEY::"}, + {"name": "DS_KEY", "valueFrom": "${STAGING_SECRETS_ARN}:DS_KEY::"}, + {"name": "ALI_API_KEYS", "valueFrom": "${STAGING_SECRETS_ARN}:ALI_API_KEYS::"}, + {"name": "ARK_API_KEY", "valueFrom": "${STAGING_SECRETS_ARN}:ARK_API_KEY::"}, + {"name": "GPT_API_KEY", "valueFrom": "${STAGING_SECRETS_ARN}:GPT_API_KEY::"}, + {"name": "MINERU_API_KEYS", "valueFrom": "${STAGING_SECRETS_ARN}:MINERU_API_KEYS::"}, + {"name": "STRIPE_SECRET_KEY", "valueFrom": "${STAGING_SECRETS_ARN}:STRIPE_SECRET_KEY::"}, + {"name": "STRIPE_WEBHOOK_SECRET", "valueFrom": "${STAGING_SECRETS_ARN}:STRIPE_WEBHOOK_SECRET::"}, + {"name": "WEBHOOK_MASTER_KEY", "valueFrom": "${STAGING_SECRETS_ARN}:WEBHOOK_MASTER_KEY::"}, + {"name": "LOGFIRE_TOKEN", "valueFrom": "${STAGING_SECRETS_ARN}:LOGFIRE_TOKEN::"}, + {"name": "QSTASH_TOKEN", "valueFrom": "${STAGING_SECRETS_ARN}:QSTASH_TOKEN::"}, + {"name": "QSTASH_CURRENT_SIGNING_KEY", "valueFrom": "${STAGING_SECRETS_ARN}:QSTASH_CURRENT_SIGNING_KEY::"}, + {"name": "QSTASH_NEXT_SIGNING_KEY", "valueFrom": "${STAGING_SECRETS_ARN}:QSTASH_NEXT_SIGNING_KEY::"} + ], + "healthCheck": { + "command": ["CMD-SHELL", "curl -f http://localhost:5005/health || exit 1"], + "interval": 30, + "timeout": 5, + "retries": 3, + "startPeriod": 30 + }, + "logConfiguration": { + "logDriver": "awslogs", + "options": { + "awslogs-group": "/ecs/knowhere-api-staging", + "awslogs-region": "us-east-1", + "awslogs-stream-prefix": "api" + } + } + } + ], + "tags": [ + {"key": "Project", "value": "knowhere"}, + {"key": "Environment", "value": "staging"}, + {"key": "Service", "value": "api"}, + {"key": "ManagedBy", "value": "knowhere-api-infra"} + ] +} diff --git a/deploy/ecs/task-definition-worker.staging.json b/deploy/ecs/task-definition-worker.staging.json new file mode 100644 index 000000000..3ceb5d23d --- /dev/null +++ b/deploy/ecs/task-definition-worker.staging.json @@ -0,0 +1,109 @@ +{ + "family": "knowhere-worker-staging", + "taskRoleArn": "${WORKER_TASK_ROLE_ARN}", + "executionRoleArn": "${EXECUTION_ROLE_ARN}", + "networkMode": "awsvpc", + "requiresCompatibilities": ["FARGATE"], + "cpu": "512", + "memory": "4096", + "ephemeralStorage": {"sizeInGiB": 21}, + "runtimePlatform": { + "cpuArchitecture": "X86_64", + "operatingSystemFamily": "LINUX" + }, + "containerDefinitions": [ + { + "name": "worker", + "image": "${WORKER_IMAGE}", + "essential": true, + "stopTimeout": 120, + "environment": [ + {"name": "ENVIRONMENT", "value": "staging"}, + {"name": "APP_ENV", "value": "staging"}, + {"name": "REMAP_SIGTERM", "value": "SIGQUIT"}, + {"name": "DB_SSL_MODE", "value": "require"}, + {"name": "DB_SYNC_POOL_SIZE", "value": "2"}, + {"name": "DB_SYNC_MAX_OVERFLOW", "value": "2"}, + {"name": "TMP_PATH", "value": "/tmp/aismart_bid"}, + {"name": "S3_TYPE", "value": "s3"}, + {"name": "S3_BUCKET_NAME", "value": "knowhere-storage-staging"}, + {"name": "S3_RESULTS_BUCKET", "value": "knowhere-storage-staging"}, + {"name": "S3_TEMP_PATH", "value": "/tmp"}, + {"name": "S3_REGION", "value": "us-east-1"}, + {"name": "S3_USE_SSL", "value": "true"}, + {"name": "S3_ADDRESSING_STYLE", "value": "virtual"}, + {"name": "SUPPORTED_EXTENSIONS", "value": ".doc,.docx,.pdf,.txt,.xls,.xlsx,.csv,.jpg,.jpeg,.png,.pptx,.md"}, + {"name": "MAX_PDF_PAGE_LIMIT", "value": "200"}, + {"name": "OVERSIZED_PDF_SHARD_ENABLED", "value": "true"}, + {"name": "OVERSIZED_PDF_SOFT_LIMIT", "value": "1500"}, + {"name": "PDF_PROFILE_TOC_ENABLED", "value": "true"}, + {"name": "REDIS_DATABASE", "value": "0"}, + {"name": "REDIS_SSL", "value": "true"}, + {"name": "REDIS_SYNC_MAX_CONNECTIONS", "value": "200"}, + {"name": "BROKER_POOL_LIMIT", "value": "5"}, + {"name": "AWS_REGION", "value": "us-east-1"}, + {"name": "AWS_ACCOUNT_ID", "value": "107424103509"}, + {"name": "RATE_LIMIT_ENABLED", "value": "false"}, + {"name": "TELEMETRY_ENABLED", "value": "false"}, + {"name": "INTERNAL_DASHBOARD_ENDPOINT", "value": "https://staging.knowhereto.ai"}, + {"name": "API_WEBHOOK_ENDPOINT", "value": "https://api-staging.knowhereto.ai/v1/internal/s3-events"}, + {"name": "SNS_TOPIC_ARN", "value": "arn:aws:sns:us-east-1:107424103509:knowhere-staging-s3-events"}, + {"name": "QSTASH_CALLBACK_BASE_URL", "value": "https://api-staging.knowhereto.ai/api/v1"}, + {"name": "HF_HOME", "value": "/mnt/models/huggingface"}, + {"name": "TRANSFORMERS_CACHE", "value": "/mnt/models/huggingface"}, + {"name": "BILLING_ENABLED", "value": "true"}, + {"name": "SUMMARY_LLM_MAX_CONCURRENT", "value": "8"}, + {"name": "DS_URL", "value": "https://api.deepseek.com/v1/chat/completions"}, + {"name": "ALI_URL", "value": "https://dashscope-intl.aliyuncs.com/compatible-mode/v1/chat/completions"}, + {"name": "ARK_URL", "value": "https://ark.cn-beijing.volces.com/api/v3/chat/completions"}, + {"name": "EMBEDDING_MODEL", "value": "text-embedding-v4"}, + {"name": "HIERARCHY_LLM_MODEL", "value": "deepseek-chat"}, + {"name": "NORMOL_MODEL", "value": "deepseek-chat"}, + {"name": "IMAGE_MODEL", "value": "qwen3.5-flash"}, + {"name": "IMAGE_MODEL_MAX", "value": "qwen3.5-flash"}, + {"name": "ALL_DF_COLS", "value": "content,path,type,length,keywords,summary,know_id,tokens,connectto,addtime,page_nums"} + ], + "secrets": [ + {"name": "DATABASE_URL", "valueFrom": "${STAGING_SECRETS_ARN}:DATABASE_URL::"}, + {"name": "REDIS_HOST", "valueFrom": "${STAGING_SECRETS_ARN}:REDIS_HOST::"}, + {"name": "REDIS_PORT", "valueFrom": "${STAGING_SECRETS_ARN}:REDIS_PORT::"}, + {"name": "REDIS_PASSWORD", "valueFrom": "${STAGING_SECRETS_ARN}:REDIS_PASSWORD::"}, + {"name": "CELERY_REDIS_URL", "valueFrom": "${STAGING_SECRETS_ARN}:CELERY_REDIS_URL::"}, + {"name": "CELERY_REDIS_PASSWORD", "valueFrom": "${STAGING_SECRETS_ARN}:CELERY_REDIS_PASSWORD::"}, + {"name": "SECRET_KEY", "valueFrom": "${STAGING_SECRETS_ARN}:SECRET_KEY::"}, + {"name": "WEBHOOK_MASTER_KEY", "valueFrom": "${STAGING_SECRETS_ARN}:WEBHOOK_MASTER_KEY::"}, + {"name": "DS_KEY", "valueFrom": "${STAGING_SECRETS_ARN}:DS_KEY::"}, + {"name": "ALI_API_KEYS", "valueFrom": "${STAGING_SECRETS_ARN}:ALI_API_KEYS::"}, + {"name": "ARK_API_KEY", "valueFrom": "${STAGING_SECRETS_ARN}:ARK_API_KEY::"}, + {"name": "GPT_API_KEY", "valueFrom": "${STAGING_SECRETS_ARN}:GPT_API_KEY::"}, + {"name": "MINERU_API_KEYS", "valueFrom": "${STAGING_SECRETS_ARN}:MINERU_API_KEYS::"}, + {"name": "ILOVEAPI_KEYS", "valueFrom": "${STAGING_SECRETS_ARN}:ILOVEAPI_KEYS::"}, + {"name": "LOGFIRE_TOKEN", "valueFrom": "${STAGING_SECRETS_ARN}:LOGFIRE_TOKEN::"}, + {"name": "QSTASH_TOKEN", "valueFrom": "${STAGING_SECRETS_ARN}:QSTASH_TOKEN::"}, + {"name": "QSTASH_CURRENT_SIGNING_KEY", "valueFrom": "${STAGING_SECRETS_ARN}:QSTASH_CURRENT_SIGNING_KEY::"}, + {"name": "QSTASH_NEXT_SIGNING_KEY", "valueFrom": "${STAGING_SECRETS_ARN}:QSTASH_NEXT_SIGNING_KEY::"} + ], + "healthCheck": { + "command": ["CMD-SHELL", "python -c \"from shared.services.worker_health import assert_worker_healthy; assert_worker_healthy()\""], + "interval": 30, + "timeout": 10, + "retries": 3, + "startPeriod": 60 + }, + "logConfiguration": { + "logDriver": "awslogs", + "options": { + "awslogs-group": "/ecs/knowhere-worker-staging", + "awslogs-region": "us-east-1", + "awslogs-stream-prefix": "worker" + } + } + } + ], + "tags": [ + {"key": "Project", "value": "knowhere"}, + {"key": "Environment", "value": "staging"}, + {"key": "Service", "value": "worker"}, + {"key": "ManagedBy", "value": "knowhere-api-infra"} + ] +} diff --git a/deploy/ecs/test_render_task_definitions.py b/deploy/ecs/test_render_task_definitions.py new file mode 100644 index 000000000..7d4bb39ac --- /dev/null +++ b/deploy/ecs/test_render_task_definitions.py @@ -0,0 +1,141 @@ +"""Contracts for ECS task-definition rendering.""" + +import json +from pathlib import Path + +import pytest + +from render_task_definitions import ( + render_template, + validate_rendered_definition, +) + + +TEMPLATE_DIRECTORY: Path = Path(__file__).parent +RENDER_VARIABLES: dict[str, str] = { + "API_IMAGE": "api-image", + "WORKER_IMAGE": "worker-image", + "EXECUTION_ROLE_ARN": "execution-role", + "API_TASK_ROLE_ARN": "api-role", + "WORKER_TASK_ROLE_ARN": "worker-role", + "STAGING_SECRETS_ARN": "secrets-arn", +} + +SHARED_STAGING_ENVIRONMENT: dict[str, str] = { + "ALI_URL": "https://dashscope-intl.aliyuncs.com/compatible-mode/v1/chat/completions", + "ARK_URL": "https://ark.cn-beijing.volces.com/api/v3/chat/completions", + "BROKER_POOL_LIMIT": "5", + "DS_URL": "https://api.deepseek.com/v1/chat/completions", + "EMBEDDING_MODEL": "text-embedding-v4", + "S3_TEMP_PATH": "/tmp", + "SUPPORTED_EXTENSIONS": ".doc,.docx,.pdf,.txt,.xls,.xlsx,.csv,.jpg,.jpeg,.png,.pptx,.md", + "TMP_PATH": "/tmp/aismart_bid", +} + +API_STAGING_ENVIRONMENT: dict[str, str] = { + **SHARED_STAGING_ENVIRONMENT, + "DB_MAX_OVERFLOW": "5", + "DB_POOL_SIZE": "5", +} + +WORKER_STAGING_ENVIRONMENT: dict[str, str] = { + **SHARED_STAGING_ENVIRONMENT, + "ALL_DF_COLS": "content,path,type,length,keywords,summary,know_id,tokens,connectto,addtime,page_nums", + "DB_SYNC_MAX_OVERFLOW": "2", + "DB_SYNC_POOL_SIZE": "2", + "HIERARCHY_LLM_MODEL": "deepseek-chat", + "IMAGE_MODEL": "qwen3.5-flash", + "IMAGE_MODEL_MAX": "qwen3.5-flash", + "MAX_PDF_PAGE_LIMIT": "200", + "NORMOL_MODEL": "deepseek-chat", + "OVERSIZED_PDF_SHARD_ENABLED": "true", + "OVERSIZED_PDF_SOFT_LIMIT": "1500", + "PDF_PROFILE_TOC_ENABLED": "true", + "REDIS_SYNC_MAX_CONNECTIONS": "200", + "SUMMARY_LLM_MAX_CONCURRENT": "8", +} + + +@pytest.mark.parametrize( + "template_name", + ["task-definition-api.staging.json", "task-definition-worker.staging.json"], +) +def test_staging_templates_render_without_long_lived_s3_keys( + tmp_path: Path, + template_name: str, +) -> None: + """Both templates render and exclude explicit S3 credential variables.""" + output_path: Path = tmp_path / template_name + render_template( + TEMPLATE_DIRECTORY / template_name, + output_path, + RENDER_VARIABLES, + ) + + rendered: dict[str, object] = json.loads(output_path.read_text(encoding="utf-8")) + validate_rendered_definition(rendered) + assert rendered["requiresCompatibilities"] == ["FARGATE"] + assert "${" not in output_path.read_text(encoding="utf-8") + + +@pytest.mark.parametrize( + ("template_name", "container_name", "expected_environment"), + [ + ( + "task-definition-api.staging.json", + "api", + API_STAGING_ENVIRONMENT, + ), + ( + "task-definition-worker.staging.json", + "worker", + WORKER_STAGING_ENVIRONMENT, + ), + ], +) +def test_staging_templates_preserve_expected_staging_configuration( + template_name: str, + container_name: str, + expected_environment: dict[str, str], +) -> None: + """Fargate preserves the verified staging settings captured on 2026-08-13.""" + definition_path: Path = TEMPLATE_DIRECTORY / template_name + definition: dict[str, object] = json.loads( + definition_path.read_text(encoding="utf-8") + ) + container_definitions: list[dict[str, object]] = definition[ + "containerDefinitions" + ] + container: dict[str, object] = next( + item for item in container_definitions if item.get("name") == container_name + ) + environment: list[dict[str, str]] = container["environment"] + environment_values: dict[str, str] = { + item["name"]: item["value"] for item in environment + } + + for name, value in expected_environment.items(): + assert environment_values[name] == value + + +def test_renderer_rejects_forbidden_s3_credential_variable() -> None: + """Task definitions must never inject long-lived S3 credentials.""" + definition: dict[str, object] = { + "containerDefinitions": [ + {"environment": [{"name": "S3_ACCESS_KEY_ID", "value": "bad"}]} + ] + } + + with pytest.raises(ValueError, match="forbidden S3 credentials"): + validate_rendered_definition(definition) + + +def test_renderer_rejects_unresolved_placeholders() -> None: + """Deployment cannot proceed with missing substitution values.""" + definition: dict[str, object] = { + "containerDefinitions": [{"environment": []}], + "image": "${API_IMAGE}", + } + + with pytest.raises(ValueError, match="Unresolved task-definition placeholders"): + validate_rendered_definition(definition) diff --git a/packages/shared-python/pyproject.toml b/packages/shared-python/pyproject.toml index 260d831d4..3abaee115 100644 --- a/packages/shared-python/pyproject.toml +++ b/packages/shared-python/pyproject.toml @@ -21,9 +21,9 @@ dependencies = [ "redis==5.3.1", # Celery task queue - "celery==5.4.0", - "kombu==5.4.0", - "billiard==4.2.0", + "celery==5.5.3", + "kombu==5.5.4", + "billiard==4.2.1", "vine==5.1.0", "celery-redbeat==2.2.0", diff --git a/packages/shared-python/shared/core/celery_app.py b/packages/shared-python/shared/core/celery_app.py index 83b6534bb..9a40c9deb 100644 --- a/packages/shared-python/shared/core/celery_app.py +++ b/packages/shared-python/shared/core/celery_app.py @@ -60,6 +60,8 @@ def get_unique_node_name() -> str: worker_prefetch_multiplier=1, task_acks_late=True, worker_disable_rate_limits=True, + worker_soft_shutdown_timeout=90, + worker_enable_soft_shutdown_on_idle=True, # Redis serverless does not support the pidbox PSUBSCRIBE control channel. worker_enable_remote_control=False, task_reject_on_worker_lost=True, diff --git a/uv.lock b/uv.lock index 4f6aadc56..ea319a249 100644 --- a/uv.lock +++ b/uv.lock @@ -439,11 +439,11 @@ wheels = [ [[package]] name = "billiard" -version = "4.2.0" +version = "4.2.1" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/09/52/f10d74fd56e73b430c37417658158ad8386202b069b70ff97d945c3ab67a/billiard-4.2.0.tar.gz", hash = "sha256:9a3c3184cb275aa17a732f93f65b20c525d3d9f253722d26a82194803ade5a2c", size = 154665, upload-time = "2023-11-06T05:23:38.562Z" } +sdist = { url = "https://files.pythonhosted.org/packages/7c/58/1546c970afcd2a2428b1bfafecf2371d8951cc34b46701bea73f4280989e/billiard-4.2.1.tar.gz", hash = "sha256:12b641b0c539073fc8d3f5b8b7be998956665c4233c7c1fcd66a7e677c4fb36f", size = 155031, upload-time = "2024-09-21T13:40:22.491Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/50/8d/6e9fdeeab04d803abc5a715175f87e88893934d5590595eacff23ca12b07/billiard-4.2.0-py3-none-any.whl", hash = "sha256:07aa978b308f334ff8282bd4a746e681b3513db5c9a514cbdd810cbbdc19714d", size = 86720, upload-time = "2023-11-06T05:23:29.122Z" }, + { url = "https://files.pythonhosted.org/packages/30/da/43b15f28fe5f9e027b41c539abc5469052e9d48fd75f8ff094ba2a0ae767/billiard-4.2.1-py3-none-any.whl", hash = "sha256:40b59a4ac8806ba2c2369ea98d876bc6108b051c227baffd928c644d15d8f3cb", size = 86766, upload-time = "2024-09-21T13:40:20.188Z" }, ] [[package]] @@ -485,7 +485,7 @@ wheels = [ [[package]] name = "celery" -version = "5.4.0" +version = "5.5.3" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "billiard" }, @@ -495,12 +495,11 @@ dependencies = [ { name = "click-repl" }, { name = "kombu" }, { name = "python-dateutil" }, - { name = "tzdata" }, { name = "vine" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/8a/9c/cf0bce2cc1c8971bf56629d8f180e4ca35612c7e79e6e432e785261a8be4/celery-5.4.0.tar.gz", hash = "sha256:504a19140e8d3029d5acad88330c541d4c3f64c789d85f94756762d8bca7e706", size = 1575692, upload-time = "2024-04-17T20:29:43.675Z" } +sdist = { url = "https://files.pythonhosted.org/packages/bb/7d/6c289f407d219ba36d8b384b42489ebdd0c84ce9c413875a8aae0c85f35b/celery-5.5.3.tar.gz", hash = "sha256:6c972ae7968c2b5281227f01c3a3f984037d21c5129d07bf3550cc2afc6b10a5", size = 1667144, upload-time = "2025-06-01T11:08:12.563Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/90/c4/6a4d3772e5407622feb93dd25c86ce3c0fee746fa822a777a627d56b4f2a/celery-5.4.0-py3-none-any.whl", hash = "sha256:369631eb580cf8c51a82721ec538684994f8277637edde2dfc0dacd73ed97f64", size = 425983, upload-time = "2024-04-17T20:29:39.406Z" }, + { url = "https://files.pythonhosted.org/packages/c9/af/0dcccc7fdcdf170f9a1585e5e96b6fb0ba1749ef6be8c89a6202284759bd/celery-5.5.3-py3-none-any.whl", hash = "sha256:0b5761a07057acee94694464ca482416b959568904c9dfa41ce8413a7d65d525", size = 438775, upload-time = "2025-06-01T11:08:09.94Z" }, ] [[package]] @@ -1440,7 +1439,7 @@ dev = [ requires-dist = [ { name = "aiohttp", specifier = "==3.13.4" }, { name = "alembic", specifier = "==1.13.1" }, - { name = "celery", specifier = "==5.4.0" }, + { name = "celery", specifier = "==5.5.3" }, { name = "fastapi", specifier = "==0.135.1" }, { name = "httpx", specifier = "==0.28.1" }, { name = "knowhere-shared", editable = "packages/shared-python" }, @@ -1525,18 +1524,18 @@ requires-dist = [ { name = "alembic", specifier = "==1.13.1" }, { name = "asyncpg", specifier = "==0.31.0" }, { name = "authlib", specifier = "==1.6.11" }, - { name = "billiard", specifier = "==4.2.0" }, + { name = "billiard", specifier = "==4.2.1" }, { name = "blingfire", specifier = ">=0.1.8" }, { name = "boto3", specifier = "==1.38.46" }, { name = "botocore", specifier = "==1.38.46" }, - { name = "celery", specifier = "==5.4.0" }, + { name = "celery", specifier = "==5.5.3" }, { name = "celery-redbeat", specifier = "==2.2.0" }, { name = "email-validator", specifier = "==2.2.0" }, { name = "fastapi-users", extras = ["sqlalchemy"], specifier = "==15.0.5" }, { name = "gevent", specifier = ">=24.11.1" }, { name = "httpx", specifier = "==0.28.1" }, { name = "jieba", specifier = "==0.42.1" }, - { name = "kombu", specifier = "==5.4.0" }, + { name = "kombu", specifier = "==5.5.4" }, { name = "loguru", specifier = "==0.7.3" }, { name = "openai", specifier = ">=1.0.0" }, { name = "pgvector", specifier = "==0.2.4" }, @@ -1580,8 +1579,8 @@ dependencies = [ { name = "knowhere-shared" }, { name = "logfire", extra = ["celery", "fastapi", "httpx", "sqlalchemy"] }, { name = "lxml" }, - { name = "markitdown" }, { name = "markdownify" }, + { name = "markitdown" }, { name = "numpy" }, { name = "openai" }, { name = "openpyxl" }, @@ -1618,8 +1617,8 @@ requires-dist = [ { name = "knowhere-shared", editable = "packages/shared-python" }, { name = "logfire", extras = ["celery", "fastapi", "httpx", "sqlalchemy"], specifier = ">=4.25.0" }, { name = "lxml", specifier = "==6.1.0" }, - { name = "markitdown", specifier = "==0.1.2" }, { name = "markdownify", specifier = "==1.2.2" }, + { name = "markitdown", specifier = "==0.1.2" }, { name = "numpy", specifier = "==2.2.6" }, { name = "openai", specifier = "==1.93.3" }, { name = "openpyxl", specifier = "==3.1.2" }, @@ -1648,15 +1647,17 @@ dev = [ [[package]] name = "kombu" -version = "5.4.0" +version = "5.5.4" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "amqp" }, + { name = "packaging" }, + { name = "tzdata" }, { name = "vine" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/b6/f4/d3e57b1c351bb47ce25b16e1cf6ea05df4613dbe56e3cf32ea80df1a8b4d/kombu-5.4.0.tar.gz", hash = "sha256:ad200a8dbdaaa2bbc5f26d2ee7d707d9a1fded353a0f4bd751ce8c7d9f449c60", size = 442120, upload-time = "2024-08-06T13:42:58.842Z" } +sdist = { url = "https://files.pythonhosted.org/packages/0f/d3/5ff936d8319ac86b9c409f1501b07c426e6ad41966fedace9ef1b966e23f/kombu-5.5.4.tar.gz", hash = "sha256:886600168275ebeada93b888e831352fe578168342f0d1d5833d88ba0d847363", size = 461992, upload-time = "2025-06-01T10:19:22.281Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/df/17/34f8ec5b9d46a1ddb598b7bf8f779c567421d05cd73742d09e549254c782/kombu-5.4.0-py3-none-any.whl", hash = "sha256:c8dd99820467610b4febbc7a9e8a0d3d7da2d35116b67184418b51cc520ea6b6", size = 200870, upload-time = "2024-08-06T13:42:55.53Z" }, + { url = "https://files.pythonhosted.org/packages/ef/70/a07dcf4f62598c8ad579df241af55ced65bed76e42e45d3c368a6d82dbc1/kombu-5.5.4-py3-none-any.whl", hash = "sha256:a12ed0557c238897d8e518f1d1fdf84bd1516c5e305af2dacd85c2015115feb8", size = 210034, upload-time = "2025-06-01T10:19:20.436Z" }, ] [[package]]