From 72895090544b14d212240ed152bff1b213f8af9c Mon Sep 17 00:00:00 2001 From: Atti Ur Rehman Date: Tue, 23 Jun 2026 13:51:51 +0300 Subject: [PATCH] fix(aca): enable reliable scale-to-zero wake-up on idle Container Apps Backend no longer blocks startup on the DB pool so /health responds during cold start; frontend nginx validates config and uses a proper health probe. CD now verifies scale-from-zero after deploy, README documents idle behaviour, and infra/fix-scale-to-zero.sh helps diagnose failed revisions. Fixes #47 --- .github/workflows/cd.yml | 59 +++++++++++++++++++++++++++++++ README.md | 8 +++-- backend/Dockerfile | 4 +-- backend/app/db/pool.py | 24 +++++++------ backend/main.py | 8 ++--- frontend/Dockerfile | 5 +-- frontend/docker-entrypoint.sh | 2 ++ infra/fix-scale-to-zero.sh | 66 +++++++++++++++++++++++++++++++++++ 8 files changed, 153 insertions(+), 23 deletions(-) create mode 100755 infra/fix-scale-to-zero.sh diff --git a/.github/workflows/cd.yml b/.github/workflows/cd.yml index 5e0f55d..2aaea12 100644 --- a/.github/workflows/cd.yml +++ b/.github/workflows/cd.yml @@ -416,3 +416,62 @@ jobs: --min-replicas 0 \ --output none || true echo "Scale-to-zero restored." + + - name: Verify scale-from-zero (real visitor path) + if: success() + shell: bash + env: + BACKEND_URL: ${{ needs.deploy-backend.outputs.backend_url }} + FRONTEND_URL: ${{ needs.deploy-frontend.outputs.frontend_url }} + run: | + set -euo pipefail + resolve_url() { + local name="$1" url="$2" + if [[ -n "$url" && "$url" != "https://" ]]; then + echo "$url" + return + fi + local fqdn + fqdn=$(az containerapp show \ + --name "$name" \ + --resource-group "$AZURE_RESOURCE_GROUP" \ + --query "properties.configuration.ingress.fqdn" -o tsv) + echo "https://${fqdn}" + } + BACKEND=$(resolve_url "$BACKEND_APP_NAME" "${BACKEND_URL:-}") + FRONTEND=$(resolve_url "$FRONTEND_APP_NAME" "${FRONTEND_URL:-}") + echo "Scale-from-zero check — min-replicas=0, no warm-up..." + http="000" + echo " backend: ${BACKEND}/health" + for i in $(seq 1 24); do + http=$(curl -sk -o /tmp/sfz-be.json -w "%{http_code}" --max-time 25 "${BACKEND}/health" || echo "000") + if [[ "$http" == "200" ]]; then + echo " backend OK after ~$((i * 10))s" + break + fi + echo " backend attempt $i/24 — HTTP $http" + sleep 10 + done + if [[ "$http" != "200" ]]; then + echo "::error::Backend did not wake from zero within 240s" + az containerapp logs show --name "$BACKEND_APP_NAME" --resource-group "$AZURE_RESOURCE_GROUP" --tail 30 || true + exit 1 + fi + http="000" + echo " frontend: ${FRONTEND}/nginx-health" + for i in $(seq 1 24); do + http=$(curl -sk -o /tmp/sfz-fe.json -w "%{http_code}" --max-time 25 "${FRONTEND}/nginx-health" || echo "000") + if [[ "$http" == "200" ]]; then + echo " frontend OK after ~$((i * 10))s" + break + fi + echo " frontend attempt $i/24 — HTTP $http" + sleep 10 + done + if [[ "$http" != "200" ]]; then + echo "::error::Frontend did not wake from zero within 240s" + az containerapp logs show --name "$FRONTEND_APP_NAME" --resource-group "$AZURE_RESOURCE_GROUP" --tail 30 || true + az containerapp revision list --name "$FRONTEND_APP_NAME" --resource-group "$AZURE_RESOURCE_GROUP" -o table || true + exit 1 + fi + echo "Scale-from-zero wake-up verified." diff --git a/README.md b/README.md index 5d099dc..74bceca 100644 --- a/README.md +++ b/README.md @@ -13,6 +13,8 @@ Live: https://insightiq-frontend.orangemeadow-25c8b891.westeurope.azurecontainerapps.io +**Scale-to-zero:** both apps sleep when idle (`min-replicas: 0`). First visit after idle may take **30–90 seconds** while Azure starts containers. Open the [frontend](https://insightiq-frontend.orangemeadow-25c8b891.westeurope.azurecontainerapps.io) before a demo; sign-in wakes the backend automatically. + --- ## Demo @@ -80,7 +82,7 @@ Browser │ All API calls go directly: browser → backend HTTPS URL. │ └─► Backend Container App (FastAPI + asyncpg + Claude) - min-replicas: 1 (always warm — no cold start on demo) + min-replicas: 0 (scale to zero — wakes on first HTTP request) max-replicas: 3 (HPA on CPU) │ ▼ @@ -272,11 +274,11 @@ All deploys are zero-downtime rolling updates. The CD pipeline applies Neon migr | Service | Cost | |---------|------| -| Azure Container Apps — backend (min 1 replica) | ~$5–8 | +| Azure Container Apps — backend (scale to zero) | ~$1–5 | | Azure Container Apps — frontend (scale to zero) | ~$1–3 | | Neon Serverless PostgreSQL | $0 (free tier) | | GitHub Actions | $0 (free tier) | -| **Total infrastructure** | **~$6–12/month** | +| **Total infrastructure** | **~$2–8/month** | Anthropic API cost scales with query volume. With the 2-query lifetime cap per user and JSONB result caching, the marginal cost per unique visitor is bounded at approximately $0.01–0.03. diff --git a/backend/Dockerfile b/backend/Dockerfile index 4afc566..404470b 100644 --- a/backend/Dockerfile +++ b/backend/Dockerfile @@ -63,11 +63,11 @@ EXPOSE 8000 HEALTHCHECK \ --interval=30s \ --timeout=10s \ - --start-period=20s \ + --start-period=15s \ --retries=3 \ CMD python3 -c "\ import urllib.request, sys; \ -r = urllib.request.urlopen('http://localhost:8000/ready', timeout=10); \ +r = urllib.request.urlopen('http://localhost:8000/health', timeout=10); \ sys.exit(0 if r.status == 200 else 1)" # Production server — 2 workers, no reload diff --git a/backend/app/db/pool.py b/backend/app/db/pool.py index 2a39805..cbae2c2 100644 --- a/backend/app/db/pool.py +++ b/backend/app/db/pool.py @@ -12,6 +12,7 @@ - All public methods log timing for observability. """ +import asyncio import logging import time from typing import Any, List, Optional @@ -37,9 +38,18 @@ class DatabasePool: def __init__(self) -> None: self._pool: Optional[Pool] = None + self._init_lock = asyncio.Lock() # ── Lifecycle ────────────────────────────────────────────────────────── + async def ensure_ready(self) -> None: + """Create the pool on first use (supports fast /health on cold start).""" + if self._pool is not None: + return + async with self._init_lock: + if self._pool is None: + await self.initialise() + async def initialise(self) -> None: """ Create the asyncpg pool. @@ -99,7 +109,7 @@ async def fetch( Raises: DatabaseError: On any asyncpg or timeout error. """ - self._assert_pool_ready() + await self.ensure_ready() start = time.perf_counter() try: async with self._pool.acquire() as conn: @@ -133,7 +143,7 @@ async def execute( Returns asyncpg status string e.g. 'INSERT 0 1'. """ - self._assert_pool_ready() + await self.ensure_ready() try: async with self._pool.acquire() as conn: return await conn.execute( @@ -152,7 +162,7 @@ async def fetchrow( Fetch a single row. Returns None if no rows match. Useful for INSERT ... RETURNING id queries. """ - self._assert_pool_ready() + await self.ensure_ready() try: async with self._pool.acquire() as conn: record = await conn.fetchrow(query, *args) @@ -162,14 +172,6 @@ async def fetchrow( # ── Private helpers ──────────────────────────────────────────────────── - def _assert_pool_ready(self) -> None: - """Guard against calls before initialise() has run.""" - if self._pool is None: - raise DatabaseError( - "Database pool is not initialised. " - "Ensure initialise() is called during app startup." - ) - @staticmethod async def _set_type_codecs(conn: asyncpg.Connection) -> None: """ diff --git a/backend/main.py b/backend/main.py index c0d933e..a44e7a8 100644 --- a/backend/main.py +++ b/backend/main.py @@ -26,13 +26,10 @@ @asynccontextmanager async def lifespan(app: FastAPI) -> AsyncGenerator[None, None]: - logger.info("InsightIQ starting — initialising database pool...") - await database_pool.initialise() - - # Log CORS origins at startup so you can verify in ACA logs + # Do not block on DB here — ACA scale-from-zero probes /health before Neon wakes. cors_origins = settings.get_all_cors_origins() + logger.info("InsightIQ starting — /health live; DB pool connects on first API call") logger.info(f"CORS allowed origins: {cors_origins}") - logger.info("Startup complete") yield @@ -88,6 +85,7 @@ async def health_check() -> dict: @app.get("/ready", tags=["Health"]) async def readiness_check() -> dict: """Readiness — confirms database connectivity.""" + await database_pool.ensure_ready() await database_pool.execute("SELECT 1") return {"status": "ready", "version": "1.0.0"} diff --git a/frontend/Dockerfile b/frontend/Dockerfile index 7b0c0b8..b6c3793 100644 --- a/frontend/Dockerfile +++ b/frontend/Dockerfile @@ -39,7 +39,7 @@ LABEL org.opencontainers.image.title="insightiq-frontend" # Render nginx template at start (BACKEND_UPSTREAM from ACA env) RUN rm -f /etc/nginx/conf.d/default.conf -RUN apk add --no-cache gettext +RUN apk add --no-cache gettext curl COPY nginx.conf.template /etc/nginx/templates/default.conf.template COPY docker-entrypoint.sh /docker-entrypoint.sh RUN chmod +x /docker-entrypoint.sh @@ -55,7 +55,8 @@ EXPOSE 80 HEALTHCHECK \ --interval=30s \ --timeout=5s \ + --start-period=10s \ --retries=3 \ - CMD wget -qO- http://localhost:80 || exit 1 + CMD curl -fsS http://localhost/nginx-health || exit 1 ENTRYPOINT ["/docker-entrypoint.sh"] diff --git a/frontend/docker-entrypoint.sh b/frontend/docker-entrypoint.sh index 746be31..247f9a7 100644 --- a/frontend/docker-entrypoint.sh +++ b/frontend/docker-entrypoint.sh @@ -8,4 +8,6 @@ envsubst '${BACKEND_UPSTREAM}' \ < /etc/nginx/templates/default.conf.template \ > /etc/nginx/conf.d/default.conf +nginx -t + exec nginx -g 'daemon off;' diff --git a/infra/fix-scale-to-zero.sh b/infra/fix-scale-to-zero.sh new file mode 100755 index 0000000..148033c --- /dev/null +++ b/infra/fix-scale-to-zero.sh @@ -0,0 +1,66 @@ +#!/usr/bin/env bash +# ================================================================= +# InsightIQ — Fix Azure scale-to-zero + failed revision activation +# +# Run when portal shows "Activation failed" or links hang forever. +# +# Prerequisites: +# az login +# az account set --subscription YOUR_SUBSCRIPTION_ID +# +# Usage: +# chmod +x infra/fix-scale-to-zero.sh +# ./infra/fix-scale-to-zero.sh +# ================================================================= + +set -euo pipefail + +RESOURCE_GROUP="${AZURE_RESOURCE_GROUP:-rg-insightiq-prod}" +BACKEND_APP="${BACKEND_APP_NAME:-insightiq-backend}" +FRONTEND_APP="${FRONTEND_APP_NAME:-insightiq-frontend}" + +echo "==================================================" +echo " InsightIQ — scale-to-zero repair" +echo "==================================================" +echo " Resource group : $RESOURCE_GROUP" +echo " Backend app : $BACKEND_APP" +echo " Frontend app : $FRONTEND_APP" +echo "" + +for app in "$BACKEND_APP" "$FRONTEND_APP"; do + echo "--- $app ---" + az containerapp show \ + --name "$app" \ + --resource-group "$RESOURCE_GROUP" \ + --query "{url:properties.configuration.ingress.fqdn,scale:properties.template.scale,running:properties.runningStatus,latest:properties.latestRevisionName}" \ + -o json + echo "Recent revisions:" + az containerapp revision list \ + --name "$app" \ + --resource-group "$RESOURCE_GROUP" \ + --query "[].{name:name,active:properties.active,traffic:properties.trafficWeight,health:properties.healthState,replicas:properties.replicas}" \ + -o table + echo "" +done + +echo "[1/3] Enforcing min-replicas=0 (idle cost) on both apps..." +az containerapp update --name "$BACKEND_APP" --resource-group "$RESOURCE_GROUP" --min-replicas 0 --output none +az containerapp update --name "$FRONTEND_APP" --resource-group "$RESOURCE_GROUP" --min-replicas 0 --output none +echo " ✓ min-replicas=0" + +echo "" +echo "[2/3] Checklist before redeploy:" +echo " • GitHub Packages → insightiq-frontend / insightiq-backend → Package settings → Public" +echo " (or set GHCR_READ_TOKEN secret with read:packages scope)" +echo " • GitHub Secrets: NEON_DATABASE_URL, JWT_SECRET_KEY, ANTHROPIC_API_KEY, GOOGLE_CLIENT_ID" +echo " • GitHub → Actions → CD → Run workflow on main" +echo "" +echo "[3/3] After CD succeeds, test wake-up (first click may take 30–90s):" +BACKEND_FQDN=$(az containerapp show --name "$BACKEND_APP" --resource-group "$RESOURCE_GROUP" --query "properties.configuration.ingress.fqdn" -o tsv) +FRONTEND_FQDN=$(az containerapp show --name "$FRONTEND_APP" --resource-group "$RESOURCE_GROUP" --query "properties.configuration.ingress.fqdn" -o tsv) +echo " Frontend : https://${FRONTEND_FQDN}/" +echo " Backend : https://${BACKEND_FQDN}/health" +echo "" +echo "If a revision still shows 'Activation failed' with 100% traffic," +echo "open Portal → Revisions → deactivate the failed revision, then re-run CD." +echo "=================================================="