From 26a663b1f606a8d9124135ca3c5f0b3a0a73edc9 Mon Sep 17 00:00:00 2001 From: manavgup Date: Wed, 20 May 2026 22:27:46 -0400 Subject: [PATCH 1/2] fix(ci): add production background job E2E verification (#588) Production deploy verification previously only checked that background_mode == "arq" in the health response, which proves config is set but not that jobs actually execute. Add a lightweight ping job that proves the full pipeline: API -> Redis -> ARQ worker -> result. Co-Authored-By: Claude Opus 4.6 (1M context) --- .github/workflows/deploy.yml | 58 ++++-- docs/openapi.yaml | 295 ++++-------------------------- src/wikimind/api/routes/health.py | 84 +++++++++ src/wikimind/jobs/background.py | 37 +++- src/wikimind/jobs/worker.py | 12 +- src/wikimind/middleware/auth.py | 1 + src/wikimind/models/enums.py | 1 + 7 files changed, 203 insertions(+), 285 deletions(-) diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index 6a07a91a..8c5c918a 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -307,16 +307,31 @@ jobs: curl -sf "$BASE/health" | jq -e '.background_mode == "arq"' || { echo "::error::background_mode is not arq — Redis/ARQ not configured"; exit 1; } echo "PASS: background_mode is arq" - # 4. Swagger docs load + # 4. Background job E2E — enqueue a ping job, wait for worker to process it + PING_RESULT=$(curl -sf "$BASE/health/job-ping") || { echo "::error::/health/job-ping request failed"; exit 1; } + PING_STATUS=$(echo "$PING_RESULT" | jq -r '.status') + if [ "$PING_STATUS" = "ok" ]; then + PING_MS=$(echo "$PING_RESULT" | jq -r '.latency_ms') + echo "PASS: background job E2E (${PING_MS}ms)" + elif [ "$PING_STATUS" = "skipped" ]; then + echo "WARN: job-ping skipped (in-process mode)" + else + PING_ERR=$(echo "$PING_RESULT" | jq -r '.error // "unknown"') + echo "::error::Background job E2E failed: $PING_ERR" + echo "$PING_RESULT" | jq . + exit 1 + fi + + # 5. Swagger docs load curl -sf "$BASE/docs" | grep -q "swagger" echo "PASS: docs" - # 5. Auth endpoint responds + # 6. Auth endpoint responds STATUS=$(curl -s -o /dev/null -w "%{http_code}" "$BASE/auth/me") [ "$STATUS" = "200" ] || [ "$STATUS" = "401" ] echo "PASS: auth ($STATUS)" - # 6. Authenticated smoke tests — exchange dev_token for JWT if needed + # 7. Authenticated smoke tests — exchange dev_token for JWT if needed if [ -z "$TOKEN" ]; then echo "::error::STAGING_DEV_TOKEN secret required for functional smoke tests" exit 1 @@ -333,23 +348,23 @@ jobs: curl -sf "$BASE/api/wiki/articles" -H "Authorization: Bearer $TOKEN" | jq -e 'type == "array"' echo "PASS: articles endpoint returns array" - # 7. Sources endpoint + # 8. Sources endpoint curl -sf "$BASE/api/ingest/sources" -H "Authorization: Bearer $TOKEN" | jq -e 'type == "array"' echo "PASS: sources endpoint returns array" - # 8. Admin stats (if available) + # 9. Admin stats (if available) ADMIN=$(curl -s -o /dev/null -w "%{http_code}" "$BASE/api/admin/stats" -H "Authorization: Bearer $TOKEN") echo "INFO: admin stats responded $ADMIN" - # 9. Concepts endpoint — MUST return array + # 10. Concepts endpoint — MUST return array curl -sf "$BASE/api/wiki/concepts" -H "Authorization: Bearer $TOKEN" | jq -e 'type == "array"' || { echo "::error::/api/concepts did not return array"; exit 1; } echo "PASS: concepts" - # 10. Compilation schemas — MUST return array + # 11. Compilation schemas — MUST return array curl -sf "$BASE/api/compilation-schemas" -H "Authorization: Bearer $TOKEN" | jq -e 'type == "array"' || { echo "::error::/api/compilation-schemas did not return array"; exit 1; } echo "PASS: compilation-schemas" - echo "Staging smoke tests complete (10/10 passed)" + echo "Staging smoke tests complete (11/11 passed)" - name: 🚨 Create alert on failure if: failure() @@ -505,16 +520,31 @@ jobs: curl -sf "$BASE/health" | jq -e '.background_mode == "arq"' || { echo "::error::background_mode is not arq — Redis/ARQ not configured"; exit 1; } echo "PASS: background_mode is arq" - # 4. Swagger docs load + # 4. Background job E2E — enqueue a ping job, wait for worker to process it + PING_RESULT=$(curl -sf "$BASE/health/job-ping") || { echo "::error::/health/job-ping request failed"; exit 1; } + PING_STATUS=$(echo "$PING_RESULT" | jq -r '.status') + if [ "$PING_STATUS" = "ok" ]; then + PING_MS=$(echo "$PING_RESULT" | jq -r '.latency_ms') + echo "PASS: background job E2E (${PING_MS}ms)" + elif [ "$PING_STATUS" = "skipped" ]; then + echo "WARN: job-ping skipped (in-process mode)" + else + PING_ERR=$(echo "$PING_RESULT" | jq -r '.error // "unknown"') + echo "::error::Background job E2E failed: $PING_ERR" + echo "$PING_RESULT" | jq . + exit 1 + fi + + # 5. Swagger docs load curl -sf "$BASE/docs" | grep -q "swagger" echo "PASS: docs" - # 5. Auth endpoint responds + # 6. Auth endpoint responds STATUS=$(curl -s -o /dev/null -w "%{http_code}" "$BASE/auth/me") [ "$STATUS" = "200" ] || [ "$STATUS" = "401" ] echo "PASS: auth ($STATUS)" - # 6. Authenticated checks — exchange dev_token for JWT + # 7. Authenticated checks — exchange dev_token for JWT if [ -z "$TOKEN" ]; then echo "::error::PROD_DEV_TOKEN or STAGING_DEV_TOKEN secret required for production verification" exit 1 @@ -536,15 +566,15 @@ jobs: curl -sf "$BASE/api/ingest/sources" -H "Authorization: Bearer $TOKEN" | jq -e 'type == "array"' echo "PASS: sources endpoint returns array" - # 7. Concepts endpoint (new tables from #546) + # 8. Concepts endpoint (new tables from #546) curl -sf "$BASE/api/wiki/concepts" -H "Authorization: Bearer $TOKEN" | jq -e 'type == "array"' || { echo "::error::/api/concepts failed"; exit 1; } echo "PASS: concepts" - # 8. Compilation schemas + # 9. Compilation schemas curl -sf "$BASE/api/compilation-schemas" -H "Authorization: Bearer $TOKEN" | jq -e 'type == "array"' || { echo "::error::/api/compilation-schemas failed"; exit 1; } echo "PASS: compilation-schemas" - # 9. Admin stats responds (may be 403 if token user is not admin — that's OK) + # 10. Admin stats responds (may be 403 if token user is not admin — that's OK) ADMIN=$(curl -s -o /dev/null -w "%{http_code}" "$BASE/api/admin/stats" -H "Authorization: Bearer $TOKEN") echo "INFO: admin stats responded $ADMIN" diff --git a/docs/openapi.yaml b/docs/openapi.yaml index 3bb87824..f5ce6492 100644 --- a/docs/openapi.yaml +++ b/docs/openapi.yaml @@ -3903,90 +3903,6 @@ paths: application/json: schema: $ref: '#/components/schemas/HTTPValidationError' - /api/billing/plans: - get: - tags: - - Billing - summary: List Plans - description: List all active billing plans (public, no auth required). - operationId: list_plans_api_billing_plans_get - responses: - '200': - description: Successful Response - content: - application/json: - schema: - items: - $ref: '#/components/schemas/PlanResponse' - type: array - title: Response List Plans Api Billing Plans Get - /api/billing/usage: - get: - tags: - - Billing - summary: Get Usage - description: Get current resource usage vs plan limits. - operationId: get_usage_api_billing_usage_get - responses: - '200': - description: Successful Response - content: - application/json: - schema: - $ref: '#/components/schemas/UsageResponse' - /api/billing/checkout: - post: - tags: - - Billing - summary: Create Checkout - description: Create a Lemon Squeezy checkout session for plan upgrade. - operationId: create_checkout_api_billing_checkout_post - requestBody: - content: - application/json: - schema: - $ref: '#/components/schemas/CheckoutRequest' - required: true - responses: - '200': - description: Successful Response - content: - application/json: - schema: - $ref: '#/components/schemas/CheckoutResponse' - '422': - description: Validation Error - content: - application/json: - schema: - $ref: '#/components/schemas/HTTPValidationError' - /api/billing/portal: - get: - tags: - - Billing - summary: Get Portal - description: Get Lemon Squeezy customer portal URL for subscription management. - operationId: get_portal_api_billing_portal_get - responses: - '200': - description: Successful Response - content: - application/json: - schema: - $ref: '#/components/schemas/PortalResponse' - /api/billing/webhook: - post: - tags: - - Billing - summary: Handle Webhook - description: Process Lemon Squeezy webhook with insert-first idempotency. - operationId: handle_webhook_api_billing_webhook_post - responses: - '200': - description: Successful Response - content: - application/json: - schema: {} /public/articles/{token}: get: tags: @@ -4071,6 +3987,36 @@ paths: additionalProperties: true type: object title: Response Deep Health Check Health Deep Get + /health/job-ping: + get: + tags: + - Admin + summary: Job Ping + description: 'Enqueue a lightweight ping job and wait for its result. + + + Proves the full background job pipeline works end-to-end: + + API -> Redis -> ARQ worker -> result. + + + Returns ``{"status": "ok", ...}`` when the job completes within the + + timeout, or ``{"status": "error", ...}`` on failure or timeout. + + In dev mode (no Redis), returns ``{"status": "skipped"}`` since + + there is no ARQ worker to verify.' + operationId: job_ping_health_job_ping_get + responses: + '200': + description: Successful Response + content: + application/json: + schema: + additionalProperties: true + type: object + title: Response Job Ping Health Job Ping Get /auth/login/{provider}: get: tags: @@ -5512,26 +5458,6 @@ components: - discarded title: CaptureStatus description: Lifecycle status of a captured item. - CheckoutRequest: - properties: - plan_id: - type: string - title: Plan Id - type: object - required: - - plan_id - title: CheckoutRequest - description: Request body for creating a checkout session. - CheckoutResponse: - properties: - checkout_url: - type: string - title: Checkout Url - type: object - required: - - checkout_url - title: CheckoutResponse - description: Checkout URL response. CitationArticleRef: properties: slug: @@ -6996,6 +6922,7 @@ components: - sync_push - sync_pull - poll_rss_feeds + - ping title: JobType description: Type of async job. LLMSettingsResponse: @@ -7557,92 +7484,6 @@ components: - description title: PipelineStep description: A single step in the source processing pipeline. - PlanResponse: - properties: - id: - type: string - title: Id - name: - type: string - title: Name - display_name: - type: string - title: Display Name - price_cents: - type: integer - title: Price Cents - billing_interval: - anyOf: - - type: string - - type: 'null' - title: Billing Interval - max_sources: - anyOf: - - type: integer - - type: 'null' - title: Max Sources - max_articles: - anyOf: - - type: integer - - type: 'null' - title: Max Articles - max_queries_per_day: - anyOf: - - type: integer - - type: 'null' - title: Max Queries Per Day - max_storage_bytes: - anyOf: - - type: integer - - type: 'null' - title: Max Storage Bytes - max_active_shares: - anyOf: - - type: integer - - type: 'null' - title: Max Active Shares - allowed_exports: - items: - type: string - type: array - title: Allowed Exports - mcp_enabled: - type: boolean - title: Mcp Enabled - byok_allowed: - type: boolean - title: Byok Allowed - sort_order: - type: integer - title: Sort Order - type: object - required: - - id - - name - - display_name - - price_cents - - billing_interval - - max_sources - - max_articles - - max_queries_per_day - - max_storage_bytes - - max_active_shares - - allowed_exports - - mcp_enabled - - byok_allowed - - sort_order - title: PlanResponse - description: Public billing plan details. - PortalResponse: - properties: - portal_url: - type: string - title: Portal Url - type: object - required: - - portal_url - title: PortalResponse - description: Customer portal URL response. ProviderDetail: properties: enabled: @@ -9275,80 +9116,6 @@ components: type: object title: UpdateCompilationSchemaRequest description: Request to update a compilation schema. - UsageResponse: - properties: - plan_name: - type: string - title: Plan Name - plan_display_name: - type: string - title: Plan Display Name - sources: - type: integer - title: Sources - sources_limit: - anyOf: - - type: integer - - type: 'null' - title: Sources Limit - articles: - type: integer - title: Articles - articles_limit: - anyOf: - - type: integer - - type: 'null' - title: Articles Limit - storage_bytes: - type: integer - title: Storage Bytes - storage_limit: - anyOf: - - type: integer - - type: 'null' - title: Storage Limit - queries_today: - type: integer - title: Queries Today - queries_limit: - anyOf: - - type: integer - - type: 'null' - title: Queries Limit - active_shares: - type: integer - title: Active Shares - shares_limit: - anyOf: - - type: integer - - type: 'null' - title: Shares Limit - llm_spend_cents_today: - type: integer - title: Llm Spend Cents Today - llm_spend_limit: - anyOf: - - type: integer - - type: 'null' - title: Llm Spend Limit - type: object - required: - - plan_name - - plan_display_name - - sources - - sources_limit - - articles - - articles_limit - - storage_bytes - - storage_limit - - queries_today - - queries_limit - - active_shares - - shares_limit - - llm_spend_cents_today - - llm_spend_limit - title: UsageResponse - description: Current resource usage vs plan limits. UserProfileResponse: properties: id: diff --git a/src/wikimind/api/routes/health.py b/src/wikimind/api/routes/health.py index 671ee33d..4607d26c 100644 --- a/src/wikimind/api/routes/health.py +++ b/src/wikimind/api/routes/health.py @@ -4,10 +4,15 @@ availability, stuck source processing jobs, Redis connectivity, and ARQ queue depth. Returns a structured JSON response with per-check status and overall system health. + +The ``/health/job-ping`` endpoint enqueues a lightweight ping job and +polls for its result to verify the full background job pipeline: +API -> Redis -> ARQ worker -> result. """ from __future__ import annotations +import asyncio import time from datetime import timedelta from typing import Any @@ -15,6 +20,8 @@ import structlog from alembic.config import Config as AlembicConfig from alembic.script import ScriptDirectory +from arq.jobs import Job as ArqJob +from arq.jobs import JobStatus as ArqJobStatus from fastapi import APIRouter from redis.asyncio import Redis from sqlalchemy import text as sa_text @@ -186,3 +193,80 @@ async def deep_health_check() -> dict[str, Any]: "status": _overall_status(checks), "checks": checks, } + + +# --------------------------------------------------------------------------- +# Job ping — end-to-end background job verification +# --------------------------------------------------------------------------- + +# Maximum time to wait for the ping job to complete (seconds). +_JOB_PING_TIMEOUT = 30 +# Interval between result polls (seconds). +_JOB_PING_POLL_INTERVAL = 1 + + +@router.get("/health/job-ping") +async def job_ping() -> dict[str, Any]: + """Enqueue a lightweight ping job and wait for its result. + + Proves the full background job pipeline works end-to-end: + API -> Redis -> ARQ worker -> result. + + Returns ``{"status": "ok", ...}`` when the job completes within the + timeout, or ``{"status": "error", ...}`` on failure or timeout. + In dev mode (no Redis), returns ``{"status": "skipped"}`` since + there is no ARQ worker to verify. + """ + from wikimind.jobs.background import get_background_compiler # noqa: PLC0415 + + compiler = get_background_compiler() + + if not compiler.is_prod: + return {"status": "skipped", "reason": "in-process mode (no Redis)"} + + start = time.monotonic() + try: + job_id = await compiler.schedule_ping() + if job_id is None: + return {"status": "error", "error": "Failed to enqueue ping job"} + + # Poll ARQ for the job result + pool = await compiler._get_pool() + for _ in range(int(_JOB_PING_TIMEOUT / _JOB_PING_POLL_INTERVAL)): + arq_job = ArqJob(job_id, redis=pool) + status = await arq_job.status() + if status == ArqJobStatus.complete: + info = await arq_job.result_info() + latency_ms = round((time.monotonic() - start) * 1000) + return { + "status": "ok", + "job_id": job_id, + "result": info.result if info else None, + "latency_ms": latency_ms, + } + if status == ArqJobStatus.not_found: + # Job disappeared — worker may have crashed + latency_ms = round((time.monotonic() - start) * 1000) + return { + "status": "error", + "job_id": job_id, + "error": "Job not found — worker may not be running", + "latency_ms": latency_ms, + } + await asyncio.sleep(_JOB_PING_POLL_INTERVAL) + + latency_ms = round((time.monotonic() - start) * 1000) + return { + "status": "error", + "job_id": job_id, + "error": f"Ping job did not complete within {_JOB_PING_TIMEOUT}s", + "latency_ms": latency_ms, + } + except Exception as exc: + latency_ms = round((time.monotonic() - start) * 1000) + log.warning("health: job-ping failed", error=str(exc)) + return { + "status": "error", + "error": str(exc), + "latency_ms": latency_ms, + } diff --git a/src/wikimind/jobs/background.py b/src/wikimind/jobs/background.py index 252c567a..ca78d6c3 100644 --- a/src/wikimind/jobs/background.py +++ b/src/wikimind/jobs/background.py @@ -72,6 +72,35 @@ def is_prod(self) -> bool: """Return True when a real Redis URL is configured.""" return self._redis_url is not None + async def schedule_ping(self) -> str | None: + """Enqueue a lightweight ping job and return the ARQ job ID. + + Returns the ARQ job ID on success (prod mode) or ``None`` + in dev mode where no Redis is available. + + Returns: + The ARQ job ID string, or None in dev mode. + """ + if not self.is_prod: + return None + + pool = await self._get_pool() + job = await pool.enqueue_job("ping") + return job.job_id if job else None + + async def _get_pool(self) -> ArqRedis: + """Return the cached ARQ Redis pool, creating it lazily. + + Returns: + The ArqRedis connection pool. + """ + if self._arq_pool is None: + async with self._pool_lock: + if self._arq_pool is None: + settings = RedisSettings.from_dsn(self._redis_url) # type: ignore[arg-type] + self._arq_pool = await create_pool(settings) + return self._arq_pool + async def schedule_compile( self, source_id: str, @@ -198,12 +227,8 @@ async def _enqueue_arq(self, func_name: str, *args: object) -> None: overhead. An asyncio.Lock guards initialization to prevent duplicate pools under concurrent calls. """ - if self._arq_pool is None: - async with self._pool_lock: - if self._arq_pool is None: # double-check after acquiring lock - settings = RedisSettings.from_dsn(self._redis_url) # type: ignore[arg-type] - self._arq_pool = await create_pool(settings) - await self._arq_pool.enqueue_job(func_name, *args) + pool = await self._get_pool() + await pool.enqueue_job(func_name, *args) async def close(self) -> None: """Close the cached ARQ Redis pool, if any.""" diff --git a/src/wikimind/jobs/worker.py b/src/wikimind/jobs/worker.py index d88e2fcd..777a458f 100644 --- a/src/wikimind/jobs/worker.py +++ b/src/wikimind/jobs/worker.py @@ -226,6 +226,16 @@ async def _on_chunk_progress(message: str) -> None: await sweep_wikilinks(ctx, user_id=user_id) +async def ping(_ctx) -> str: + """Lightweight no-op job that proves the full ARQ pipeline works. + + Returns ``"pong"`` immediately. Used by the production deploy + verification step to confirm API -> Redis -> ARQ worker -> result. + """ + log.info("ping job executed") + return "pong" + + async def compile_source( ctx, source_id: str, @@ -683,7 +693,7 @@ def get_redis_settings() -> RedisSettings: class WorkerSettings: """ARQ worker configuration for production (requires Redis).""" - functions: ClassVar[list] = [compile_source, lint_wiki, recompile_article, sweep_wikilinks] + functions: ClassVar[list] = [ping, compile_source, lint_wiki, recompile_article, sweep_wikilinks] redis_settings = get_redis_settings() _worker_cfg = get_settings().worker max_jobs = _worker_cfg.max_jobs diff --git a/src/wikimind/middleware/auth.py b/src/wikimind/middleware/auth.py index 0827b2fb..4499f5f4 100644 --- a/src/wikimind/middleware/auth.py +++ b/src/wikimind/middleware/auth.py @@ -21,6 +21,7 @@ EXEMPT_PATHS = { "/health", "/health/deep", + "/health/job-ping", "/metrics", "/docs", "/openapi.json", diff --git a/src/wikimind/models/enums.py b/src/wikimind/models/enums.py index 1600be75..b6b1bdf7 100644 --- a/src/wikimind/models/enums.py +++ b/src/wikimind/models/enums.py @@ -122,6 +122,7 @@ class JobType(StrEnum): SYNC_PUSH = "sync_push" SYNC_PULL = "sync_pull" POLL_RSS_FEEDS = "poll_rss_feeds" + PING = "ping" class JobStatus(StrEnum): From 1c10395772a5df95763816fa412aa3f9a906e4f4 Mon Sep 17 00:00:00 2001 From: manavgup Date: Sat, 23 May 2026 14:05:23 -0400 Subject: [PATCH 2/2] fix(ci): address review feedback on job E2E verification MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Remove /health/job-ping from auth EXEMPT_PATHS so it requires authentication (prevents unauthenticated abuse of side-effecting endpoint) - Add Depends(get_current_user_id) to the job_ping route handler - Move job-ping check after auth token exchange in both staging and production verification steps so the Bearer token is available - Remove acceptance of status=="skipped" — contradicts the background_mode == "arq" check that runs first (if mode is arq, skipped is an error) - Add retry with backoff (5 attempts, 5/10/15/20/25s delays) to handle worker startup delays after deploy Co-Authored-By: Claude Opus 4.6 (1M context) --- .github/workflows/deploy.yml | 114 ++++++++----- docs/openapi.yaml | 268 +++++++++++++++++++++++++++++- src/wikimind/api/routes/health.py | 11 +- src/wikimind/middleware/auth.py | 1 - 4 files changed, 346 insertions(+), 48 deletions(-) diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index 8c5c918a..148aaab1 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -307,31 +307,16 @@ jobs: curl -sf "$BASE/health" | jq -e '.background_mode == "arq"' || { echo "::error::background_mode is not arq — Redis/ARQ not configured"; exit 1; } echo "PASS: background_mode is arq" - # 4. Background job E2E — enqueue a ping job, wait for worker to process it - PING_RESULT=$(curl -sf "$BASE/health/job-ping") || { echo "::error::/health/job-ping request failed"; exit 1; } - PING_STATUS=$(echo "$PING_RESULT" | jq -r '.status') - if [ "$PING_STATUS" = "ok" ]; then - PING_MS=$(echo "$PING_RESULT" | jq -r '.latency_ms') - echo "PASS: background job E2E (${PING_MS}ms)" - elif [ "$PING_STATUS" = "skipped" ]; then - echo "WARN: job-ping skipped (in-process mode)" - else - PING_ERR=$(echo "$PING_RESULT" | jq -r '.error // "unknown"') - echo "::error::Background job E2E failed: $PING_ERR" - echo "$PING_RESULT" | jq . - exit 1 - fi - - # 5. Swagger docs load + # 4. Swagger docs load curl -sf "$BASE/docs" | grep -q "swagger" echo "PASS: docs" - # 6. Auth endpoint responds + # 5. Auth endpoint responds STATUS=$(curl -s -o /dev/null -w "%{http_code}" "$BASE/auth/me") [ "$STATUS" = "200" ] || [ "$STATUS" = "401" ] echo "PASS: auth ($STATUS)" - # 7. Authenticated smoke tests — exchange dev_token for JWT if needed + # 6. Authenticated smoke tests — exchange dev_token for JWT if needed if [ -z "$TOKEN" ]; then echo "::error::STAGING_DEV_TOKEN secret required for functional smoke tests" exit 1 @@ -348,22 +333,51 @@ jobs: curl -sf "$BASE/api/wiki/articles" -H "Authorization: Bearer $TOKEN" | jq -e 'type == "array"' echo "PASS: articles endpoint returns array" - # 8. Sources endpoint + # 7. Sources endpoint curl -sf "$BASE/api/ingest/sources" -H "Authorization: Bearer $TOKEN" | jq -e 'type == "array"' echo "PASS: sources endpoint returns array" - # 9. Admin stats (if available) + # 8. Admin stats (if available) ADMIN=$(curl -s -o /dev/null -w "%{http_code}" "$BASE/api/admin/stats" -H "Authorization: Bearer $TOKEN") echo "INFO: admin stats responded $ADMIN" - # 10. Concepts endpoint — MUST return array + # 9. Concepts endpoint — MUST return array curl -sf "$BASE/api/wiki/concepts" -H "Authorization: Bearer $TOKEN" | jq -e 'type == "array"' || { echo "::error::/api/concepts did not return array"; exit 1; } echo "PASS: concepts" - # 11. Compilation schemas — MUST return array + # 10. Compilation schemas — MUST return array curl -sf "$BASE/api/compilation-schemas" -H "Authorization: Bearer $TOKEN" | jq -e 'type == "array"' || { echo "::error::/api/compilation-schemas did not return array"; exit 1; } echo "PASS: compilation-schemas" + # 11. Background job E2E — enqueue a ping job, wait for worker to process it. + # Requires auth (side-effecting endpoint). Retry with backoff to handle + # worker startup delays after deploy. + PING_OK=false + for attempt in 1 2 3 4 5; do + PING_RESULT=$(curl -sf "$BASE/health/job-ping" \ + -H "Authorization: Bearer $TOKEN") || { + echo "Attempt $attempt: /health/job-ping request failed, retrying in $((attempt * 5))s..." + sleep $((attempt * 5)) + continue + } + PING_STATUS=$(echo "$PING_RESULT" | jq -r '.status') + if [ "$PING_STATUS" = "ok" ]; then + PING_MS=$(echo "$PING_RESULT" | jq -r '.latency_ms') + echo "PASS: background job E2E (${PING_MS}ms, attempt $attempt)" + PING_OK=true + break + else + PING_ERR=$(echo "$PING_RESULT" | jq -r '.error // "unknown"') + echo "Attempt $attempt: job-ping status=$PING_STATUS error=$PING_ERR, retrying in $((attempt * 5))s..." + sleep $((attempt * 5)) + fi + done + if [ "$PING_OK" != "true" ]; then + echo "::error::Background job E2E failed after 5 attempts" + echo "$PING_RESULT" | jq . 2>/dev/null || true + exit 1 + fi + echo "Staging smoke tests complete (11/11 passed)" - name: 🚨 Create alert on failure @@ -520,31 +534,16 @@ jobs: curl -sf "$BASE/health" | jq -e '.background_mode == "arq"' || { echo "::error::background_mode is not arq — Redis/ARQ not configured"; exit 1; } echo "PASS: background_mode is arq" - # 4. Background job E2E — enqueue a ping job, wait for worker to process it - PING_RESULT=$(curl -sf "$BASE/health/job-ping") || { echo "::error::/health/job-ping request failed"; exit 1; } - PING_STATUS=$(echo "$PING_RESULT" | jq -r '.status') - if [ "$PING_STATUS" = "ok" ]; then - PING_MS=$(echo "$PING_RESULT" | jq -r '.latency_ms') - echo "PASS: background job E2E (${PING_MS}ms)" - elif [ "$PING_STATUS" = "skipped" ]; then - echo "WARN: job-ping skipped (in-process mode)" - else - PING_ERR=$(echo "$PING_RESULT" | jq -r '.error // "unknown"') - echo "::error::Background job E2E failed: $PING_ERR" - echo "$PING_RESULT" | jq . - exit 1 - fi - - # 5. Swagger docs load + # 4. Swagger docs load curl -sf "$BASE/docs" | grep -q "swagger" echo "PASS: docs" - # 6. Auth endpoint responds + # 5. Auth endpoint responds STATUS=$(curl -s -o /dev/null -w "%{http_code}" "$BASE/auth/me") [ "$STATUS" = "200" ] || [ "$STATUS" = "401" ] echo "PASS: auth ($STATUS)" - # 7. Authenticated checks — exchange dev_token for JWT + # 6. Authenticated checks — exchange dev_token for JWT if [ -z "$TOKEN" ]; then echo "::error::PROD_DEV_TOKEN or STAGING_DEV_TOKEN secret required for production verification" exit 1 @@ -566,18 +565,47 @@ jobs: curl -sf "$BASE/api/ingest/sources" -H "Authorization: Bearer $TOKEN" | jq -e 'type == "array"' echo "PASS: sources endpoint returns array" - # 8. Concepts endpoint (new tables from #546) + # 7. Concepts endpoint (new tables from #546) curl -sf "$BASE/api/wiki/concepts" -H "Authorization: Bearer $TOKEN" | jq -e 'type == "array"' || { echo "::error::/api/concepts failed"; exit 1; } echo "PASS: concepts" - # 9. Compilation schemas + # 8. Compilation schemas curl -sf "$BASE/api/compilation-schemas" -H "Authorization: Bearer $TOKEN" | jq -e 'type == "array"' || { echo "::error::/api/compilation-schemas failed"; exit 1; } echo "PASS: compilation-schemas" - # 10. Admin stats responds (may be 403 if token user is not admin — that's OK) + # 9. Admin stats responds (may be 403 if token user is not admin — that's OK) ADMIN=$(curl -s -o /dev/null -w "%{http_code}" "$BASE/api/admin/stats" -H "Authorization: Bearer $TOKEN") echo "INFO: admin stats responded $ADMIN" + # 10. Background job E2E — enqueue a ping job, wait for worker to process it. + # Requires auth (side-effecting endpoint). Retry with backoff to handle + # worker startup delays after deploy. + PING_OK=false + for attempt in 1 2 3 4 5; do + PING_RESULT=$(curl -sf "$BASE/health/job-ping" \ + -H "Authorization: Bearer $TOKEN") || { + echo "Attempt $attempt: /health/job-ping request failed, retrying in $((attempt * 5))s..." + sleep $((attempt * 5)) + continue + } + PING_STATUS=$(echo "$PING_RESULT" | jq -r '.status') + if [ "$PING_STATUS" = "ok" ]; then + PING_MS=$(echo "$PING_RESULT" | jq -r '.latency_ms') + echo "PASS: background job E2E (${PING_MS}ms, attempt $attempt)" + PING_OK=true + break + else + PING_ERR=$(echo "$PING_RESULT" | jq -r '.error // "unknown"') + echo "Attempt $attempt: job-ping status=$PING_STATUS error=$PING_ERR, retrying in $((attempt * 5))s..." + sleep $((attempt * 5)) + fi + done + if [ "$PING_OK" != "true" ]; then + echo "::error::Background job E2E failed after 5 attempts" + echo "$PING_RESULT" | jq . 2>/dev/null || true + exit 1 + fi + echo "Production functional verification complete" # ----------------------------------------------------------- diff --git a/docs/openapi.yaml b/docs/openapi.yaml index f5ce6492..3e294e9f 100644 --- a/docs/openapi.yaml +++ b/docs/openapi.yaml @@ -3903,6 +3903,90 @@ paths: application/json: schema: $ref: '#/components/schemas/HTTPValidationError' + /api/billing/plans: + get: + tags: + - Billing + summary: List Plans + description: List all active billing plans (public, no auth required). + operationId: list_plans_api_billing_plans_get + responses: + '200': + description: Successful Response + content: + application/json: + schema: + items: + $ref: '#/components/schemas/PlanResponse' + type: array + title: Response List Plans Api Billing Plans Get + /api/billing/usage: + get: + tags: + - Billing + summary: Get Usage + description: Get current resource usage vs plan limits. + operationId: get_usage_api_billing_usage_get + responses: + '200': + description: Successful Response + content: + application/json: + schema: + $ref: '#/components/schemas/UsageResponse' + /api/billing/checkout: + post: + tags: + - Billing + summary: Create Checkout + description: Create a Lemon Squeezy checkout session for plan upgrade. + operationId: create_checkout_api_billing_checkout_post + requestBody: + content: + application/json: + schema: + $ref: '#/components/schemas/CheckoutRequest' + required: true + responses: + '200': + description: Successful Response + content: + application/json: + schema: + $ref: '#/components/schemas/CheckoutResponse' + '422': + description: Validation Error + content: + application/json: + schema: + $ref: '#/components/schemas/HTTPValidationError' + /api/billing/portal: + get: + tags: + - Billing + summary: Get Portal + description: Get Lemon Squeezy customer portal URL for subscription management. + operationId: get_portal_api_billing_portal_get + responses: + '200': + description: Successful Response + content: + application/json: + schema: + $ref: '#/components/schemas/PortalResponse' + /api/billing/webhook: + post: + tags: + - Billing + summary: Handle Webhook + description: Process Lemon Squeezy webhook with insert-first idempotency. + operationId: handle_webhook_api_billing_webhook_post + responses: + '200': + description: Successful Response + content: + application/json: + schema: {} /public/articles/{token}: get: tags: @@ -3997,7 +4081,9 @@ paths: Proves the full background job pipeline works end-to-end: - API -> Redis -> ARQ worker -> result. + API -> Redis -> ARQ worker -> result. Requires authentication + + because it enqueues a real job (side effect). Returns ``{"status": "ok", ...}`` when the job completes within the @@ -5458,6 +5544,26 @@ components: - discarded title: CaptureStatus description: Lifecycle status of a captured item. + CheckoutRequest: + properties: + plan_id: + type: string + title: Plan Id + type: object + required: + - plan_id + title: CheckoutRequest + description: Request body for creating a checkout session. + CheckoutResponse: + properties: + checkout_url: + type: string + title: Checkout Url + type: object + required: + - checkout_url + title: CheckoutResponse + description: Checkout URL response. CitationArticleRef: properties: slug: @@ -7484,6 +7590,92 @@ components: - description title: PipelineStep description: A single step in the source processing pipeline. + PlanResponse: + properties: + id: + type: string + title: Id + name: + type: string + title: Name + display_name: + type: string + title: Display Name + price_cents: + type: integer + title: Price Cents + billing_interval: + anyOf: + - type: string + - type: 'null' + title: Billing Interval + max_sources: + anyOf: + - type: integer + - type: 'null' + title: Max Sources + max_articles: + anyOf: + - type: integer + - type: 'null' + title: Max Articles + max_queries_per_day: + anyOf: + - type: integer + - type: 'null' + title: Max Queries Per Day + max_storage_bytes: + anyOf: + - type: integer + - type: 'null' + title: Max Storage Bytes + max_active_shares: + anyOf: + - type: integer + - type: 'null' + title: Max Active Shares + allowed_exports: + items: + type: string + type: array + title: Allowed Exports + mcp_enabled: + type: boolean + title: Mcp Enabled + byok_allowed: + type: boolean + title: Byok Allowed + sort_order: + type: integer + title: Sort Order + type: object + required: + - id + - name + - display_name + - price_cents + - billing_interval + - max_sources + - max_articles + - max_queries_per_day + - max_storage_bytes + - max_active_shares + - allowed_exports + - mcp_enabled + - byok_allowed + - sort_order + title: PlanResponse + description: Public billing plan details. + PortalResponse: + properties: + portal_url: + type: string + title: Portal Url + type: object + required: + - portal_url + title: PortalResponse + description: Customer portal URL response. ProviderDetail: properties: enabled: @@ -9116,6 +9308,80 @@ components: type: object title: UpdateCompilationSchemaRequest description: Request to update a compilation schema. + UsageResponse: + properties: + plan_name: + type: string + title: Plan Name + plan_display_name: + type: string + title: Plan Display Name + sources: + type: integer + title: Sources + sources_limit: + anyOf: + - type: integer + - type: 'null' + title: Sources Limit + articles: + type: integer + title: Articles + articles_limit: + anyOf: + - type: integer + - type: 'null' + title: Articles Limit + storage_bytes: + type: integer + title: Storage Bytes + storage_limit: + anyOf: + - type: integer + - type: 'null' + title: Storage Limit + queries_today: + type: integer + title: Queries Today + queries_limit: + anyOf: + - type: integer + - type: 'null' + title: Queries Limit + active_shares: + type: integer + title: Active Shares + shares_limit: + anyOf: + - type: integer + - type: 'null' + title: Shares Limit + llm_spend_cents_today: + type: integer + title: Llm Spend Cents Today + llm_spend_limit: + anyOf: + - type: integer + - type: 'null' + title: Llm Spend Limit + type: object + required: + - plan_name + - plan_display_name + - sources + - sources_limit + - articles + - articles_limit + - storage_bytes + - storage_limit + - queries_today + - queries_limit + - active_shares + - shares_limit + - llm_spend_cents_today + - llm_spend_limit + title: UsageResponse + description: Current resource usage vs plan limits. UserProfileResponse: properties: id: diff --git a/src/wikimind/api/routes/health.py b/src/wikimind/api/routes/health.py index 4607d26c..91cf604b 100644 --- a/src/wikimind/api/routes/health.py +++ b/src/wikimind/api/routes/health.py @@ -22,12 +22,13 @@ from alembic.script import ScriptDirectory from arq.jobs import Job as ArqJob from arq.jobs import JobStatus as ArqJobStatus -from fastapi import APIRouter +from fastapi import APIRouter, Depends from redis.asyncio import Redis from sqlalchemy import text as sa_text from sqlmodel import select from wikimind._datetime import utcnow_naive +from wikimind.api.deps import get_current_user_id from wikimind.config import get_settings from wikimind.database import get_session_factory from wikimind.models import IngestStatus, Source @@ -205,12 +206,16 @@ async def deep_health_check() -> dict[str, Any]: _JOB_PING_POLL_INTERVAL = 1 +# NOTE: This endpoint intentionally has side effects — it enqueues a real +# (lightweight) ARQ job to verify the job queue is functional end-to-end. +# Requires authentication to prevent unauthenticated abuse. @router.get("/health/job-ping") -async def job_ping() -> dict[str, Any]: +async def job_ping(_user_id: str = Depends(get_current_user_id)) -> dict[str, Any]: """Enqueue a lightweight ping job and wait for its result. Proves the full background job pipeline works end-to-end: - API -> Redis -> ARQ worker -> result. + API -> Redis -> ARQ worker -> result. Requires authentication + because it enqueues a real job (side effect). Returns ``{"status": "ok", ...}`` when the job completes within the timeout, or ``{"status": "error", ...}`` on failure or timeout. diff --git a/src/wikimind/middleware/auth.py b/src/wikimind/middleware/auth.py index 4499f5f4..0827b2fb 100644 --- a/src/wikimind/middleware/auth.py +++ b/src/wikimind/middleware/auth.py @@ -21,7 +21,6 @@ EXEMPT_PATHS = { "/health", "/health/deep", - "/health/job-ping", "/metrics", "/docs", "/openapi.json",