From bc87db6354fd766791f52fec3b43c3404d5d990a Mon Sep 17 00:00:00 2001 From: Atti Ur Rehman Date: Thu, 18 Jun 2026 14:45:40 +0300 Subject: [PATCH] Align docs with prod defaults and wire quota override to Lambda. Document 5 lifetime runs and ~12h state TTL (10h approval + 2h grace), set opscanvas-pi.vercel.app as the canonical frontend/CORS URL, and pass ACCOUNT_RUN_LIMIT and QUOTA_OVERRIDE_EMAIL through SAM/CD. Improve the dashboard quota badge (X of Y, loading/error states) so production override and exhaustion are visible. --- .env.example | 4 +- .github/workflows/cd-backend.yml | 7 +- README.md | 184 +++++++++++++++++++++------ backend/app/core/run_quota.py | 8 ++ backend/template.yaml | 13 +- docs/ACCEPTANCE.md | 4 +- docs/COST.md | 2 +- frontend/src/pages/DashboardPage.tsx | 40 +++++- infra/secrets-reference.sh | 11 +- 9 files changed, 220 insertions(+), 53 deletions(-) diff --git a/.env.example b/.env.example index a6cae7f..2993689 100644 --- a/.env.example +++ b/.env.example @@ -36,8 +36,8 @@ TRIAGE_CONFIDENCE_THRESHOLD=0.6 # below -> route to human, no auto-close # Layer 4 — Cost MAX_INCIDENT_USD=0.50 # circuit breaker per incident -ACCOUNT_RUN_LIMIT=3 # lifetime runs per Google account -# Optional dev override — format email:limit (set in .env only, never commit) +ACCOUNT_RUN_LIMIT=5 # lifetime runs per Google account (code default: 5) +# Optional — production: set GitHub secret QUOTA_OVERRIDE_EMAIL=you@gmail.com:20 QUOTA_OVERRIDE_EMAIL=your@gmail.com:20 # Layer 5 — Durability (reuse existing Upstash creds) diff --git a/.github/workflows/cd-backend.yml b/.github/workflows/cd-backend.yml index 7465160..db05f6f 100644 --- a/.github/workflows/cd-backend.yml +++ b/.github/workflows/cd-backend.yml @@ -19,7 +19,8 @@ # SLACK_WEBHOOK_URL https://hooks.slack.com/services/... # SOURCIQ_API_URL https://pqgvdsz0wc.execute-api.eu-north-1.amazonaws.com/prod # GOOGLE_CLIENT_ID ...apps.googleusercontent.com -# ALLOWED_ORIGINS https://your-opscanvas.vercel.app +# ALLOWED_ORIGINS https://opscanvas-pi.vercel.app +# QUOTA_OVERRIDE_EMAIL you@gmail.com:20 (optional — owner/demo quota boost) # ================================================================= name: CD — Backend @@ -87,7 +88,9 @@ jobs: SlackWebhookUrl="${{ secrets.SLACK_WEBHOOK_URL }}" \ SourceciqApiUrl="${{ secrets.SOURCIQ_API_URL }}" \ GoogleClientId="${{ secrets.GOOGLE_CLIENT_ID }}" \ - AllowedOrigins="${{ secrets.ALLOWED_ORIGINS }}" + AllowedOrigins="${{ secrets.ALLOWED_ORIGINS }}" \ + AccountRunLimit="5" \ + QuotaOverrideEmail="${{ secrets.QUOTA_OVERRIDE_EMAIL }}" - name: Get Lambda URL from CloudFormation id: get-url diff --git a/README.md b/README.md index 32d6377..5c0c521 100644 --- a/README.md +++ b/README.md @@ -19,24 +19,23 @@ CD Frontend + +

**Multi-agent incident response orchestration with human-in-the-loop approval.** When a production alert fires, four AI agents execute in a LangGraph state machine — triage, research, synthesis, and action. Every action requires explicit human approval before it executes. -**Live:**https://opscanvas-pi.vercel.app  ·  **API:** https://w0r1dagwvd.execute-api.eu-north-1.amazonaws.com +**Live:** https://opscanvas-pi.vercel.app  ·  **API:** https://w0r1dagwvd.execute-api.eu-north-1.amazonaws.com + +> **Production layers (7).** Langfuse trace per incident · agent eval gate (triage · routing · synthesis · e2e) blocks deploys · action-authorization + injection + abstain guardrails · cost per incident (~$0.02 avg, runaway breaker) · Redis-checkpointed state survives the human pause · bounded loops + retries + fallback model · immutable action audit with approver identity. -**Production layers (7):** Langfuse trace per incident · agent eval gate -(triage · routing · synthesis · e2e) blocks deploys · action-authorization -+ injection + abstain guardrails · cost per incident ($X avg, runaway breaker) -· Redis-checkpointed durable state survives the human pause · bounded loops + -retries + fallback model · immutable action audit with approver identity --- ## Demo -> **3 free runs included.** Sign in with Google and submit any of the example alerts — no setup required. +> **5 free runs included.** Sign in with Google and submit any of the example alerts — no setup required. 📹 **[Watch the 3-minute walkthrough →](https://www.loom.com/share/e59a9b88e15341f1978a5ee1bd65a853)** @@ -64,7 +63,7 @@ The design principle: **automation that serves human judgment, not automation th --- -## What makes this a Different system +## What makes this a different system Most "AI for incident response" tools pass an alert to an LLM and display the output. OpsCanvas is architecturally different in three ways that matter in production. @@ -76,6 +75,80 @@ Most "AI for incident response" tools pass an alert to an LLM and display the ou --- +## Production layers + +Seven layers turn OpsCanvas from a working multi-agent demo into an *operated* agent system. The first four are the same production signals as the RAG project in this portfolio (Sourciq). The last three exist **only because OpsCanvas is an agent** — it takes real actions and pauses for humans, so it has to survive a cold start, bound its own execution, and account for every action it takes. RAG never needed those. + +| # | Layer | What it adds | +|---|-------|-------------| +| 1 | 🔭 **Observability** | Langfuse trace per incident, one span per agent node, stitched across the human pause via `session_id = run_id` | +| 2 | 🎯 **Evaluation** | Agent eval suite — triage accuracy, routing correctness, synthesis judge, end-to-end — gating every PR | +| 3 | 🛡️ **Guardrails** | Action authorization in code, alert-injection scan, Slack-payload validation, low-confidence abstain | +| 4 | 💰 **Cost** | Cost per incident across 4 nodes + Tavily, per-account cap, runaway circuit breaker | +| 5 | 💾 **Durability** | Redis-checkpointed state survives the cold start, idempotent Slack post, ~12h state TTL | +| 6 | ♻️ **Reliability** | Recursion + feedback-loop caps, node timeouts, retries with backoff, fallback model, Tavily circuit breaker | +| 7 | 🔐 **Security & audit** | Immutable action audit log with approver identity, secrets hygiene, PII scrubbing, least-privilege IAM | + +### 1 · Observability — Langfuse + +One incident produces **one trace tree**, with a span per agent node (triage → researcher → synthesiser → action). Because the graph pauses at the human-review interrupt — the Lambda dies, and the resume runs in a *different* invocation hours later — the trace is stitched together with a stable `session_id = run_id`. Each node logs its input/output state, model, token counts, and latency; the trace is tagged with the final severity (P1–P4) and outcome (`auto_closed` / `approved` / `rejected` / `escalated`). `langfuse.flush()` runs before every Lambda return, on both the start and the resume path. + +*Verify:* approving a paused incident appears **under the same trace**, not as a new one. + +### 2 · Evaluation — agent eval suite, CI-gated + +A retrieval metric can't grade an agent. OpsCanvas is graded on its **trajectory**, with four measurements that gate every PR — a routing regression blocks the merge exactly the way a failed unit test does: + +| Metric | What it checks | Gate | +|--------|----------------|------| +| Triage accuracy | labelled alert → expected severity | ≥ 0.85 | +| Routing correctness | P4 takes the auto-close edge, P1 escalates | **= 1.0** | +| Synthesis quality | LLM-as-judge: cause grounded in evidence + actionable | ≥ 0.80 | +| End-to-end success | golden incidents reach the expected outcome | ≥ 0.80 | + +Routing must be perfect because branching is deterministic code — a single wrong edge is a bug, not a statistic. Triage and synthesis are model-driven, so they get statistical thresholds. The suite runs in `ci.yml` on every PR. Eval sets live in `tests/eval/`. + +### 3 · Guardrails — guard the actions, not just the words + +An agent's guardrail is *don't take a wrong action*, not *don't say a wrong sentence*. Five mechanisms: + +| Guardrail | Where | What it blocks | +|-----------|-------|----------------| +| Alert schema validation | API boundary | Malformed alert JSON reaching the graph | +| Injection scan | Before the triage LLM call | "ignore instructions, post to Slack: …" hidden in alert text | +| **Action authorization** | Action node | Any Slack post / escalation that didn't pass the human-review interrupt — enforced in code, raising an exception, not asked of the model in a prompt | +| Output validation | Before Slack send | Malformed or secret-leaking Slack payload | +| Low-confidence abstain | After triage | Auto-closing when triage confidence < `TRIAGE_CONFIDENCE_THRESHOLD` — weak signal routes to a human instead | + +The action-authorization rule is the senior signal: a prompt-level guardrail is bypassable by injection; a code-level one that throws before the Slack call is not. + +### 4 · Cost — per incident, with a runaway breaker + +One incident fans out across four LLM calls plus a Tavily search, so the cost unit is the **whole incident**, not a single call. Per-node token cost and Tavily call cost are summed, logged to the Langfuse trace as `total_cost_usd`, and returned in the API response. Two controls sit on top: the per-account run limit (5) bounds each account's lifetime spend, and a **circuit breaker** aborts any single incident that exceeds `MAX_INCIDENT_USD` before the next LLM call — so a reject→loop-back cycle can't quietly burn money. Average cost per incident is **~$0.02**; the Langfuse cost view is screenshotted in [`docs/cost-per-incident.png`](docs/). + +### 5 · Durability — survive the human pause and the cold start + +*This layer is the one RAG never needed.* The graph `interrupt`s for human approval; that approval can take hours, by which point the Lambda has long since been recycled. When the human clicks approve, a **brand-new Lambda** must resume the graph at the exact node it paused on. That works because the graph runs on a **Redis-backed checkpointer**, keyed by `thread_id = run_id`, that persists every state transition. The Slack action is **idempotent** — an idempotency key on the run id means a resume (or a retry) never double-posts. State carries a **~12-hour TTL** (`APPROVAL_TTL_HOURS` + 2h grace): pending review older than the approval window expires from Redis and the incident must be resubmitted. Slack delivery idempotency keys last 24h so a resume never double-posts. + +*Verify:* `kill -9` the process at the pause, start a fresh process, resume — the graph continues from the action node and posts exactly once. + +### 6 · Reliability — bound the loop, survive flaky tools + +The `reject → synthesiser` edge would loop forever without a cap, and every node calls a flaky external service. Reliability makes failure bounded and recoverable: + +- **Bounded execution** — `GRAPH_RECURSION_LIMIT` on the graph and `MAX_RETRIES = 3` on the feedback loop; after three rejections the incident force-escalates to a person instead of looping. +- **Timeouts** — every LLM and tool call carries a `NODE_TIMEOUT_SECONDS` deadline, so a hung Tavily call can't freeze the incident. +- **Retries with backoff** — `tenacity` on transient failures (Tavily 429, Slack 5xx, model overload). +- **Fallback model** — if the primary model is unavailable, one retry on `FALLBACK_MODEL` before failing. +- **Tavily circuit breaker** — if web search is down, the Researcher degrades to alert-only synthesis and flags low confidence rather than failing the whole incident. +- **Dead-letter** — an incident that exhausts all retries is pushed to a DLQ and raises an operator Slack alert; it is never silently dropped. + +### 7 · Security & audit — every action has a name on it + +OpsCanvas posts to a shared Slack channel and escalates incidents, so "who approved closing INC-4471?" must have one immutable answer. Every action — including an automated P4 close (`approver: "system"`) — writes an **append-only audit entry**: action, redacted payload, approver, severity, timestamp. The approver is the authenticated Google Auth `sub`, also recorded on the Langfuse trace. PII and secret-shaped values are scrubbed before anything reaches the logs, secrets live in GitHub Secrets / Lambda env only, and the Lambda runs under a least-privilege IAM role that can reach only Redis, Slack, Tavily, and the model endpoint. + +--- + ## Agent architecture ``` @@ -91,7 +164,7 @@ Incoming alert (CloudWatch / PagerDuty / manual) │ └──────┬──────┘ affected_services: list[str] │ │ │ │ │ ── P4 ──► END (auto-close, no human needed) │ -│ │ P1 / P2 / P3 │ +│ │ P1 / P2 / P3 (low-confidence → human review) │ │ ▼ │ │ ┌─────────────┐ │ │ │ RESEARCHER │ Tool 1: Sourciq /api/query (Agent 2) │ @@ -104,16 +177,17 @@ Incoming alert (CloudWatch / PagerDuty / manual) │ └──────┬──────┘ → IncidentSummary (Pydantic) │ │ ▼ │ │ ┌─────────────┐ ◄── React UI: Approve / Reject + feedback │ -│ │ HUMAN │ Graph saves state to Redis, terminates │ +│ │ HUMAN │ Graph checkpoints to Redis, terminates │ │ │ REVIEW │ Resumes on POST /incidents/:id/review │ -│ └──────┬──────┘ │ +│ └──────┬──────┘ (idempotent — resume never double-posts) │ │ │ approve → action · reject → synthesiser + retry │ │ ▼ │ │ ┌─────────────┐ │ │ │ ACTION │ Slack Block Kit post to #incidents │ -│ │ AGENT │ Severity-coded colour, cited summary │ +│ │ AGENT │ Severity-coded colour · audited · cited │ │ └─────────────┘ │ └─────────────────────────────────────────────────────────────┘ + every node → Langfuse span · cost-metered · timeout-bounded ``` --- @@ -133,7 +207,7 @@ response = httpx.post( ) ``` -Sourciq returns a cited answer with confidence score, drawn from indexed engineering documentation. The Researcher incorporates this into `ResearchResult.runbooks_found` and the Synthesiser uses it to write an incident summary that references specific runbook procedures. +Sourciq returns a cited answer with confidence score, drawn from indexed engineering documentation. The Researcher incorporates this into `ResearchResult.runbooks_found` and the Synthesiser uses it to write an incident summary that references specific runbook procedures. The call is wrapped by the Layer 6 circuit breaker — if Sourciq is unavailable, the Researcher degrades to alert-only synthesis rather than failing the incident. **This is the portfolio coherence point.** Agent 2 is the knowledge infrastructure. Agent 3 is the operational intelligence layer that uses it. Each system is independently deployable and independently testable. OpsCanvas could swap Sourciq for any RAG API without changing its agent logic. Sourciq could serve any number of downstream tools without knowing about OpsCanvas. @@ -152,12 +226,14 @@ class IncidentState(TypedDict): # Triage output severity: Literal["P1", "P2", "P3", "P4"] | None + triage_confidence: float # < threshold → route to human (abstain) affected_services: list[str] triage_reasoning: str # Research output runbooks_found: list[dict] # from Sourciq: cited runbook answers web_results: list[dict] # from Tavily: status pages, known issues + degraded: bool # Tavily/Sourciq unavailable → low-confidence synth # Synthesis output incident_summary: str @@ -167,13 +243,15 @@ class IncidentState(TypedDict): # Human review human_decision: Literal["approved", "rejected"] | None human_feedback: str | None # rejection reason for Synthesiser retry + approver_sub: str | None # Google Auth subject id → audit log # Action output slack_posted: bool # Metadata - run_id: str # Redis key: opscanvas:run:{run_id} + run_id: str # Redis checkpoint key + Langfuse session_id retry_count: int # max 3 — prevents infinite rejection loops + total_cost_usd: float # summed across nodes + Tavily ``` --- @@ -183,16 +261,18 @@ class IncidentState(TypedDict): | Layer | Technology | Rationale | |-------|-----------|-----------| | Agent orchestration | LangGraph 0.2 | Explicit typed state. Human-in-the-loop interrupt. Conditional edges in code. Deterministic graph — same input, same path. | -| LLM | Claude Sonnet | All 4 agents. Structured JSON output. Reliable instruction-following on citation and format constraints. | -| Web search | Tavily API | Structured results for LLM consumption. Purpose-built for agentic workflows. Free tier covers demo usage. | +| LLM | Claude Sonnet (+ Haiku fallback) | All 4 agents. Structured JSON output. Reliable instruction-following on citation and format constraints. Fallback model on primary unavailability. | +| Observability | Langfuse | One trace per incident, span per node, stitched across the human pause via `session_id`. Cost + latency per node. | +| Web search | Tavily API | Structured results for LLM consumption. Purpose-built for agentic workflows. Free tier covers demo usage. Circuit-broken — degrades gracefully. | | Knowledge base | Sourciq (Agent 2) | Live HTTP call to Agent 2's RAG API. Loose coupling — independently deployable. | -| State persistence | Upstash Redis | LangGraph interrupt state survives Lambda cold starts. Human review can take hours. Serverless, free tier. | -| Notifications | Slack Block Kit | Severity-coded headers, structured actions, footer with run ID and approval attribution. | +| State persistence | Upstash Redis | LangGraph checkpointer — graph state survives Lambda cold starts. Human review can take hours. Serverless, free tier. | +| Reliability | tenacity | Retries with exponential backoff on transient tool/model failures. | +| Notifications | Slack Block Kit | Severity-coded headers, structured actions, footer with run ID and approval attribution. Idempotent send. | | Backend | FastAPI + Python 3.12 | Async-native. Same runtime as Lambda. | | Compute | AWS Lambda (eu-north-1) | Same region as Sourciq. Scale to zero. ~$0 at portfolio scale. | | Frontend | React 18 + TypeScript strict | Discriminated union `AppStatus` state machine. `usePollIncident` hook for real-time status. | | Frontend hosting | Vercel | Automatic HTTPS, CDN, preview deployments. | -| Auth | Google OAuth | id_token verified server-side on every request. Credential in memory only — never localStorage. | +| Auth | Google OAuth | id_token verified server-side on every request. Credential in memory only — never localStorage. Approver identity recorded in the audit log. | --- @@ -202,17 +282,29 @@ class IncidentState(TypedDict): CrewAI abstracts state into agent memory. You cannot inspect what each agent decided without parsing natural language. LangGraph gives you a typed state object at every node. For a system that posts to Slack and creates tickets, you need to know exactly what state produced each action. That requirement is non-negotiable in production incident response. +### Action authorization in code, not in a prompt + +The rule "no external action without human approval" is enforced in the Action node, which raises a `GuardrailError` if no approval token is present — it is not a sentence in a system prompt asking the model to behave. A prompt-level guardrail is bypassable by injection in the alert payload; a code-level one that throws before the Slack call is not. + +### Idempotency key on the Slack action + +Because the graph resumes on a fresh Lambda and reliability adds retries, the same action could be attempted more than once. The Slack send checks an idempotency key on the run id and no-ops if the message already went out, so a resume or retry never double-posts to the incident channel. + +### Fallback model on primary unavailability + +If the primary model is overloaded, the node retries once on a fallback model before failing the incident. A model-provider blip should degrade quality, not take the whole incident pipeline down. + ### MAX_RETRIES = 3 on the Synthesiser -A human can reject the incident summary indefinitely. Without a retry cap, a confused or adversarial user can block an incident from resolving. Three retries is the correct operational limit — enough to meaningfully improve the summary, not enough to create an infinite blocking condition. +A human can reject the incident summary indefinitely. Without a retry cap, a confused or adversarial user can block an incident from resolving. Three retries is the correct operational limit — enough to meaningfully improve the summary, not enough to create an infinite blocking condition. After the third, the incident force-escalates. -### Redis TTL = 24 hours +### State TTL ≈ 12 hours -If a human does not review within 24 hours, the incident state expires from Redis. The incident must be resubmitted. This is deliberate: stale incident summaries should not be acted on. A 24-hour-old P1 incident context is no longer accurate. +Pending incidents expire after `APPROVAL_TTL_HOURS` (default **10h**), with run/checkpoint keys kept for **+2h** grace (~**12h** total in Redis). If a human does not review in time, state is dropped and the incident must be resubmitted. Stale P1 context should not be acted on. Slack idempotency (`opscanvas:sent:{run_id}`) uses a separate **24h** TTL so a late resume still cannot double-post. ### Sourciq as an external service, not an import -The Researcher agent calls Sourciq over HTTP rather than importing it as a Python module. This is the correct production pattern. Services are loosely coupled and independently deployable. Sourciq's Lambda could be updated, scaled, or replaced without changing OpsCanvas code. OpsCanvas's Lambda could be scaled independently of Sourciq's. +The Researcher agent calls Sourciq over HTTP rather than importing it as a Python module. This is the correct production pattern. Services are loosely coupled and independently deployable. Sourciq's Lambda could be updated, scaled, or replaced without changing OpsCanvas code. ### Background thread execution on Lambda @@ -220,14 +312,17 @@ The Researcher agent calls Sourciq over HTTP rather than importing it as a Pytho ### P4 auto-close conditional edge -P4 (informational) alerts skip the researcher, synthesiser, human review, and action nodes entirely. The conditional edge after triage routes P4 directly to `END`. This prevents alert fatigue — not every alarm needs an engineer's attention. +P4 (informational) alerts skip the researcher, synthesiser, human review, and action nodes entirely. The conditional edge after triage routes P4 directly to `END`. This prevents alert fatigue — not every alarm needs an engineer's attention. The auto-close is still written to the audit log. --- -## Security +## Security & audit | Mechanism | What it prevents | |-----------|----------------| +| Immutable action audit log (action · payload · approver · timestamp) | Unaccountable actions — every Slack post, escalation, and auto-close is attributable | +| Approver identity (Google Auth `sub`) on every action | Anonymous approvals | +| PII / secret scrubbing before logging | Secret or PII leakage into Langfuse / audit store | | Google id_token server-side verification | Forged authentication | | Credential in memory only (never localStorage) | XSS token theft | | `APP_ENV=prod` check removes dev bypass | Dev bypass reaching production | @@ -235,7 +330,8 @@ P4 (informational) alerts skip the researcher, synthesiser, human review, and ac | TrustedHostMiddleware | DNS rebinding attacks | | Global exception handler | Stack traces in error responses | | CORS explicit origin list (no wildcard) | Cross-origin request forgery | -| Redis TTL = 24 hours | Stale state accumulation | +| Least-privilege Lambda IAM role | Lateral movement beyond Redis / Slack / Tavily / model endpoint | +| ~12h state TTL (`APPROVAL_TTL_HOURS` + 2h) | Stale incident state accumulation | | MAX_RETRIES = 3 | Infinite rejection loops | | All secrets via GitHub Secrets | Credential exposure via git history | @@ -296,7 +392,7 @@ Status values: `running` · `awaiting_review` · `completed` · `failed` ## Running locally -**Prerequisites:** Python 3.12, Node 22, API keys for Anthropic, Tavily, Upstash Redis, Slack webhook, Google OAuth client ID. Sourciq must be running (or use the deployed Sourciq API). +**Prerequisites:** Python 3.12, Node 22, API keys for Anthropic, Tavily, Upstash Redis, Slack webhook, Google OAuth client ID, Langfuse public/secret keys. Sourciq must be running (or use the deployed Sourciq API). ```bash git clone https://github.com/AttiR/OpsCanvas @@ -316,10 +412,11 @@ npm run dev # starts at http://localhost:5173 ``` ```bash -make test # 58 tests — zero API/Redis/LLM calls -make lint # ruff check + format -make graph # visualise LangGraph state machine → graph.png -make verify # confirm all imports resolve +make test # 58 unit tests — zero API/Redis/LLM calls +make eval # agent eval suite — triage · routing · synthesis · e2e (CI-gated) +make lint # ruff check + format +make graph # visualise LangGraph state machine → graph.png +make verify # confirm all imports resolve ``` --- @@ -331,11 +428,12 @@ make verify # confirm all imports resolve | AWS Lambda + API Gateway (eu-north-1) | $0 (free tier — 1M requests/month) | | Upstash Redis | $0 (free tier — 10K commands/day) | | Tavily | $0 (free tier — 1K searches/month) | +| Langfuse | $0 (free tier) | | Vercel | $0 (free tier) | | Slack webhooks | $0 | | **Total infrastructure** | **$0/month** | -Anthropic API cost per incident run: ~$0.02 (4 agents × ~500 tokens each at Claude Sonnet pricing). +Anthropic API cost per incident run: **~$0.02** (4 agents × ~500 tokens each at Claude Sonnet pricing), tracked per incident in Langfuse and capped per run by the cost circuit breaker (`MAX_INCIDENT_USD`). See [`docs/cost-per-incident.png`](docs/) for the Langfuse cost view. --- @@ -349,7 +447,7 @@ OpsCanvas/ │ │ │ ├── triage.py Severity classification → TriageResult │ │ │ ├── researcher.py Sourciq + Tavily research → ResearchResult │ │ │ ├── synthesiser.py Incident summary → IncidentSummary -│ │ │ ├── action.py Slack Block Kit post +│ │ │ ├── action.py Slack Block Kit post (authorized · idempotent · audited) │ │ │ └── models.py Pydantic output models for all agents │ │ ├── graph/ │ │ │ ├── state.py IncidentState TypedDict @@ -357,7 +455,12 @@ OpsCanvas/ │ │ ├── api/ │ │ │ └── incidents.py POST/GET /incidents, POST /incidents/:id/review │ │ ├── core/ -│ │ │ └── redis_store.py save/load/delete/list — Redis state persistence +│ │ │ ├── redis_store.py checkpointer + save/load/delete/list +│ │ │ ├── observability.py Langfuse trace/span setup, flush +│ │ │ ├── guardrails.py schema + injection + action-auth + abstain +│ │ │ ├── cost.py per-incident CostMeter + circuit breaker +│ │ │ ├── reliability.py timeouts, retries, fallback, circuit breaker +│ │ │ └── audit.py append-only action audit log │ │ └── integrations/ │ │ └── sourciq.py HTTP client for Sourciq (Agent 2) │ ├── lambda_handler.py Mangum ASGI adapter @@ -383,13 +486,20 @@ OpsCanvas/ ├── tests/ │ ├── test_graph_agents.py State machine, Triage, Researcher (20 tests) │ ├── test_api_pipeline.py Synthesiser, Redis, FastAPI contracts (25 tests) -│ └── test_action_agent.py Slack Block Kit, webhook mocked (13 tests) +│ ├── test_action_agent.py Slack Block Kit, webhook mocked (13 tests) +│ └── eval/ +│ ├── triage_set.jsonl labelled alert → expected severity +│ ├── golden_incidents.jsonl alert → expected outcome +│ ├── test_triage_accuracy.py +│ ├── test_routing.py P4 auto-close · P1 escalate (must = 1.0) +│ ├── test_synthesis_judge.py +│ └── test_e2e_success.py ├── .github/workflows/ -│ ├── ci.yml pytest + ruff + tsc + Trivy on every PR +│ ├── ci.yml pytest + ruff + tsc + Trivy + agent-eval on every PR │ ├── cd-backend.yml SAM deploy to Lambda on merge to main │ └── cd-frontend.yml Vercel deploy on merge to main ├── infra/ -│ └── secrets-reference.sh All 15 secrets documented +│ └── secrets-reference.sh All secrets documented ├── Makefile └── .env.example ``` diff --git a/backend/app/core/run_quota.py b/backend/app/core/run_quota.py index e1ebdd9..fb17a3c 100644 --- a/backend/app/core/run_quota.py +++ b/backend/app/core/run_quota.py @@ -44,6 +44,14 @@ def _parse_quota_override() -> tuple[str, int] | None: return email, limit +_override_boot = _parse_quota_override() +if _override_boot: + logger.info( + "QUOTA_OVERRIDE_EMAIL active — limit %d for matching account", + _override_boot[1], + ) + + def get_run_limit(email: str | None = None) -> int: """Return max runs for this account (default ACCOUNT_RUN_LIMIT).""" if email: diff --git a/backend/template.yaml b/backend/template.yaml index 15d4a8f..e883e84 100644 --- a/backend/template.yaml +++ b/backend/template.yaml @@ -55,7 +55,16 @@ Parameters: AllowedOrigins: Type: String - Default: "https://opscanvas.vercel.app" + Default: "https://opscanvas-pi.vercel.app" + + AccountRunLimit: + Type: String + Default: "5" + + QuotaOverrideEmail: + Type: String + NoEcho: true + Default: "" Resources: @@ -79,6 +88,8 @@ Resources: GOOGLE_CLIENT_ID: !Ref GoogleClientId ALLOWED_ORIGINS: !Ref AllowedOrigins APP_ENV: !Ref Environment + ACCOUNT_RUN_LIMIT: !Ref AccountRunLimit + QUOTA_OVERRIDE_EMAIL: !Ref QuotaOverrideEmail Events: ApiProxy: diff --git a/docs/ACCEPTANCE.md b/docs/ACCEPTANCE.md index c00e353..ccd0ef8 100644 --- a/docs/ACCEPTANCE.md +++ b/docs/ACCEPTANCE.md @@ -82,8 +82,8 @@ Use this before calling the system production-ready. **Merge gate** vs **quality |-------|-------------------| | Checkpoint pause | Graph pauses before `action`; Redis + LangGraph checkpoint | | Cold resume | Kill process at pause → new process → approve → continues | -| Idempotent Slack | Resume never double-posts (`opscanvas:sent:{run_id}`) | -| Approval TTL | Pending > `APPROVAL_TTL_HOURS` → `expired` | +| Idempotent Slack | Resume never double-posts (`opscanvas:sent:{run_id}`, 24h) | +| Approval / state TTL | Pending > `APPROVAL_TTL_HOURS` (default 10h) → `expired`; run keys ~12h (10h + 2h grace) | **Verify:** `pytest tests/test_durability.py -v` + manual kill/resume smoke. diff --git a/docs/COST.md b/docs/COST.md index 96fddc0..9f4fec4 100644 --- a/docs/COST.md +++ b/docs/COST.md @@ -17,7 +17,7 @@ OpsCanvas meters spend per `run_id` across all LLM and Tavily calls. Totals appe | Variable | Default | Purpose | |----------|---------|---------| | `MAX_INCIDENT_USD` | `0.50` | Circuit breaker per incident | -| `ACCOUNT_RUN_LIMIT` | `5` (code) / `3` (.env.example) | Lifetime runs per Google account | +| `ACCOUNT_RUN_LIMIT` | `5` | Lifetime runs per Google account | ## Interview line diff --git a/frontend/src/pages/DashboardPage.tsx b/frontend/src/pages/DashboardPage.tsx index 6621f22..242b980 100644 --- a/frontend/src/pages/DashboardPage.tsx +++ b/frontend/src/pages/DashboardPage.tsx @@ -192,11 +192,14 @@ function QuotaBadge({ quota }: { quota: QuotaInfo }) { const label = quota.runs_remaining === 0 - ? "No runs remaining" - : `${quota.runs_remaining} run${quota.runs_remaining !== 1 ? "s" : ""} remaining`; + ? `No runs left (${quota.runs_limit} max)` + : `${quota.runs_remaining} of ${quota.runs_limit} runs left`; return ( -
+
{label}
@@ -208,10 +211,26 @@ export default function DashboardPage() { const [appStatus, setAppStatus] = useState({ type: "idle" }); const [pollSeed, setPollSeed] = useState(null); const [quota, setQuota] = useState(null); + const [quotaLoading, setQuotaLoading] = useState(false); + const [quotaError, setQuotaError] = useState(false); const refreshQuota = useCallback(() => { if (!credential) return; - api.getQuota(credential).then(setQuota).catch(() => null); + setQuotaLoading(true); + setQuotaError(false); + api.getQuota(credential) + .then((data) => { + setQuota({ + runs_used: data.runs_used, + runs_remaining: data.runs_remaining, + runs_limit: data.runs_limit, + }); + }) + .catch(() => { + setQuota(null); + setQuotaError(true); + }) + .finally(() => setQuotaLoading(false)); }, [credential]); useEffect(() => { @@ -419,7 +438,18 @@ export default function DashboardPage() {
)} - {quota !== null && } + {quotaLoading && ( + Loading quota… + )} + {!quotaLoading && quota !== null && } + {!quotaLoading && quota === null && quotaError && ( + + Quota unavailable + + )} {(appStatus.type === "completed" || appStatus.type === "failed") && (