Develop

.dockerignore

@@ -0,0 +1,9 @@
+.venv
+__pycache__
+*.pyc
+.env
+.git
+web/
+eval/.cache.jsonl
+data/audit_log.jsonl
+data/evincta.sqlite

.example.env

@@ -1,6 +1,11 @@
 ANTHROPIC_API_KEY=
-VOYAGE_API_KEY=
+VOYAGE_API_KEY=                  # evincta project key
 PINECONE_API_KEY=
+PINECONE_INDEX=                    # dedicated index name
+PINECONE_CLOUD=aws
+PINECONE_REGION=                   # match your Pinecone project's region
 LANGFUSE_PUBLIC_KEY=
 LANGFUSE_SECRET_KEY=
 LANGFUSE_HOST=https://cloud.langfuse.com
+EVINCTA_API_KEY=                   # required for write routes in production
+ALLOWED_ORIGINS=http://localhost:5173

.github/workflows/ci.yml

@@ -0,0 +1,12 @@
+name: CI
+on: { pull_request: {}, push: { branches: [main, develop] } }
+jobs:
+  check:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
+        with: { python-version: "3.12" }
+      - run: pip install -r requirements.txt ruff pytest
+      - run: ruff check .
+      - run: pytest -q            # integration tests skipped by default (see pyproject.toml)

.github/workflows/eval.yml

@@ -0,0 +1,17 @@
+name: Eval Gate
+on:
+  pull_request:
+  workflow_dispatch:            # also run on demand
+jobs:
+  eval:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
+        with: { python-version: "3.12" }
+      - run: pip install pyyaml
+      # eval/results.json is generated locally (needs the synthetic dataset in
+      # data/generated/, which is gitignored, plus paid API calls) and committed.
+      # CI only gates that committed result against thresholds — deterministic,
+      # no secrets, no network. Regenerate locally with: python -m eval.run_eval
+      - run: python -m eval.gate          # ← fails the job on regression

.gitignore

@@ -1,12 +1,44 @@
-# secrets — never commit
+# ── Secrets (never commit) ─────────────────────────────
 .env
+.env.*
+!.env.example
 *.key
+*.pem
 
-# data — keep samples, ignore the bulk generated set
+# ── Python ─────────────────────────────────────────────
+__pycache__/
+*.py[cod]
+.venv/
+venv/
+*.egg-info/
+.pytest_cache/
+.ruff_cache/
+.mypy_cache/
+
+# ── Data: keep the code + samples, ignore the bulk ─────
+data/raw/
 data/generated/
-*.png
+!data/samples/
 !data/samples/**
 
-# python / node / build
-__pycache__/  *.pyc  .venv/  venv/
-node_modules/  dist/  .pytest_cache/  .ruff_cache/
+# ── Node / React UI (Phase 6) ──────────────────────────
+node_modules/
+ui/dist/
+ui/build/
+*.local
+
+# ── OS / editor noise ──────────────────────────────────
+.DS_Store
+.idea/
+.vscode/
+*.swp
+
+# ── SQLite database ───────────────────────────────────
+data/evincta.sqlite*
+
+# ── Runtime audit log ─────────────────────────────────
+data/audit_log.jsonl
+
+# ── Runtime artifacts ──────────────────────────────────
+data/audit_log.jsonl
+data/claims_index.json

Dockerfile

@@ -0,0 +1,20 @@
+FROM python:3.12-slim
+
+# system deps for pillow etc.
+RUN apt-get update && apt-get install -y --no-install-recommends \
+    build-essential && rm -rf /var/lib/apt/lists/*
+
+WORKDIR /app
+COPY requirements.txt .
+RUN pip install --no-cache-dir -r requirements.txt
+
+COPY services/ services/
+COPY api/ api/
+# Demo claims (committed). Mount or bake data/generated/ for the full seeded set (issue #47).
+COPY data/samples/ data/samples/
+
+# NOTE: no .env, no secrets copied — injected at runtime by Render
+ENV PYTHONUNBUFFERED=1
+EXPOSE 8000
+# exec makes uvicorn PID 1 so SIGINT/SIGTERM (Ctrl+C / docker stop) reach it
+CMD ["sh", "-c", "exec uvicorn api.main:app --host 0.0.0.0 --port ${PORT:-8000}"]

README.md

@@ -32,11 +32,11 @@ vision   multimodal  policy/cost   orchestrator  accuracy  traced/
 ```
 
 ## Roadmap
-- [ ] Phase 1 — Multimodal ingest (vision → structured JSON)
-- [ ] Phase 2 — Multimodal vector DB + precedent retrieval
-- [ ] Phase 3 — MCP servers (policy · cost · fraud · precedent)
-- [ ] Phase 4 — Multi-agent recommender
-- [ ] Phase 5 — Eval harness + CI accuracy gate
+- [x] Phase 1 — Multimodal ingest (vision → structured JSON)
+- [x] Phase 2 — Multimodal vector DB + precedent retrieval
+- [x] Phase 3 — MCP servers (policy · cost · fraud · precedent)
+- [x] Phase 4 — Multi-agent recommender
+- [x] Phase 5 — Eval harness + CI accuracy gate
 - [ ] Phase 6 — Production layer + deploy + review UI
 
 ## Stack
@@ -53,3 +53,32 @@ tuning.
 Targets, gated in CI once Phase 5 lands: decision accuracy ≥ 0.85 ·
 fraud recall ≥ 0.90 · extraction accuracy ≥ 0.90 · one traced,
 cost-metered run per claim.
+
+## Evaluation
+
+Evincta is evaluated end-to-end on 18 held-out claims (never indexed, never
+seen during retrieval), scored against ground truth with a CI gate that fails
+the build on regression.
+
+| Metric | Value |
+|---|---|
+| Decision accuracy (weighted) | 0.722 |
+| Decision accuracy (exact) | 0.722 |
+| Payout in range | 0.769 (n=13) |
+| Extraction — severity | 0.889 |
+| Extraction — incident | 0.833 |
+| Latency p50 / p95 | 14.9s / 16.6s |
+
+**Weighted decision accuracy** penalises dangerous errors (approving a claim
+that should be denied) far more than cautious ones; weighted == exact here
+means the system makes **no dangerous errors** on the eval set — when it is
+wrong, it errs toward caution, not toward over-paying.
+
+**Scope & honesty:** Fraud recall is reported separately (0.0, n=2). The
+synthetic frauds are image-reuse type; reliable detection requires perceptual
+image hashing across claims — a roadmapped capability, not yet implemented.
+Every recommendation is gated by human approval regardless, so a missed fraud
+flag is never an automatic payout.
+
+Gated metrics and thresholds: [`eval/thresholds.yaml`](eval/thresholds.yaml).
+Full results: [`eval/REPORT.md`](eval/REPORT.md).

api/auth.py

@@ -0,0 +1,9 @@
+import os
+from fastapi import Header, HTTPException
+
+API_KEY = os.getenv("EVINCTA_API_KEY")     # injected by Render
+
+
+def require_key(x_api_key: str = Header(None)):
+    if not API_KEY or x_api_key != API_KEY:
+        raise HTTPException(401, "invalid or missing API key")

api/claims.py

@@ -0,0 +1,98 @@
+from fastapi import APIRouter, HTTPException
+from fastapi.concurrency import run_in_threadpool
+from pydantic import BaseModel
+from services.agents.run import start_claim, resume_claim
+from services.ingest.extractor import extract_claim
+from fastapi import Depends
+from .auth import require_key
+from . import store
+import json
+
+
+router = APIRouter()
+
+
+class ClaimIn(BaseModel):
+    claim_dir: str            # path to a claim folder (6a). 6b: real upload.
+
+
+@router.post("/claims", dependencies=[Depends(require_key)])
+async def submit(body: ClaimIn):
+    def _run():
+        lbl = json.load(open(f"{body.claim_dir}/label.json"))
+        pol = lbl["policy"]
+        rec = extract_claim(body.claim_dir).model_dump()
+        state = {"claim_id": lbl["claim_id"],
+                 "image_path": f"{body.claim_dir}/images/img_0.jpg",
+                 "exclusions": pol["exclusions"], "record": {
+                    "incident_type": rec["incident_type"], "severity": rec["severity"],
+                    "damaged_parts": rec["damaged_parts"],
+                    "notes_summary": rec["notes_summary"],
+                    "deductible_usd": pol["deductible_usd"],
+                    "coverage_limit_usd": pol["coverage_limit_usd"],
+                    "image_inconsistency": rec["image_inconsistency"],
+                    "visible_pre_existing_damage": rec["visible_pre_existing_damage"]}}
+        tid, recommendation = start_claim(state)
+        return lbl["claim_id"], tid, recommendation, rec
+    cid, tid, rec, extracted = await run_in_threadpool(_run)
+    store.upsert(cid, thread_id=tid, status="pending",
+                 recommendation=rec, evidence=extracted)
+    return {"claim_id": cid, "status": "pending", "recommendation": rec}
+
+
+@router.get("/claims")
+def list_claims():
+    # compact rows for the queue view
+    return [{"claim_id": c["claim_id"], "status": c["status"],
+             "decision": c["recommendation"]["decision"],
+             "fraud_risk": c["recommendation"]["fraud_risk"]}
+
+            for c in store.all_claims()]
+
+
+@router.get("/claims/{claim_id}")
+def get_claim(claim_id: str):
+    c = store.get(claim_id)
+    if not c:
+        raise HTTPException(404, "claim not found")
+    return c        # full detail: recommendation + evidence + status
+
+
+class DecisionIn(BaseModel):
+    decision: str            # "approve" | "override"
+    approver: str
+    override_to: str | None = None   # required when decision == "override"
+
+
+@router.post("/claims/{claim_id}/decision", dependencies=[Depends(require_key)])
+async def decide(claim_id: str, body: DecisionIn):
+    c = store.get(claim_id)
+    if not c:
+        raise HTTPException(404, "claim not found")
+    if c["status"] != "pending":
+        raise HTTPException(409, "already decided")
+
+    # the human decision the audit records: the model's rec, or the override
+    final = (body.override_to if body.decision == "override"
+             else c["recommendation"]["decision"])
+
+    def _resume():
+        resume_claim(c["thread_id"], human_decision=final, approver=body.approver)
+    await run_in_threadpool(_resume)
+
+    store.upsert(claim_id, status="decided",
+        human_decision=final, approver=body.approver,
+        was_override=(body.decision == "override"))
+    return {"claim_id": claim_id, "status": "decided", "final_decision": final}
+
+
+@router.post("/admin/reset", dependencies=[Depends(require_key)])
+def reset_demo():
+    # restore every seeded claim to "pending" (clears human decisions)
+    d = store._read()
+    for cid, c in d.items():
+        c["status"] = "pending"
+        c.pop("human_decision", None)
+        c.pop("approver", None)
+    store._write(d)
+    return {"reset": len(d)}

api/main.py

@@ -0,0 +1,36 @@
+import asyncio
+import os
+from contextlib import asynccontextmanager
+from fastapi import FastAPI
+from fastapi.middleware.cors import CORSMiddleware
+from services.mcp.host import Host
+from services.agents.tools import set_shared_host
+from .claims import router as claims_router
+
+
+ORIGINS = os.getenv("ALLOWED_ORIGINS", "http://localhost:5173").split(",")
+
+
+@asynccontextmanager
+async def lifespan(app: FastAPI):
+    # startup: boot ONE MCP host, reused by every request. Pass the running
+    # loop so threadpool workers drive tool calls on the host's own loop.
+    app.state.host = await Host().connect()
+    set_shared_host(app.state.host, asyncio.get_running_loop())
+    yield
+    # shutdown: tear it down cleanly
+    set_shared_host(None, None)
+    await app.state.host.close()
+
+app = FastAPI(title="Evincta API", lifespan=lifespan)
+
+# dev CORS — locked down to the real origin in 6b
+app.add_middleware(CORSMiddleware, allow_origins=ORIGINS,
+    allow_methods=["GET", "POST"],
+    allow_headers=["Content-Type", "X-API-Key"])
+
+app.include_router(claims_router)
+
+
+@app.get("/health")
+def health(): return {"status": "ok"}

api/store.py

@@ -0,0 +1,27 @@
+import json
+from pathlib import Path
+from threading import Lock
+
+DB = Path("data/claims_index.json")
+_lock = Lock()
+
+
+def _read() -> dict:
+    return json.loads(DB.read_text()) if DB.exists() else {}
+
+
+def _write(d): DB.write_text(json.dumps(d, indent=2))
+
+
+def upsert(claim_id, **fields):
+    with _lock:
+        d = _read()
+        d.setdefault(claim_id, {})
+        d[claim_id].update(fields)
+        d[claim_id]["claim_id"] = claim_id
+        _write(d)
+        return d[claim_id]
+
+
+def get(claim_id): return _read().get(claim_id)
+def all_claims(): return list(_read().values())

data/SCHEMA.md

@@ -0,0 +1,19 @@
+{
+  "claim_id": "claim_0001",
+  "split": "index",                 // "index" (~60) | "eval" (~20)
+  "ground_truth": {
+    "incident_type": "collision",    // collision|theft|weather|vandalism|other
+    "severity": "moderate",         // minor|moderate|severe|total
+    "decision": "approve",          // approve|investigate|deny
+    "payout_usd": 3200.0,
+    "is_fraud": false,
+    "fraud_type": null              // image_reuse|incident_mismatch|pre_existing
+  },
+  "policy": {
+    "policy_number": "AUTO-48213",
+    "vehicle_value_usd": 18000,
+    "deductible_usd": 500,
+    "coverage_limit_usd": 18000,
+    "exclusions": ["flood"]
+  }
+}