feat(ai-assistant): conversational analytic layer (BgGPT)

.dev.vars.example

@@ -7,3 +7,30 @@
 
 # Web worker privacy-preserving request log IP HMAC key.
 LOG_IP_KEY=dev-only-log-ip-key-change-me
+
+# ── AI assistant (docs/spec/ai-assistant.md) ───────────────────────────────────────────────────────
+# Provider API key (OpenRouter today) — SECRET. Without it /assistant/chat returns a controlled 503
+# (endpoint unprovisioned). In the cloud set via `wrangler secret put ASSISTANT_API_KEY --name <worker>`,
+# never committed.
+# ASSISTANT_API_KEY=
+#
+# All model traffic MUST transit the Cloudflare AI Gateway (fail closed): without AI_GATEWAY_BASE_URL the
+# assistant returns 503 rather than calling the provider directly. Set both to your gateway (account
+# <account-id>); AI_GATEWAY_ID is the gateway slug used to route Workers AI embeddings.
+# ASSISTANT_MODEL is the provider-scoped model id — swap models by editing it alone.
+# AI_GATEWAY_BASE_URL=https://gateway.ai.cloudflare.com/v1/<account-id>/<gateway>/openrouter/v1
+# AI_GATEWAY_ID=<gateway>
+# ASSISTANT_MODEL=google/gemma-4-31b-it   # append ':free' for the rate-limited free tier
+#
+# NOTE: Workers AI (`AI`) and Vectorize (`VECTORIZE`) have NO local miniflare emulation, and
+# vite.config.ts configures no remote bindings — so under a plain `pnpm dev` the assistant route
+# degrades (RAG grounding/semantic_search no-op, then 503 without a key). To exercise the assistant
+# locally, run with remote bindings (`wrangler dev --remote`) against the dev account's resources.
+#
+# Optional: enables the one-shot schema-corpus seed route POST /assistant/reindex (off when unset).
+# High-entropy value; sent as `Authorization: Bearer <token>`. See app/lib/assistant/README.md.
+# ASSISTANT_SEED_TOKEN=
+#
+# Transcript signing key (HMAC-SHA-256 over server-emitted messages).
+# Prod: `wrangler secret put ASSISTANT_HMAC_KEY`. Never commit the real value.
+ASSISTANT_HMAC_KEY=dev-only-assistant-hmac-key-change-me

.github/workflows/ci.yml

@@ -57,7 +57,11 @@ jobs:
       - uses: pnpm/action-setup@0ebf47130e4866e96fce0953f49152a61190b271 # v6.0.9
       - uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0
         with:
-          node-version: 22
+          # Pinned to an exact minor: node:sqlite bundles SQLite, whose version (and thus the EXPLAIN
+          # opcode universe the assistant opcode-guard test pins) changes with the Node minor. 22.23.0
+          # ships SQLite 3.51.3, which READ_ONLY_OPCODES/KNOWN_SQLITE_VERSIONS cover. Bump deliberately
+          # and re-harvest the allowlist (see sql-opcode-guard.test.ts) rather than letting 22.x float.
+          node-version: 22.23.0
           cache: pnpm
       - run: pnpm install --frozen-lockfile
       - name: Dependency audit
@@ -86,13 +90,29 @@ jobs:
       - name: Typecheck
         if: ${{ !cancelled() }}
         run: pnpm typecheck
+      # `pnpm test` also runs the assistant adversarial suites — the two-layer SQL guard
+      # (sql-guard-adversarial), the EXPLAIN-opcode allowlist (sql-opcode-guard), and the
+      # prompt-injection boundary tests (system-prompt / tool-results / chat-input) — so a regression in
+      # any read-only / injection defence blocks merge (launch gate §9.9, issue #83).
       - name: Test
         if: ${{ !cancelled() }}
         run: pnpm test
+      # Launch gate §9.9 (issue #83): replays the golden report fixtures through the real bind/reconcile
+      # pipeline and asserts A–E2 (canonical amount_eur, default filters, rollup reconciliation, no prose
+      # figures). Blocking, so a model / schema / prompt change can't silently regress a published report.
+      - name: Golden reports
+        if: ${{ !cancelled() }}
+        run: pnpm test:golden
       # Docs integrity (#102): fail on a dangling docs reference in source or an
       # unindexed doc. The checker self-tests first, so the gate is itself gated.
       - name: Docs integrity
         if: ${{ !cancelled() }}
         run: |
           pnpm check:docs:test
           pnpm check:docs
+      # Root-level tooling (scripts/*.mjs) lives outside the turbo/vitest workspaces, so `pnpm test` never
+      # ran its node:test suites. Gate them here — the CI-provisioning + teardown tools (ensure-kv-namespace,
+      # ensure-voice-provider, teardown-remote, reap-previews, load-eop) are deploy-critical and must stay covered.
+      - name: Script tests
+        if: ${{ !cancelled() }}
+        run: node --test scripts/*.test.mjs

.github/workflows/deploy.yml

@@ -8,11 +8,17 @@ name: Deploy (web + etl → Cloudflare)
 # Event → target mapping (see the `detect` job):
 #   - push to `main`     → staging      (continuous staging on every merge to main, so staging leads prod)
 #   - version TAG (v*)   → production    (cut a release: git tag v1.0.1 && git push origin v1.0.1)
-#   - workflow_dispatch  → the chosen environment
-# midt-bg/sigma is the single repo carrying both the `staging` and `production` Environments (the former
+#   - workflow_dispatch  → the chosen environment (incl. `dev`)
+# midt-bg/sigma is the single repo carrying the `dev`, `staging` and `production` Environments (the former
 # midt-bg/sigma-stage mirror was consolidated into it). Production is cut deliberately as a tag; staging
 # tracks `main`. (To deploy on GitHub *Releases* instead of tags, swap the tag trigger for
 # `release: { types: [published] }`.)
+#
+# `dev` is a long-lived, manual-only target: deploy ANY branch to it on demand from the Actions tab or
+#   gh workflow run deploy.yml --ref <branch> -f environment=dev
+# It reuses the same render/guard machinery — it just needs a `dev` GitHub Environment provisioned with
+# its own SIGMA_* names + a dev D1 id. Ephemeral per-PR previews are a separate workflow (preview.yml).
+# Setup runbook: docs/dev-environments.md.
 
 on:
   push:
@@ -24,7 +30,14 @@ on:
         description: Deploy target
         type: choice
         options:
+          - dev
           - staging
+          # Once provision-environments.sh has been applied, selecting `production` here only
+          # works when the workflow is dispatched from a `v*` tag ref, e.g.:
+          #   gh workflow run deploy.yml --ref v1.0.1 -f environment=production
+          # Dispatching from a branch ref (e.g. main) is refused by the environment tag policy.
+          # Admins may bypass this via the `admins_can_bypass` setting (Settings → Environments
+          # → production) for emergency situations.
           - production
         default: staging
 
@@ -44,12 +57,19 @@ jobs:
         #   workflow_dispatch  -> the chosen environment
         #   tag v*             -> production
         #   push to main       -> staging      (continuous staging — main always leads prod)
+        # Pass the GitHub expressions through `env:` and reference the quoted shell vars, rather than
+        # interpolating `${{ github.ref }}` (attacker-influenceable via a crafted tag/branch name)
+        # directly into the script body — that would let a ref like `v"; <cmd>; "` break out of the test.
+        env:
+          GITHUB_REF: ${{ github.ref }}
+          EVENT_NAME: ${{ github.event_name }}
+          INPUT_ENV: ${{ inputs.environment }}
         run: |
-          if [ "${{ github.event_name }}" = "workflow_dispatch" ]; then
-            echo "env=${{ inputs.environment }}" >> "$GITHUB_OUTPUT"
-          elif [[ "${{ github.ref }}" == refs/tags/v* ]]; then
+          if [ "$EVENT_NAME" = "workflow_dispatch" ]; then
+            echo "env=$INPUT_ENV" >> "$GITHUB_OUTPUT"
+          elif [[ "$GITHUB_REF" == refs/tags/v* ]]; then
             echo "env=production" >> "$GITHUB_OUTPUT"
-          elif [ "${{ github.ref }}" = "refs/heads/main" ]; then
+          elif [ "$GITHUB_REF" = "refs/heads/main" ]; then
             echo "env=staging" >> "$GITHUB_OUTPUT"
           else
             echo "env=" >> "$GITHUB_OUTPUT"
@@ -59,6 +79,11 @@ jobs:
     needs: detect
     if: needs.detect.outputs.env != ''
     runs-on: ubuntu-latest
+    # Explicit cap instead of the 360-min GitHub default: a full build + two-Worker wrangler deploy
+    # runs in single-digit minutes, so a job past 10 min is hung (stalled network I/O, wedged
+    # wrangler) — fail fast rather than burn a 6-hour slot. The required-reviewers wait on
+    # `production` does NOT count against this; the timer starts only once the job hits a runner.
+    timeout-minutes: 10
     environment: ${{ needs.detect.outputs.env }}
     concurrency:
       group: deploy-${{ needs.detect.outputs.env }}
@@ -76,6 +101,20 @@ jobs:
       SIGMA_CSV_CACHE_NAME: ${{ vars.SIGMA_CSV_CACHE_NAME }}
       SIGMA_REPORTS_NAME: ${{ vars.SIGMA_REPORTS_NAME }}
       SIGMA_VECTORIZE_NAME: ${{ vars.SIGMA_VECTORIZE_NAME }}
+      # Per-build dedup freshness `c` — wrangler-render stamps this over the committed "dev" so a code-only
+      # deploy (report shape / FX / CPV logic) busts the report cache, and each env's build keys stay distinct.
+      SIGMA_BUILD_ID: ${{ github.sha }}
+      # Master kill-switch override (#83), per-environment: set the SIGMA_ASSISTANT_ENABLED *variable* to
+      # "true" on the environments that should serve the assistant (dev now; staging at go-live). Unset →
+      # wrangler-render leaves the committed fail-dark "false", so production stays dark until deliberately
+      # flipped. Killing a live assistant is then just clearing this var + redeploy (or the runtime flag).
+      SIGMA_ASSISTANT_ENABLED: ${{ vars.SIGMA_ASSISTANT_ENABLED }}
+      # Runtime deploy-env for the §9.3 HMAC gate (ADR-0012). Set the SIGMA_ENVIRONMENT *variable* per
+      # GitHub Environment: "production" and "staging" are public/unauthenticated → the gate REQUIRES the
+      # signing key (fail-closed 503 without it); any other value (or unset → committed "development")
+      # fails open. wrangler-render stamps this into ENVIRONMENT. Not derived from import.meta.env.PROD,
+      # which Vite inlines true for the staging build too (it would misclassify staging as production).
+      SIGMA_ENVIRONMENT: ${{ vars.SIGMA_ENVIRONMENT }}
     steps:
       - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
       - uses: pnpm/action-setup@0ebf47130e4866e96fce0953f49152a61190b271 # v6.0.9
@@ -121,8 +160,34 @@ jobs:
           check "$SIGMA_VECTORIZE_NAME" sigma-assistant SIGMA_VECTORIZE_NAME
           [ "$fail" = 0 ] || { echo "Refusing to deploy '${{ needs.detect.outputs.env }}' with production resource names."; exit 1; }
 
+      # GitOps: enforce this environment's report-dedup KV namespace from git (idempotent create-if-absent)
+      # and hand its id to wrangler-render via SIGMA_DEDUP_KV_ID. One namespace per real env — dev (shared
+      # with the ephemeral previews), staging, prod — kept distinct by title so no env can touch another's
+      # cache. `production` maps to the short `prod` suffix; dev/staging use the env name verbatim.
+      - name: Ensure DEDUP_KV namespace
+        if: steps.guard.outputs.ok == 'true' && success()
+        run: |
+          case "${{ needs.detect.outputs.env }}" in
+            production) title=sigma-dedup-prod ;;
+            *)          title="sigma-dedup-${{ needs.detect.outputs.env }}" ;;
+          esac
+          id="$(node scripts/ensure-kv-namespace.mjs "$title")"
+          echo "::add-mask::$id"
+          echo "SIGMA_DEDUP_KV_ID=$id" >> "$GITHUB_ENV"
+
+      # GitOps: ensure the account-scoped `bggpt-voice` custom provider (+ the shared `sigma-assistant`
+      # gateway) the VOICE lane calls directly. Provider-only — the assistant hits the gateway's provider
+      # endpoints for transcription, NOT a dynamic route (dynamic routing can't carry audio; see ADR-0011).
+      # Idempotent: a no-op when the provider already exists. VOICE_ASSISTANT_API_KEY is used only to
+      # first-create the provider; absent ⇒ the existing provider is reused as-is.
+      - name: Ensure voice AI-Gateway provider
+        if: steps.guard.outputs.ok == 'true' && success()
+        env:
+          VOICE_ASSISTANT_API_KEY: ${{ secrets.VOICE_ASSISTANT_API_KEY }}
+        run: node scripts/ensure-voice-provider.mjs --apply
+
       - name: Typecheck
-        if: steps.guard.outputs.ok == 'true'
+        if: steps.guard.outputs.ok == 'true' && success()
         run: pnpm typecheck
 
       # The base schema was created out-of-band with `d1 execute --file`, so wrangler's migration
@@ -196,11 +261,11 @@ jobs:
 
       # `run deploy`, not `deploy` — bare `pnpm deploy` is a pnpm built-in, not our package script.
       - name: Deploy explorer (sigma)
-        if: steps.guard.outputs.ok == 'true'
+        if: steps.guard.outputs.ok == 'true' && success()
         run: pnpm --filter @sigma/web run deploy
 
       - name: Initialize LOG_IP_KEY secret if absent
-        if: steps.guard.outputs.ok == 'true'
+        if: steps.guard.outputs.ok == 'true' && success()
         run: |
           # LOG_IP_KEY is long-lived so IP HMAC tokens remain stable for same-client correlation.
           # Never overwrite it; a new blue-green worker name starts without a key and correlation
@@ -238,8 +303,22 @@ jobs:
           fi
 
           key="$(openssl rand -hex 32)"
+          # Register the value as a secret with the runner BEFORE it is used anywhere else, so an
+          # accidental echo / set -x trace in a later edit can never leak it. Masking is not
+          # retroactive within a line, hence add-mask on the line right after generation.
+          echo "::add-mask::$key"
           printf '%s' "$key" | pnpm --filter @sigma/web exec wrangler secret put LOG_IP_KEY --name "$SIGMA_WEB_NAME"
 
+      # The §9.3 transcript-signing key (ADR-0011/0012). Same generate-if-absent story as LOG_IP_KEY above:
+      # a purely internal HMAC key with no human value, generated once per worker and left stable across
+      # redeploys (rotating it would invalidate every in-flight client transcript at once). On production +
+      # staging the runtime gate fails CLOSED without it, so this step must run on every deploy to keep the
+      # required key present. The key is generated in-process and streamed to wrangler over stdin — it never
+      # appears in a log line, so no ::add-mask:: is needed. Runs after the deploy: the worker must exist first.
+      - name: Initialize ASSISTANT_HMAC_KEY secret if absent
+        if: steps.guard.outputs.ok == 'true' && success()
+        run: node scripts/ensure-worker-secret.mjs ASSISTANT_HMAC_KEY
+
       - name: Deploy refresh Workflow (sigma-etl)
-        if: steps.guard.outputs.ok == 'true'
+        if: steps.guard.outputs.ok == 'true' && success()
         run: pnpm --filter @sigma/etl run deploy

.gitignore

@@ -43,3 +43,6 @@ worker-configuration.d.ts
 
 # Local-only policy/regulatory source materials copied in for review (never commit)
 .policy-source/
+
+# Playwright MCP session artifacts (screenshots / a11y snapshots)
+.playwright-mcp/

apps/etl/package.json

@@ -11,6 +11,8 @@
     "typecheck": "tsc --noEmit"
   },
   "dependencies": {
-    "@sigma/ingest": "workspace:*"
+    "@sigma/config": "workspace:*",
+    "@sigma/ingest": "workspace:*",
+    "@sigma/shared": "workspace:*"
   }
 }

apps/etl/src/cron-guard.test.ts

@@ -0,0 +1,27 @@
+/// <reference types="node" />
+import { readFileSync } from 'node:fs';
+import { dirname, resolve } from 'node:path';
+import { fileURLToPath } from 'node:url';
+import { describe, expect, it } from 'vitest';
+import { PROMPTS_CRON, REFRESH_CRON } from './crons';
+
+// Routing safety: scheduled() branches on controller.cron against the named constants. A typo in
+// wrangler.toml's `crons` (or in the constants) would silently misroute a trigger, so this parses the
+// committed `crons` array and asserts it equals exactly [REFRESH_CRON, PROMPTS_CRON] — a mismatch
+// fails CI instead of misfiring in production.
+
+const wranglerPath = resolve(dirname(fileURLToPath(import.meta.url)), '../wrangler.toml');
+
+function parseCrons(toml: string): string[] {
+  const match = toml.match(/crons\s*=\s*\[([^\]]*)\]/);
+  const inner = match?.[1];
+  if (inner === undefined) throw new Error('no `crons = [...]` array found in wrangler.toml');
+  return [...inner.matchAll(/"([^"]*)"/g)].map((m) => m[1] ?? '');
+}
+
+describe('cron routing guard', () => {
+  it('wrangler crons equal [REFRESH_CRON, PROMPTS_CRON] in order', () => {
+    const crons = parseCrons(readFileSync(wranglerPath, 'utf8'));
+    expect(crons).toStrictEqual([REFRESH_CRON, PROMPTS_CRON]);
+  });
+});

apps/etl/src/crons.ts

@@ -0,0 +1,5 @@
+// Cron strings shared by wrangler.toml's `crons`, scheduled()'s routing branch (index.ts), and the
+// cron-guard test. Kept in a dependency-free module (no `cloudflare:workers` / `.sql` text imports) so
+// the guard test can import them under plain vitest without pulling in the Workflow runtime.
+export const REFRESH_CRON = '0 */6 * * *';
+export const PROMPTS_CRON = '0 6 * * 1';