Implement Phase 2 webhook delivery truth and suppression

.github/workflows/ci.yml

@@ -54,7 +54,7 @@ jobs:
           PGPASSWORD=postgres createdb -h 127.0.0.1 -U postgres ONA_upgrade
           rm -rf /tmp/db-pre-lifecycle
           cp -a db /tmp/db-pre-lifecycle
-          sed -i '/v1_6_survey_lifecycle_email_delivery.sql/d' /tmp/db-pre-lifecycle/changelogs/master-changelog.xml
+          sed -i -e '/v1_6_survey_lifecycle_email_delivery.sql/d' -e '/v1_7_email_webhook_delivery_truth.sql/d' /tmp/db-pre-lifecycle/changelogs/master-changelog.xml
           docker run --rm --network host \
             -v /tmp/db-pre-lifecycle:/db \
             liquibase/liquibase:4.29 \
@@ -141,6 +141,16 @@ jobs:
       - name: Smoke test API against migrated database
         run: bash scripts/ci/api-smoke.sh
 
+      - name: Smoke test webhook projection and suppression fencing
+        env:
+          DB_USER: postgres
+          DB_PASSWORD: postgres
+          DB_HOST: 127.0.0.1
+          DB_PORT: 5432
+          DB_NAME: ONA
+          SURVEY_URL: http://localhost:3002
+        run: node scripts/ci/webhook-worker-smoke.js
+
   frontend:
     name: Build & test (${{ matrix.app }})
     runs-on: ubuntu-latest

.github/workflows/deploy.yml

@@ -135,13 +135,11 @@ jobs:
           mkdir -p "$STAGE_DIR/deploy" "$STAGE_DIR/db"
           rsync -a --exclude node_modules api "$STAGE_DIR/"
           cp -a db/changelogs "$STAGE_DIR/db/"
-          cp scripts/deploy/remote-deploy.sh \
-             scripts/deploy/bootstrap-admin.js \
-             scripts/deploy/finalize-legacy-accounts.js \
-             scripts/deploy/ecosystem.config.js \
-             scripts/deploy/set-email-claiming.js \
-             "$STAGE_DIR/deploy/"
+          cp scripts/deploy/remote-deploy.sh scripts/deploy/*.js \
+             scripts/deploy/CAPABILITIES.json "$STAGE_DIR/deploy/"
           echo "$GITHUB_SHA" > "$STAGE_DIR/REVISION"
+          node "$STAGE_DIR/deploy/validate-release-capabilities.js" "$STAGE_DIR"
+          test -f "$STAGE_DIR/api/webhook-worker.js"
           tar -czf api-release.tar.gz -C "$STAGE_DIR" .
 
       - name: Upload artifact to S3
@@ -199,6 +197,15 @@ jobs:
             exit 1
           fi
 
+      - name: Promote bootstrappable API artifact
+        # The on-instance installer has already verified the exact API and both
+        # workers. Promote this pointer before unrelated frontend publication so
+        # replacement instances cannot bootstrap an older capability floor.
+        run: |
+          aws s3 cp \
+            "s3://${{ steps.env.outputs.ARTIFACTS_BUCKET }}/api/$GITHUB_SHA.tar.gz" \
+            "s3://${{ steps.env.outputs.ARTIFACTS_BUCKET }}/api/latest-compatible.tar.gz"
+
       # ---------- Frontends: build → S3 sync → CloudFront invalidation ----------
 
       - name: Build dashboard

.github/workflows/rollback-api.yml

@@ -130,7 +130,8 @@ jobs:
               "rm -rf /tmp/ona-deploy && mkdir -p /tmp/ona-deploy",
               "aws s3 cp s3://\($bucket)/api/\($sha).tar.gz /tmp/ona-deploy/release.tar.gz",
               "tar -xzf /tmp/ona-deploy/release.tar.gz -C /tmp/ona-deploy",
-              "test -f /tmp/ona-deploy/api/lifecycle.js && test -f /tmp/ona-deploy/api/email-worker.js && test -f /tmp/ona-deploy/deploy/ecosystem.config.js && test -f /tmp/ona-deploy/deploy/set-email-claiming.js || { echo Refusing rollback to an invalid or pre-lifecycle artifact >&2; exit 1; }",
+              "test -f /tmp/ona-deploy/api/lifecycle.js && test -f /tmp/ona-deploy/api/email-worker.js && test -f /tmp/ona-deploy/api/webhook-worker.js && test -f /tmp/ona-deploy/deploy/ecosystem.config.js && test -f /tmp/ona-deploy/deploy/set-email-claiming.js && test -f /tmp/ona-deploy/deploy/CAPABILITIES.json && test -f /tmp/ona-deploy/deploy/validate-release-capabilities.js || { echo Refusing rollback to an invalid or capability-incompatible artifact >&2; exit 1; }",
+              "node /tmp/ona-deploy/deploy/validate-release-capabilities.js /tmp/ona-deploy",
               "bash /tmp/ona-deploy/deploy/remote-deploy.sh /tmp/ona-deploy",
               "rm -rf /tmp/ona-deploy"
             ]
@@ -174,3 +175,6 @@ jobs:
           aws s3 cp \
             "s3://${{ steps.env.outputs.ARTIFACTS_BUCKET }}/api/${{ inputs.artifact_sha }}.tar.gz" \
             "s3://${{ steps.env.outputs.ARTIFACTS_BUCKET }}/api/latest.tar.gz"
+          aws s3 cp \
+            "s3://${{ steps.env.outputs.ARTIFACTS_BUCKET }}/api/${{ inputs.artifact_sha }}.tar.gz" \
+            "s3://${{ steps.env.outputs.ARTIFACTS_BUCKET }}/api/latest-compatible.tar.gz"

.github/workflows/terraform-apply.yml

@@ -17,7 +17,7 @@ permissions:
 concurrency:
   # Production infrastructure/config updates serialize with production API
   # releases so deploy cannot read partially updated IAM or runtime config.
-  group: ${{ inputs.environment == 'prod' && 'api-release-production' || 'terraform-apply-staging' }}
+  group: ${{ inputs.environment == 'prod' && 'api-release-production' || 'api-release-staging' }}
   cancel-in-progress: false
 
 jobs:

api/.env.local.example

@@ -20,6 +20,12 @@ RESEND_API_KEY=replace-me
 # Durable lifecycle launch/worker settings.
 SURVEY_DELIVERY_V2_ENABLED=true
 LEGACY_START_ENABLED=true
+RESEND_PROVIDER_ACCOUNT_SCOPE=network-survey-resend-team
+RESEND_WEBHOOK_INGEST_ENABLED=false
+RESEND_WEBHOOK_SECRET=
+RESEND_WEBHOOK_PREVIOUS_SECRET=
+WEBHOOK_PAYLOAD_RETENTION_DAYS=30
+WEBHOOK_METRIC_NAMESPACE=NetworkSurvey/Email
 EMAIL_WORKER_ENV=local
 EMAIL_WORKER_HEARTBEAT_MAX_AGE_SECONDS=45
 EMAIL_LEASE_SECONDS=60

api/email-metrics.js

@@ -0,0 +1,22 @@
+'use strict';
+
+function emitMetrics({ namespace = process.env.WEBHOOK_METRIC_NAMESPACE || 'NetworkSurvey/Email', environment, release, metrics }) {
+  const names = Object.keys(metrics || {});
+  if (!environment || names.length === 0) return;
+  const record = {
+    _aws: {
+      Timestamp: Date.now(),
+      CloudWatchMetrics: [{
+        Namespace: namespace,
+        Dimensions: [['Environment']],
+        Metrics: names.map((Name) => ({ Name, Unit: Name.endsWith('AgeSeconds') ? 'Seconds' : 'Count' })),
+      }],
+    },
+    Environment: environment,
+    Release: release || 'unknown',
+    ...metrics,
+  };
+  console.log(JSON.stringify(record));
+}
+
+module.exports = { emitMetrics };