diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index d9acc33..b413ffb 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -54,7 +54,7 @@ jobs: PGPASSWORD=postgres createdb -h 127.0.0.1 -U postgres ONA_upgrade rm -rf /tmp/db-pre-lifecycle cp -a db /tmp/db-pre-lifecycle - sed -i '/v1_6_survey_lifecycle_email_delivery.sql/d' /tmp/db-pre-lifecycle/changelogs/master-changelog.xml + sed -i -e '/v1_6_survey_lifecycle_email_delivery.sql/d' -e '/v1_7_email_webhook_delivery_truth.sql/d' /tmp/db-pre-lifecycle/changelogs/master-changelog.xml docker run --rm --network host \ -v /tmp/db-pre-lifecycle:/db \ liquibase/liquibase:4.29 \ @@ -141,6 +141,16 @@ jobs: - name: Smoke test API against migrated database run: bash scripts/ci/api-smoke.sh + - name: Smoke test webhook projection and suppression fencing + env: + DB_USER: postgres + DB_PASSWORD: postgres + DB_HOST: 127.0.0.1 + DB_PORT: 5432 + DB_NAME: ONA + SURVEY_URL: http://localhost:3002 + run: node scripts/ci/webhook-worker-smoke.js + frontend: name: Build & test (${{ matrix.app }}) runs-on: ubuntu-latest diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index df011d5..d9f4b8e 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -135,13 +135,11 @@ jobs: mkdir -p "$STAGE_DIR/deploy" "$STAGE_DIR/db" rsync -a --exclude node_modules api "$STAGE_DIR/" cp -a db/changelogs "$STAGE_DIR/db/" - cp scripts/deploy/remote-deploy.sh \ - scripts/deploy/bootstrap-admin.js \ - scripts/deploy/finalize-legacy-accounts.js \ - scripts/deploy/ecosystem.config.js \ - scripts/deploy/set-email-claiming.js \ - "$STAGE_DIR/deploy/" + cp scripts/deploy/remote-deploy.sh scripts/deploy/*.js \ + scripts/deploy/CAPABILITIES.json "$STAGE_DIR/deploy/" echo "$GITHUB_SHA" > "$STAGE_DIR/REVISION" + node "$STAGE_DIR/deploy/validate-release-capabilities.js" "$STAGE_DIR" + test -f "$STAGE_DIR/api/webhook-worker.js" tar -czf api-release.tar.gz -C "$STAGE_DIR" . - name: Upload artifact to S3 @@ -199,6 +197,15 @@ jobs: exit 1 fi + - name: Promote bootstrappable API artifact + # The on-instance installer has already verified the exact API and both + # workers. Promote this pointer before unrelated frontend publication so + # replacement instances cannot bootstrap an older capability floor. + run: | + aws s3 cp \ + "s3://${{ steps.env.outputs.ARTIFACTS_BUCKET }}/api/$GITHUB_SHA.tar.gz" \ + "s3://${{ steps.env.outputs.ARTIFACTS_BUCKET }}/api/latest-compatible.tar.gz" + # ---------- Frontends: build → S3 sync → CloudFront invalidation ---------- - name: Build dashboard diff --git a/.github/workflows/rollback-api.yml b/.github/workflows/rollback-api.yml index ea7d50c..cad7309 100644 --- a/.github/workflows/rollback-api.yml +++ b/.github/workflows/rollback-api.yml @@ -130,7 +130,8 @@ jobs: "rm -rf /tmp/ona-deploy && mkdir -p /tmp/ona-deploy", "aws s3 cp s3://\($bucket)/api/\($sha).tar.gz /tmp/ona-deploy/release.tar.gz", "tar -xzf /tmp/ona-deploy/release.tar.gz -C /tmp/ona-deploy", - "test -f /tmp/ona-deploy/api/lifecycle.js && test -f /tmp/ona-deploy/api/email-worker.js && test -f /tmp/ona-deploy/deploy/ecosystem.config.js && test -f /tmp/ona-deploy/deploy/set-email-claiming.js || { echo Refusing rollback to an invalid or pre-lifecycle artifact >&2; exit 1; }", + "test -f /tmp/ona-deploy/api/lifecycle.js && test -f /tmp/ona-deploy/api/email-worker.js && test -f /tmp/ona-deploy/api/webhook-worker.js && test -f /tmp/ona-deploy/deploy/ecosystem.config.js && test -f /tmp/ona-deploy/deploy/set-email-claiming.js && test -f /tmp/ona-deploy/deploy/CAPABILITIES.json && test -f /tmp/ona-deploy/deploy/validate-release-capabilities.js || { echo Refusing rollback to an invalid or capability-incompatible artifact >&2; exit 1; }", + "node /tmp/ona-deploy/deploy/validate-release-capabilities.js /tmp/ona-deploy", "bash /tmp/ona-deploy/deploy/remote-deploy.sh /tmp/ona-deploy", "rm -rf /tmp/ona-deploy" ] @@ -174,3 +175,6 @@ jobs: aws s3 cp \ "s3://${{ steps.env.outputs.ARTIFACTS_BUCKET }}/api/${{ inputs.artifact_sha }}.tar.gz" \ "s3://${{ steps.env.outputs.ARTIFACTS_BUCKET }}/api/latest.tar.gz" + aws s3 cp \ + "s3://${{ steps.env.outputs.ARTIFACTS_BUCKET }}/api/${{ inputs.artifact_sha }}.tar.gz" \ + "s3://${{ steps.env.outputs.ARTIFACTS_BUCKET }}/api/latest-compatible.tar.gz" diff --git a/.github/workflows/terraform-apply.yml b/.github/workflows/terraform-apply.yml index 6e34d7c..f11d4ef 100644 --- a/.github/workflows/terraform-apply.yml +++ b/.github/workflows/terraform-apply.yml @@ -17,7 +17,7 @@ permissions: concurrency: # Production infrastructure/config updates serialize with production API # releases so deploy cannot read partially updated IAM or runtime config. - group: ${{ inputs.environment == 'prod' && 'api-release-production' || 'terraform-apply-staging' }} + group: ${{ inputs.environment == 'prod' && 'api-release-production' || 'api-release-staging' }} cancel-in-progress: false jobs: diff --git a/api/.env.local.example b/api/.env.local.example index d60a93f..9e20347 100644 --- a/api/.env.local.example +++ b/api/.env.local.example @@ -20,6 +20,12 @@ RESEND_API_KEY=replace-me # Durable lifecycle launch/worker settings. SURVEY_DELIVERY_V2_ENABLED=true LEGACY_START_ENABLED=true +RESEND_PROVIDER_ACCOUNT_SCOPE=network-survey-resend-team +RESEND_WEBHOOK_INGEST_ENABLED=false +RESEND_WEBHOOK_SECRET= +RESEND_WEBHOOK_PREVIOUS_SECRET= +WEBHOOK_PAYLOAD_RETENTION_DAYS=30 +WEBHOOK_METRIC_NAMESPACE=NetworkSurvey/Email EMAIL_WORKER_ENV=local EMAIL_WORKER_HEARTBEAT_MAX_AGE_SECONDS=45 EMAIL_LEASE_SECONDS=60 diff --git a/api/email-metrics.js b/api/email-metrics.js new file mode 100644 index 0000000..c14ed9d --- /dev/null +++ b/api/email-metrics.js @@ -0,0 +1,22 @@ +'use strict'; + +function emitMetrics({ namespace = process.env.WEBHOOK_METRIC_NAMESPACE || 'NetworkSurvey/Email', environment, release, metrics }) { + const names = Object.keys(metrics || {}); + if (!environment || names.length === 0) return; + const record = { + _aws: { + Timestamp: Date.now(), + CloudWatchMetrics: [{ + Namespace: namespace, + Dimensions: [['Environment']], + Metrics: names.map((Name) => ({ Name, Unit: Name.endsWith('AgeSeconds') ? 'Seconds' : 'Count' })), + }], + }, + Environment: environment, + Release: release || 'unknown', + ...metrics, + }; + console.log(JSON.stringify(record)); +} + +module.exports = { emitMetrics }; diff --git a/api/email-worker.js b/api/email-worker.js index 85c0aec..1e2d113 100644 --- a/api/email-worker.js +++ b/api/email-worker.js @@ -5,7 +5,7 @@ const crypto = require('crypto'); const fs = require('fs'); const dotenvFlow = require('dotenv-flow'); const { Pool } = require('pg'); -const { ResendProvider, buildInvitationPayload, payloadHash, classifyProviderError, sanitizeProviderMessage, reserveProviderRate } = require('./email'); +const { ResendProvider, buildInvitationPayload, payloadHash, classifyProviderError, sanitizeProviderMessage, reserveProviderRateInTransaction } = require('./email'); const { environmentName } = require('./lifecycle'); dotenvFlow.config(); @@ -41,35 +41,49 @@ class DeliveryWorker { const token=crypto.randomUUID();const updated=await client.query(`UPDATE survey_email_deliveries SET status='leased',lease_owner=$2,lease_token=$3,lease_expires_at=now()+($4::text||' seconds')::interval,attempt_count=attempt_count+1,updated_at=now() WHERE id=$1 RETURNING *`,[previous.id,this.instanceId,token,this.leaseSeconds]);const delivery=updated.rows[0]; await client.query(`INSERT INTO survey_email_attempts(delivery_id,attempt_number,lease_token,outcome) VALUES($1,$2,$3,'in_progress')`,[delivery.id,delivery.attempt_count,token]);await client.query('COMMIT');return delivery; }catch(error){await client.query('ROLLBACK').catch(()=>{});throw error;}finally{client.release();}} - async startProviderRequest(delivery){const client=await this.pool.connect();let globalBoundaryLock=false;let surveyBoundaryLock=false;try{await client.query('BEGIN'); + async startProviderRequest(delivery){const client=await this.pool.connect();let globalBoundaryLock=false;let surveyBoundaryLock=false;let addressBoundaryLock=false;let addressBoundaryKey=null;try{await client.query('BEGIN'); await client.query(`SELECT pg_advisory_lock(hashtextextended($1,0))`,[`email-provider-boundary:${this.environment}`]);globalBoundaryLock=true; const control=(await client.query(`SELECT claiming_enabled,minimum_release FROM email_worker_control WHERE environment=$1 FOR SHARE`,[this.environment])).rows[0]; + const sendingControl=(await client.query(`SELECT sending_enabled,minimum_release FROM email_sending_control WHERE environment=$1 FOR SHARE`,[this.environment])).rows[0]; await client.query(`SELECT pg_advisory_lock(hashtextextended($1,0))`,[`survey-provider-boundary:${delivery.survey_id}`]);surveyBoundaryLock=true; const survey=(await client.query(`SELECT id,name,lifecycle_status,archived_at FROM survey WHERE id=$1 FOR SHARE`,[delivery.survey_id])).rows[0]; const result=await client.query(`SELECT d.*,r.uuid,t.body_text,l.cancelled_at FROM survey_email_deliveries d JOIN respondent r ON r.respondent_id=d.respondent_id AND r.survey_id=d.survey_id JOIN survey_launches l ON l.id=d.launch_id JOIN survey_launch_templates t ON t.launch_id=d.launch_id AND t.language=d.language WHERE d.id=$1 AND d.status='leased' AND d.lease_token=$2 FOR UPDATE OF d`,[delivery.id,delivery.lease_token]); const row=result.rows[0];if(!row||!survey){await client.query('COMMIT');return {action:'stale'};} row.survey_name=survey.name; - if(!control?.claiming_enabled||(control.minimum_release&&this.release!==control.minimum_release)){await this.releaseWithoutSend(client,row,'worker_disabled_before_send');await client.query('COMMIT');return {action:'disabled'};} + if(!control?.claiming_enabled||(control.minimum_release&&this.release!==control.minimum_release)||!sendingControl?.sending_enabled||(sendingControl.minimum_release&&this.release!==sendingControl.minimum_release)){await this.releaseWithoutSend(client,row,'worker_disabled_before_send');await client.query('COMMIT');return {action:'disabled'};} if(row.cancellation_requested_at||row.cancelled_at||survey.lifecycle_status!=='active'||survey.archived_at){await this.cancel(client,row,'survey_inactive');await client.query('COMMIT');return {action:'cancelled'};} const unresolved=await client.query(`SELECT MIN(provider_started_at) AS first_provider_started_at,COUNT(provider_started_at)::int AS provider_attempt_count FROM survey_email_attempts WHERE delivery_id=$1 AND outcome='uncertain'`,[row.id]); const unresolvedBoundary=unresolved.rows[0]?.first_provider_started_at; if(unresolvedBoundary&&!canRetryAmbiguous({firstProviderStartedAt:unresolvedBoundary,providerAttemptCount:unresolved.rows[0]?.provider_attempt_count,createdAt:row.created_at,now:this.clock(),idempotencyHours:this.idempotencyHours,maxAttempts:this.maxAttempts,maxAgeHours:this.maxAgeHours})){await this.finalizeTerminal(client,row,'uncertain','ambiguous_retry_window_exhausted','Provider acceptance could not be reconciled safely before retry');await client.query('COMMIT');return {action:'uncertain'};} - const payload=buildInvitationPayload({to:row.to_address,sender:row.sender,subject:row.subject,bodyText:row.body_text,surveyBaseUrl:row.survey_base_url,surveyName:survey.name,token:row.uuid,language:row.language}); + addressBoundaryKey=`email-suppression-boundary:${this.env.RESEND_PROVIDER_ACCOUNT_SCOPE||this.environment}:${String(row.to_address).trim().toLowerCase()}`; + await client.query(`SELECT pg_advisory_lock(hashtextextended($1,0))`,[addressBoundaryKey]);addressBoundaryLock=true; + const suppressionControl=(await client.query(`SELECT enforcement_enabled,minimum_release FROM email_suppression_control WHERE environment=$1`,[this.environment])).rows[0]; + if(suppressionControl?.enforcement_enabled&&suppressionControl.minimum_release&&suppressionControl.minimum_release!==this.release){await this.releaseWithoutSend(client,row,'suppression_release_incompatible');await client.query('COMMIT');return {action:'disabled'};} + if(suppressionControl?.enforcement_enabled){ + const blocked=await client.query(`SELECT 1 FROM email_suppressions WHERE provider_account_scope=$1 AND normalized_address=$2 AND (provider_active OR locally_overridden_at IS NULL) LIMIT 1`,[this.env.RESEND_PROVIDER_ACCOUNT_SCOPE||this.environment,String(row.to_address).trim().toLowerCase()]); + if(blocked.rowCount){await this.cancelSuppressed(client,row);await client.query('COMMIT');return {action:'suppressed'};} + } + const tagged=row.renderer_version==='survey-invitation-v2'; + const payload=buildInvitationPayload({to:row.to_address,sender:row.sender,subject:row.subject,bodyText:row.body_text,surveyBaseUrl:row.survey_base_url,surveyName:survey.name,token:row.uuid,language:row.language,deliveryId:tagged?row.id:null,environment:tagged?this.environment:null}); if(payloadHash(payload)!==row.expected_payload_hash){await this.finalizeTerminal(client,row,'uncertain','payload_hash_mismatch','Rendered provider payload no longer matches launch snapshot');await client.query('COMMIT');return {action:'mismatch'};} const remaining=new Date(row.lease_expires_at).getTime()-this.clock().getTime()-2000; if(remaining<=1000){await this.releaseWithoutSend(client,row,'lease_too_short_before_provider');await client.query('COMMIT');return {action:'short_lease'};} - if(!await reserveProviderRate(this.pool,this.rateBudgetEnvironment,this.rate)){await this.releaseWithoutSend(client,row,'provider_rate_wait');await client.query('COMMIT');return {action:'rate_wait'};} + if(!await reserveProviderRateInTransaction(client,this.rateBudgetEnvironment,this.rate)){await this.releaseWithoutSend(client,row,'provider_rate_wait');await client.query('COMMIT');return {action:'rate_wait'};} await client.query(`UPDATE survey_email_attempts SET provider_started_at=now() WHERE delivery_id=$1 AND lease_token=$2 AND outcome='in_progress'`,[row.id,row.lease_token]); await client.query('COMMIT'); - const providerResult=this.provider.send(payload,{idempotencyKey:delivery.provider_idempotency_key,timeoutMs:Math.min(Number(this.env.EMAIL_PROVIDER_TIMEOUT_MS||15000),remaining)}).then((result)=>({result}),(error)=>({error})); + let providerResult; + try{providerResult={result:await this.provider.send(payload,{idempotencyKey:delivery.provider_idempotency_key,timeoutMs:Math.min(Number(this.env.EMAIL_PROVIDER_TIMEOUT_MS||15000),remaining)})};} + catch(error){providerResult={error};} return {action:'send',row,providerResult}; }catch(error){await client.query('ROLLBACK').catch(()=>{});throw error;}finally{ + if(addressBoundaryLock)await client.query(`SELECT pg_advisory_unlock(hashtextextended($1,0))`,[addressBoundaryKey]).catch(()=>{}); if(surveyBoundaryLock)await client.query(`SELECT pg_advisory_unlock(hashtextextended($1,0))`,[`survey-provider-boundary:${delivery.survey_id}`]).catch(()=>{}); if(globalBoundaryLock)await client.query(`SELECT pg_advisory_unlock(hashtextextended($1,0))`,[`email-provider-boundary:${this.environment}`]).catch(()=>{}); client.release(); }} async releaseWithoutSend(client,row,reason){await client.query(`UPDATE survey_email_attempts SET outcome='cancelled',finished_at=now(),error_message=$3 WHERE delivery_id=$1 AND lease_token=$2 AND outcome='in_progress'`,[row.id,row.lease_token,reason]);await client.query(`UPDATE survey_email_deliveries SET status='pending',lease_owner=NULL,lease_token=NULL,lease_expires_at=NULL,next_attempt_at=CASE WHEN $3='provider_rate_wait' THEN now()+interval '100 milliseconds' ELSE now() END,updated_at=now(),last_error_code=$3 WHERE id=$1 AND status='leased' AND lease_token=$2`,[row.id,row.lease_token,reason]);} async cancel(client,row,reason){await client.query(`UPDATE survey_email_attempts SET outcome='cancelled',finished_at=now(),error_message=$3 WHERE delivery_id=$1 AND lease_token=$2 AND outcome='in_progress'`,[row.id,row.lease_token,reason]);await client.query(`UPDATE survey_email_deliveries SET status='cancelled',lease_owner=NULL,lease_token=NULL,lease_expires_at=NULL,updated_at=now(),last_error_code=$3 WHERE id=$1 AND status='leased' AND lease_token=$2`,[row.id,row.lease_token,reason]);} + async cancelSuppressed(client,row){await client.query(`UPDATE survey_email_attempts SET outcome='cancelled',finished_at=now(),error_message='suppressed' WHERE delivery_id=$1 AND lease_token=$2 AND outcome='in_progress'`,[row.id,row.lease_token]);await client.query(`UPDATE survey_email_deliveries SET status='cancelled',provider_suppressed_at=COALESCE(provider_suppressed_at,now()),lease_owner=NULL,lease_token=NULL,lease_expires_at=NULL,updated_at=now(),last_error_code='suppressed',last_error_message='Recipient is suppressed' WHERE id=$1 AND status='leased' AND lease_token=$2`,[row.id,row.lease_token]);} async finalizeTerminal(client,row,status,code,message,providerId=null){const outcome=status==='failed'?'permanent_failure':status;await client.query(`UPDATE survey_email_attempts SET outcome=$3,finished_at=now(),provider_code=$4,error_message=$5,provider_message_id=$6 WHERE delivery_id=$1 AND lease_token=$2 AND outcome='in_progress'`,[row.id,row.lease_token,outcome,code,bounded(message),providerId]);return client.query(`UPDATE survey_email_deliveries SET status=$3,provider_message_id=COALESCE($6,provider_message_id),dispatch_accepted_at=CASE WHEN $3='accepted' THEN now() ELSE dispatch_accepted_at END,dispatch_failed_at=CASE WHEN $3 IN ('failed','uncertain') THEN now() ELSE dispatch_failed_at END,last_error_code=$4,last_error_message=$5,lease_owner=NULL,lease_token=NULL,lease_expires_at=NULL,updated_at=now() WHERE id=$1 AND status='leased' AND lease_token=$2`,[row.id,row.lease_token,status,code,bounded(message),providerId]);} backoff(attempt,retryAfter){const seconds=Number.parseInt(retryAfter,10);if(Number.isFinite(seconds)&&seconds>0)return Math.min(seconds*1000,3600000);const cap=Math.min(3600000,1000*(2**Math.min(attempt,12)));return Math.floor(this.random()*cap);} async finalizeFailure(row,error){const classification=classifyProviderError(error);const client=await this.pool.connect();try{await client.query('BEGIN');if(classification==='quota'){await client.query(`SELECT pg_advisory_xact_lock(hashtextextended($1,0))`,[`email-provider-boundary:${this.environment}`]);await client.query(`UPDATE email_worker_control SET claiming_enabled=false,updated_at=now(),reason='provider quota requires operator action' WHERE environment=$1`,[this.environment]);}const locked=await client.query(`SELECT * FROM survey_email_deliveries WHERE id=$1 AND status='leased' AND lease_token=$2 FOR UPDATE`,[row.id,row.lease_token]);const current=locked.rows[0];if(!current){await client.query('COMMIT');return;}const providerStats=(await client.query(`SELECT COUNT(provider_started_at)::int AS provider_attempt_count,MIN(provider_started_at) AS first_provider_started_at FROM survey_email_attempts WHERE delivery_id=$1`,[current.id])).rows[0];if(current.cancellation_requested_at){await this.cancel(client,current,'cancelled_after_provider_failure');} diff --git a/api/email.js b/api/email.js index 8f51834..6e20470 100644 --- a/api/email.js +++ b/api/email.js @@ -3,7 +3,7 @@ const crypto = require('crypto'); const DEFAULT_SENDER = 'CLA Survey '; -const RENDERER_VERSION = 'survey-invitation-v1'; +const RENDERER_VERSION = 'survey-invitation-v2'; function escapeHtml(value) { return String(value ?? '').replace(/[&<>"']/g, (character) => ({ @@ -45,10 +45,18 @@ function renderInvitation({ bodyText, link, language = 'en' }) { return { html, text }; } -function buildInvitationPayload({ to, sender = DEFAULT_SENDER, subject = 'CLA Network Survey', bodyText, surveyBaseUrl, surveyName, token, language }) { +function buildInvitationPayload({ to, sender = DEFAULT_SENDER, subject = 'CLA Network Survey', bodyText, surveyBaseUrl, surveyName, token, language, deliveryId, environment }) { const link = buildSurveyLink(surveyBaseUrl, surveyName, token); const rendered = renderInvitation({ bodyText, link, language }); - return { from: sender, to, subject, html: rendered.html, text: rendered.text }; + const payload = { from: sender, to, subject, html: rendered.html, text: rendered.text }; + if (deliveryId && environment) { + payload.tags = [ + { name: 'app', value: 'network_survey' }, + { name: 'environment', value: String(environment).replace(/[^A-Za-z0-9_-]/g, '_').slice(0, 256) }, + { name: 'delivery_id', value: String(deliveryId) }, + ]; + } + return payload; } function payloadHash(payload) { @@ -122,28 +130,33 @@ class ResendProvider { } } -async function reserveProviderRate(pool, environment, rate) { - const client = await pool.connect(); +async function reserveProviderRateInTransaction(client, environment, rate) { + await client.query(`SELECT pg_advisory_xact_lock(hashtextextended($1,0))`, [`email-rate-budget:${environment}`]); + await client.query(`DELETE FROM email_rate_reservations WHERE environment=$1 AND reserved_atclock_timestamp()-interval '1 second'`, [environment]); + if (Number(used.rows[0]?.count || 0) >= rate) return false; + await client.query(`INSERT INTO email_rate_reservations(environment,reserved_at) VALUES($1,clock_timestamp())`, [environment]); + return true; +} + +async function reserveProviderRateOnClient(client, environment, rate) { try { await client.query('BEGIN'); - await client.query(`SELECT pg_advisory_xact_lock(hashtextextended($1,0))`, [`email-rate-budget:${environment}`]); - await client.query(`DELETE FROM email_rate_reservations WHERE environment=$1 AND reserved_atclock_timestamp()-interval '1 second'`, [environment]); - if (Number(used.rows[0]?.count || 0) >= rate) { - await client.query('COMMIT'); - return false; - } - await client.query(`INSERT INTO email_rate_reservations(environment,reserved_at) VALUES($1,clock_timestamp())`, [environment]); + const reserved=await reserveProviderRateInTransaction(client,environment,rate); await client.query('COMMIT'); - return true; + return reserved; } catch (error) { await client.query('ROLLBACK').catch(() => {}); throw error; - } finally { - client.release(); } } +async function reserveProviderRate(pool, environment, rate) { + const client = await pool.connect(); + try { return await reserveProviderRateOnClient(client, environment, rate); } + finally { client.release(); } +} + function classifyProviderError(error) { const status = Number(error?.status || 0); if (error?.uncertain || error?.code === 'concurrent_idempotent_requests' || status >= 500) return 'ambiguous'; @@ -152,4 +165,4 @@ function classifyProviderError(error) { return 'permanent'; } -module.exports = { DEFAULT_SENDER, RENDERER_VERSION, escapeHtml, normalizeTemplateText, documentLanguage, buildSurveyLink, renderInvitation, buildInvitationPayload, payloadHash, ResendProvider, ProviderError, classifyProviderError, sanitizeProviderMessage, reserveProviderRate }; +module.exports = { DEFAULT_SENDER, RENDERER_VERSION, escapeHtml, normalizeTemplateText, documentLanguage, buildSurveyLink, renderInvitation, buildInvitationPayload, payloadHash, ResendProvider, ProviderError, classifyProviderError, sanitizeProviderMessage, reserveProviderRate, reserveProviderRateOnClient, reserveProviderRateInTransaction }; diff --git a/api/lifecycle.js b/api/lifecycle.js index 63ce9d9..0875cb1 100644 --- a/api/lifecycle.js +++ b/api/lifecycle.js @@ -125,15 +125,38 @@ function evaluateReadiness(survey, data, config = process.env) { }; } +async function applySuppressionReadiness(client, readiness, data, config) { + const environment = environmentName(config); + const control = await client.query('SELECT enforcement_enabled FROM email_suppression_control WHERE environment=$1', [environment]); + if (!control.rows[0]?.enforcement_enabled) return readiness; + const scope = String(config.RESEND_PROVIDER_ACCOUNT_SCOPE || '').trim(); + if (!scope) { + readiness.blockers.push({ code:'provider_account_scope_missing', message:'Provider account scope is not configured.' }); + readiness.blockerCount += 1; + readiness.canLaunch = false; + return readiness; + } + const addresses = data.recipients.map((row) => String(row.contact_info || '').trim().toLowerCase()); + const result = await client.query(`SELECT count(DISTINCT normalized_address)::int AS suppressed_count FROM email_suppressions WHERE provider_account_scope=$1 AND normalized_address=ANY($2::text[]) AND (provider_active OR locally_overridden_at IS NULL)`, [scope,addresses]); + const count = Number(result.rows[0]?.suppressed_count || 0); + if (count > 0) { + readiness.blockers.push({ code:'recipients_suppressed', count, message:`${count} eligible recipient${count === 1 ? ' is' : 's are'} suppressed. Resolve suppression before launching.` }); + readiness.blockerCount += 1; + readiness.canLaunch = false; + } + return readiness; +} + async function getReadiness(pool, user, surveyId, config = process.env) { const client = await pool.connect(); try { const survey = await loadAuthorizedSurvey(client, user, surveyId, 'editor'); const data = await loadReadinessData(client, survey); const readiness = evaluateReadiness(survey, data, config); + await applySuppressionReadiness(client, readiness, data, config); const env = environmentName(config); const maxAge = Math.max(5, Number(config.EMAIL_WORKER_HEARTBEAT_MAX_AGE_SECONDS || 45)); - const worker = await client.query(`SELECT 1 FROM email_worker_control c WHERE c.environment=$1 AND c.claiming_enabled=true AND EXISTS(SELECT 1 FROM email_worker_heartbeats h WHERE h.environment=c.environment AND h.enabled=true AND h.claiming=true AND h.heartbeat_at>now()-($2::text||' seconds')::interval AND (c.minimum_release='' OR h.release_revision=c.minimum_release))`, [env,maxAge]); + const worker = await client.query(`SELECT 1 FROM email_worker_control c JOIN email_sending_control s USING(environment) WHERE c.environment=$1 AND c.claiming_enabled=true AND s.sending_enabled=true AND (s.minimum_release='' OR c.minimum_release='' OR s.minimum_release=c.minimum_release) AND EXISTS(SELECT 1 FROM email_worker_heartbeats h WHERE h.environment=c.environment AND h.enabled=true AND h.claiming=true AND h.heartbeat_at>now()-($2::text||' seconds')::interval AND (c.minimum_release='' OR h.release_revision=c.minimum_release) AND (s.minimum_release='' OR h.release_revision=s.minimum_release))`, [env,maxAge]); if (!worker.rowCount) readiness.blockers.push({code:'worker_unavailable',message:'No fresh compatible email worker is available.'}); readiness.canLaunch = readiness.blockers.length === 0; return readiness; @@ -150,6 +173,14 @@ function aggregateSelect(whereSql) { count(DISTINCT d.id) FILTER(WHERE d.status='failed')::int AS failed_count, count(DISTINCT d.id) FILTER(WHERE d.status='uncertain')::int AS uncertain_count, count(DISTINCT d.id) FILTER(WHERE d.status='cancelled')::int AS cancelled_count, + count(DISTINCT d.id) FILTER(WHERE d.provider_sent_at IS NOT NULL)::int AS provider_sent_count, + count(DISTINCT d.id) FILTER(WHERE d.provider_delivered_at IS NOT NULL)::int AS provider_delivered_count, + count(DISTINCT d.id) FILTER(WHERE d.provider_delayed_at IS NOT NULL)::int AS provider_delayed_count, + count(DISTINCT d.id) FILTER(WHERE d.provider_bounced_at IS NOT NULL)::int AS provider_bounced_count, + count(DISTINCT d.id) FILTER(WHERE d.provider_complained_at IS NOT NULL)::int AS provider_complained_count, + count(DISTINCT d.id) FILTER(WHERE d.provider_suppressed_at IS NOT NULL)::int AS provider_suppressed_count, + count(DISTINCT d.id) FILTER(WHERE d.provider_failed_at IS NOT NULL)::int AS provider_failed_count, + count(DISTINCT d.id) FILTER(WHERE d.status='accepted' AND d.provider_sent_at IS NULL AND d.provider_delivered_at IS NULL AND d.provider_delayed_at IS NULL AND d.provider_bounced_at IS NULL AND d.provider_complained_at IS NULL AND d.provider_suppressed_at IS NULL AND d.provider_failed_at IS NULL)::int AS accepted_unverified_count, min(a.started_at) AS started_at,max(a.finished_at) FILTER(WHERE d.status IN ('accepted','failed','uncertain','cancelled')) AS finished_at, CASE WHEN count(DISTINCT d.id)>0 AND count(DISTINCT d.id) FILTER(WHERE d.status='pending')=count(DISTINCT d.id) AND count(a.id)=0 THEN 'queued' @@ -172,6 +203,8 @@ async function launchSurvey(pool, user, surveyId, { kind = 'initial', idempotenc const env = environmentName(config); const controlResult = await client.query('SELECT * FROM email_worker_control WHERE environment=$1 FOR SHARE', [env]); if (!controlResult.rows[0]) throw new LifecycleError(503, 'worker_unavailable', 'Email worker control is not configured.'); + const sendingResult = await client.query('SELECT * FROM email_sending_control WHERE environment=$1 FOR SHARE', [env]); + if (!sendingResult.rows[0]?.sending_enabled || (sendingResult.rows[0].minimum_release && controlResult.rows[0].minimum_release && sendingResult.rows[0].minimum_release !== controlResult.rows[0].minimum_release)) throw new LifecycleError(503, 'sending_disabled', 'Application email sending is disabled.'); const survey = await loadAuthorizedSurvey(client, user, surveyId, 'editor', 'UPDATE'); const data = await loadReadinessData(client, survey); const targetIds = data.recipients.map((row) => Number(row.respondent_id)).sort((a,b) => a-b); @@ -191,9 +224,11 @@ async function launchSurvey(pool, user, surveyId, { kind = 'initial', idempotenc return { ...result.rows[0], lifecycleStatus: survey.lifecycle_status, replayed: true }; } const heartbeatSeconds = Math.max(5, Number(config.EMAIL_WORKER_HEARTBEAT_MAX_AGE_SECONDS || 45)); - const heartbeat = await client.query(`SELECT 1 FROM email_worker_heartbeats WHERE environment=$1 AND enabled=true AND claiming=true AND heartbeat_at > now()-($2::text||' seconds')::interval AND ($3='' OR release_revision = $3) LIMIT 1`, [env, heartbeatSeconds, controlResult.rows[0].minimum_release || '']); + const requiredRelease = sendingResult.rows[0].minimum_release || controlResult.rows[0].minimum_release || ''; + const heartbeat = await client.query(`SELECT 1 FROM email_worker_heartbeats WHERE environment=$1 AND enabled=true AND claiming=true AND heartbeat_at > now()-($2::text||' seconds')::interval AND ($3='' OR release_revision = $3) LIMIT 1`, [env, heartbeatSeconds, requiredRelease]); if (!controlResult.rows[0].claiming_enabled || heartbeat.rowCount === 0) throw new LifecycleError(503, 'worker_unavailable', 'No fresh compatible email worker is available.'); const readiness = evaluateReadiness(survey, data, config); + await applySuppressionReadiness(client, readiness, data, config); if (!readiness.canLaunch) throw new LifecycleError(422, 'survey_not_ready', 'Survey is not ready to launch.', readiness); const launchResult = await client.query(`INSERT INTO survey_launches(survey_id,organization_id,kind,idempotency_key,request_fingerprint,requested_by_user_id) VALUES($1,$2,'initial',$3,$4,$5) RETURNING id,created_at`, [survey.id,survey.organization_id,effectiveKey,requestFingerprint,user.id]); @@ -206,15 +241,15 @@ async function launchSurvey(pool, user, surveyId, { kind = 'initial', idempotenc for (const recipient of data.recipients) { const language = normalizeLanguage(recipient.lang); const bodyText = readiness.templateMap.get(language); - const payload = buildInvitationPayload({ to:String(recipient.contact_info).trim().toLowerCase(),sender,subject,bodyText,surveyBaseUrl:config.SURVEY_URL,surveyName:survey.name,token:recipient.uuid,language }); const deliveryId = crypto.randomUUID(); + const payload = buildInvitationPayload({ to:String(recipient.contact_info).trim().toLowerCase(),sender,subject,bodyText,surveyBaseUrl:config.SURVEY_URL,surveyName:survey.name,token:recipient.uuid,language,deliveryId,environment:env }); await client.query(`INSERT INTO survey_email_deliveries(id,launch_id,survey_id,organization_id,respondent_id,to_address,recipient_display_name,language,sender,subject,template_hash,survey_base_url,renderer_version,render_inputs,expected_payload_hash,provider_idempotency_key) VALUES($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,$12,$13,$14::jsonb,$15,$16)`, [deliveryId,launch.id,survey.id,survey.organization_id,recipient.respondent_id,String(recipient.contact_info).trim().toLowerCase(),recipient.name,language,sender,subject,fingerprint(bodyText),config.SURVEY_URL,RENDERER_VERSION,JSON.stringify({surveyName:survey.name}),payloadHash(payload),`survey-delivery-${deliveryId}`]); } await client.query(`UPDATE survey SET lifecycle_status='active',started_at=now(),started_by_user_id=$1,closed_at=NULL,closed_by_user_id=NULL,lifecycle_version=lifecycle_version+1 WHERE id=$2`, [user.id,survey.id]); await strictAudit(client,{organizationId:survey.organization_id,actorUserId:user.id,surveyId:survey.id,eventType:'survey.launch_requested',metadata:{launchId:launch.id,targetCount:data.recipients.length}}); await strictAudit(client,{organizationId:survey.organization_id,actorUserId:user.id,surveyId:survey.id,eventType:'survey.lifecycle_changed',metadata:{from:'draft',to:'active',launchId:launch.id}}); await client.query('COMMIT'); - return { id:launch.id,survey_id:survey.id,kind,status:'queued',target_count:data.recipients.length,pending_count:data.recipients.length,leased_count:0,retry_wait_count:0,accepted_count:0,failed_count:0,uncertain_count:0,cancelled_count:0,created_at:launch.created_at,lifecycleStatus:'active',replayed:false }; + return { id:launch.id,survey_id:survey.id,kind,status:'queued',target_count:data.recipients.length,pending_count:data.recipients.length,leased_count:0,retry_wait_count:0,accepted_count:0,failed_count:0,uncertain_count:0,cancelled_count:0,provider_sent_count:0,provider_delivered_count:0,provider_delayed_count:0,provider_bounced_count:0,provider_complained_count:0,provider_suppressed_count:0,provider_failed_count:0,accepted_unverified_count:0,created_at:launch.created_at,lifecycleStatus:'active',replayed:false }; } catch (error) { await client.query('ROLLBACK').catch(()=>{}); if (error.code === '23505') throw new LifecycleError(409,'launch_conflict','An initial launch already exists.'); throw error; } finally { client.release(); } } @@ -228,7 +263,7 @@ async function listDeliveries(pool,user,surveyId,{status,cursor,limit=50}={}) { const client=await pool.connect(); try { await loadAuthorizedSurvey(client,user,surveyId,'analyst'); const values=[surveyId]; const clauses=['d.survey_id=$1']; if(status){values.push(status);clauses.push(`d.status=$${values.length}`);} if(cursor){values.push(cursor);clauses.push(`d.id < $${values.length}`);} values.push(Math.min(100,Math.max(1,Number(limit)||50))); - const result=await client.query(`SELECT d.id,d.launch_id,d.respondent_id,d.recipient_display_name,d.to_address,d.language,d.status,d.attempt_count,d.dispatch_accepted_at,d.dispatch_failed_at,d.last_error_code,d.last_error_message,d.created_at,d.updated_at,(SELECT max(started_at) FROM survey_email_attempts WHERE delivery_id=d.id) AS last_attempt_at FROM survey_email_deliveries d WHERE ${clauses.join(' AND ')} ORDER BY d.id DESC LIMIT $${values.length}`,values); + const result=await client.query(`SELECT d.id,d.launch_id,d.respondent_id,d.recipient_display_name,d.to_address,d.language,d.status,d.attempt_count,d.dispatch_accepted_at,d.dispatch_failed_at,d.provider_sent_at,d.provider_delivered_at,d.provider_delayed_at,d.provider_bounced_at,d.provider_complained_at,d.provider_suppressed_at,d.provider_failed_at,CASE WHEN d.provider_complained_at IS NOT NULL THEN 'complained' WHEN d.provider_bounced_at IS NOT NULL THEN 'bounced' WHEN d.provider_suppressed_at IS NOT NULL THEN 'suppressed' WHEN d.provider_failed_at IS NOT NULL THEN 'failed' WHEN d.provider_delivered_at IS NOT NULL THEN 'delivered' WHEN d.provider_delayed_at IS NOT NULL THEN 'delayed' WHEN d.provider_sent_at IS NOT NULL THEN 'sent' WHEN d.status='accepted' THEN 'accepted_unverified' ELSE NULL END AS provider_outcome,d.last_error_code,d.last_error_message,d.created_at,d.updated_at,(SELECT max(started_at) FROM survey_email_attempts WHERE delivery_id=d.id) AS last_attempt_at FROM survey_email_deliveries d WHERE ${clauses.join(' AND ')} ORDER BY d.id DESC LIMIT $${values.length}`,values); return {deliveries:result.rows,nextCursor:result.rows.length===values[values.length-1]?result.rows.at(-1).id:null}; } finally {client.release();} } diff --git a/api/package.json b/api/package.json index d852335..ab363b6 100644 --- a/api/package.json +++ b/api/package.json @@ -8,6 +8,8 @@ "start": "node server.js", "worker": "node email-worker.js", "worker:dev": "nodemon email-worker.js", + "webhook-worker": "node webhook-worker.js", + "webhook-worker:dev": "nodemon webhook-worker.js", "test": "node --test" }, "author": "", diff --git a/api/server.js b/api/server.js index 2f18a6c..ff56fb2 100644 --- a/api/server.js +++ b/api/server.js @@ -2,7 +2,6 @@ const express = require('express'); const fs = require('fs'); const path = require('path'); const cors = require('cors'); -const { Resend } = require('resend'); const { nanoid } = require('nanoid'); const { Pool } = require('pg'); const session = require('express-session'); @@ -16,8 +15,9 @@ const { Model, Serializer, Question } = require('survey-core'); dotenvFlow.config(); -const { ResendProvider, reserveProviderRate } = require('./email'); +const { ResendProvider, reserveProviderRateOnClient } = require('./email'); const lifecycle = require('./lifecycle'); +const { createResendWebhookHandler } = require('./webhooks'); const resendApiKey = process.env.RESEND_KEY || process.env.RESEND_API_KEY; // Keep server-side validation in step with the respondent's custom SurveyJS type. @@ -40,31 +40,69 @@ const pool = new Pool({ rejectUnauthorized: Boolean(process.env.DB_SSL_CA) } : undefined, }); -const resend = resendApiKey ? new Resend(resendApiKey) : null; const directSurveyProvider = resendApiKey ? new ResendProvider({ apiKey: resendApiKey }) : null; -async function reserveSynchronousEmailRate() { +async function reserveSynchronousEmailRate(client) { const environment = process.env.EMAIL_RATE_BUDGET_ENV || lifecycle.environmentName(process.env); const rate = Math.max(1, Number(process.env.EMAIL_RATE_PER_SECOND || 5)); - const deadline = Date.now() + Math.max(1000, Number(process.env.SYNC_EMAIL_RATE_WAIT_MS || 10000)); - while (Date.now() < deadline) { - if (await reserveProviderRate(pool, environment, rate)) return; - await new Promise((resolve) => setTimeout(resolve, 100 + Math.floor(Math.random() * 150))); - } + if (await reserveProviderRateOnClient(client, environment, rate)) return; const error = new Error('Email provider rate budget is busy; retry shortly.'); error.statusCode = 503; throw error; } +async function invokeSynchronousProvider(toAddress, factory) { + const environment = lifecycle.environmentName(process.env); + const hosted=['staging','prod'].includes(environment); + const scope = process.env.RESEND_PROVIDER_ACCOUNT_SCOPE || (hosted ? '' : 'local-resend-account'); + if (!scope) { const error=new Error('Provider account suppression scope is not configured.'); error.statusCode=503; throw error; } + const normalizedAddress = String(toAddress || '').trim().toLowerCase(); + const globalKey = `email-provider-boundary:${environment}`; + const addressKey = `email-suppression-boundary:${scope}:${normalizedAddress}`; + const client = await pool.connect(); + let globalLocked = false, addressLocked = false; + try { + await client.query('SELECT pg_advisory_lock(hashtextextended($1,0))', [globalKey]); + globalLocked = true; + await client.query('SELECT pg_advisory_lock(hashtextextended($1,0))', [addressKey]); + addressLocked = true; + const control = await client.query('SELECT sending_enabled,minimum_release FROM email_sending_control WHERE environment=$1', [environment]); + const row = control.rows[0]; + const release = process.env.RELEASE_REVISION || process.env.REVISION || 'local'; + if (!row?.sending_enabled || (row.minimum_release && row.minimum_release !== release)) { + const error = new Error('Application email sending is disabled.'); + error.statusCode = 503; + throw error; + } + const enforcement = await client.query('SELECT enforcement_enabled FROM email_suppression_control WHERE environment=$1', [environment]); + if (enforcement.rows[0]?.enforcement_enabled) { + const suppressed = await client.query(`SELECT 1 FROM email_suppressions WHERE provider_account_scope=$1 AND normalized_address=$2 AND (provider_active OR locally_overridden_at IS NULL) LIMIT 1`, [scope, normalizedAddress]); + if (suppressed.rowCount) { + const error = new Error('Recipient is suppressed.'); + error.statusCode = 409; + throw error; + } + } + await reserveSynchronousEmailRate(client); + return await factory(); + } finally { + if (addressLocked) await client.query('SELECT pg_advisory_unlock(hashtextextended($1,0))', [addressKey]).catch(() => {}); + if (globalLocked) await client.query('SELECT pg_advisory_unlock(hashtextextended($1,0))', [globalKey]).catch(() => {}); + client.release(); + } +} + // Account/demo mail remains bounded request work in Phase 1. Resend resolves // provider errors in {error}; only a response containing an id is accepted. async function sendAccountEmail({ to, subject, html, text }) { - if (!resend) return { sent: false, message: 'Email is not configured; deliver the returned link manually.' }; + if (!directSurveyProvider) return { sent: false, message: 'Email is not configured; deliver the returned link manually.' }; try { - await reserveSynchronousEmailRate(); - const result = await resend.emails.send({ from: 'CLA Survey ', to, subject, html, text }); - if (result?.error || !result?.data?.id) throw new Error(result?.error?.message || 'Provider response did not include a message ID'); - return { sent: true, providerMessageId: result.data.id }; + const result = await invokeSynchronousProvider(to, () => directSurveyProvider.send( + { from: 'CLA Survey ', to, subject, html, text }, + { idempotencyKey: `account-email/${crypto.randomUUID()}` } + )); + if (!result?.id) throw new Error('Provider response did not include a message ID'); + return { sent: true, providerMessageId: result.id }; } catch (error) { console.error('Account email provider request failed:', String(error.message || error).slice(0, 500)); return { sent: false, message: 'Email provider request failed; deliver the returned link manually.' }; @@ -77,11 +115,12 @@ function buildSurveyEmailHtml(text, link, language = 'en') { async function sendDemoMail(email, survey, text, demoToken, subject = 'CLA Network Survey', language = 'en') { if (!directSurveyProvider) throw new Error('Missing RESEND_KEY or RESEND_API_KEY environment variable'); - await reserveSynchronousEmailRate(); const link = `${process.env.SURVEY_URL}/?surveyName=${encodeURIComponent(survey.name)}&demoToken=${encodeURIComponent(demoToken)}`; const rendered = require('./email').renderInvitation({ bodyText: text, link, language }); - return directSurveyProvider.send({ from: 'CLA Survey ', to: email, subject: `[Demo] ${subject}`, ...rendered }, - { idempotencyKey: `survey-demo/${crypto.randomUUID()}` }); + return invokeSynchronousProvider(email, () => directSurveyProvider.send( + { from: 'CLA Survey ', to: email, subject: `[Demo] ${subject}`, ...rendered }, + { idempotencyKey: `survey-demo/${crypto.randomUUID()}` } + )); } // Function to execute a query @@ -220,6 +259,8 @@ function prepareSurveyForDemo(value) { return prepare(value); } +// Resend signatures cover the exact bytes; this route must precede JSON parsing. +app.post('/api/webhooks/resend', express.raw({ type: 'application/json', limit: '256kb' }), createResendWebhookHandler({ pool, env: process.env })); app.use(express.json()); app.use(cors({ @@ -2932,9 +2973,12 @@ app.get('/api/targets', requireAuth, async(req, res) => { if (!survey) { client.release(); return; } const query = `SELECT r.name, r.contact_info, r.respondent_id, r.can_respond, r.lang, r.response IS NULL AS response_status, - r.email_sent, d.status AS email_status, a.started_at AS last_email_attempt + r.email_sent, d.status AS email_status, d.provider_outcome, d.provider_outcome_at, a.started_at AS last_email_attempt FROM Respondent r - LEFT JOIN LATERAL (SELECT status,id FROM survey_email_deliveries WHERE respondent_id=r.respondent_id AND survey_id=r.survey_id ORDER BY created_at DESC LIMIT 1) d ON true + LEFT JOIN LATERAL (SELECT status,id, + CASE WHEN provider_complained_at IS NOT NULL THEN 'complained' WHEN provider_bounced_at IS NOT NULL THEN 'bounced' WHEN provider_suppressed_at IS NOT NULL THEN 'suppressed' WHEN provider_failed_at IS NOT NULL THEN 'failed' WHEN provider_delivered_at IS NOT NULL THEN 'delivered' WHEN provider_delayed_at IS NOT NULL THEN 'delayed' WHEN provider_sent_at IS NOT NULL THEN 'sent' WHEN status='accepted' THEN 'accepted_unverified' ELSE NULL END AS provider_outcome, + CASE WHEN provider_complained_at IS NOT NULL THEN provider_complained_at WHEN provider_bounced_at IS NOT NULL THEN provider_bounced_at WHEN provider_suppressed_at IS NOT NULL THEN provider_suppressed_at WHEN provider_failed_at IS NOT NULL THEN provider_failed_at WHEN provider_delivered_at IS NOT NULL THEN provider_delivered_at WHEN provider_delayed_at IS NOT NULL THEN provider_delayed_at WHEN provider_sent_at IS NOT NULL THEN provider_sent_at ELSE dispatch_accepted_at END AS provider_outcome_at + FROM survey_email_deliveries WHERE respondent_id=r.respondent_id AND survey_id=r.survey_id ORDER BY created_at DESC LIMIT 1) d ON true LEFT JOIN LATERAL (SELECT started_at FROM survey_email_attempts WHERE delivery_id=d.id ORDER BY attempt_number DESC LIMIT 1) a ON true WHERE ${legacySurveyPredicate('r')}`; client.query(query, [survey.id, survey.name]) @@ -2948,6 +2992,9 @@ app.get('/api/targets', requireAuth, async(req, res) => { status: row.response_status ? 'Incomplete' : 'Complete', responseStatus: row.response_status ? 'incomplete' : 'complete', emailStatus: row.email_status || (row.email_sent ? 'legacy_assumed_accepted' : 'not_queued'), + dispatchStatus: row.email_status || (row.email_sent ? 'legacy_assumed_accepted' : 'not_queued'), + providerOutcome: row.provider_outcome || (row.email_sent ? 'accepted_unverified' : null), + providerOutcomeAt: row.provider_outcome_at || null, lastEmailAttempt: row.last_email_attempt || null })); res.status(200).json(respondents); @@ -2982,7 +3029,15 @@ app.get('/api/surveys', requireAuth, async (req, res) => { 'acceptedCount', count(*) FILTER (WHERE d.status='accepted'), 'failedCount', count(*) FILTER (WHERE d.status='failed'), 'uncertainCount', count(*) FILTER (WHERE d.status='uncertain'), - 'cancelledCount', count(*) FILTER (WHERE d.status='cancelled') + 'cancelledCount', count(*) FILTER (WHERE d.status='cancelled'), + 'providerSentCount', count(*) FILTER (WHERE d.provider_sent_at IS NOT NULL), + 'providerDeliveredCount', count(*) FILTER (WHERE d.provider_delivered_at IS NOT NULL), + 'providerDelayedCount', count(*) FILTER (WHERE d.provider_delayed_at IS NOT NULL), + 'providerBouncedCount', count(*) FILTER (WHERE d.provider_bounced_at IS NOT NULL), + 'providerComplainedCount', count(*) FILTER (WHERE d.provider_complained_at IS NOT NULL), + 'providerSuppressedCount', count(*) FILTER (WHERE d.provider_suppressed_at IS NOT NULL), + 'providerFailedCount', count(*) FILTER (WHERE d.provider_failed_at IS NOT NULL), + 'acceptedUnverifiedCount', count(*) FILTER (WHERE d.status='accepted' AND d.provider_sent_at IS NULL AND d.provider_delivered_at IS NULL AND d.provider_delayed_at IS NULL AND d.provider_bounced_at IS NULL AND d.provider_complained_at IS NULL AND d.provider_suppressed_at IS NULL AND d.provider_failed_at IS NULL) ) FROM survey_launches l JOIN survey_email_deliveries d ON d.launch_id=l.id WHERE l.survey_id=s.id GROUP BY l.id,l.created_at ORDER BY l.created_at DESC LIMIT 1) AS latest_launch, COUNT(r.respondent_id) AS number_of_respondents, COALESCE(jsonb_array_length(s.questions->'elements'), 0) AS number_of_questions @@ -3006,7 +3061,15 @@ app.get('/api/surveys', requireAuth, async (req, res) => { 'acceptedCount', count(*) FILTER (WHERE d.status='accepted'), 'failedCount', count(*) FILTER (WHERE d.status='failed'), 'uncertainCount', count(*) FILTER (WHERE d.status='uncertain'), - 'cancelledCount', count(*) FILTER (WHERE d.status='cancelled') + 'cancelledCount', count(*) FILTER (WHERE d.status='cancelled'), + 'providerSentCount', count(*) FILTER (WHERE d.provider_sent_at IS NOT NULL), + 'providerDeliveredCount', count(*) FILTER (WHERE d.provider_delivered_at IS NOT NULL), + 'providerDelayedCount', count(*) FILTER (WHERE d.provider_delayed_at IS NOT NULL), + 'providerBouncedCount', count(*) FILTER (WHERE d.provider_bounced_at IS NOT NULL), + 'providerComplainedCount', count(*) FILTER (WHERE d.provider_complained_at IS NOT NULL), + 'providerSuppressedCount', count(*) FILTER (WHERE d.provider_suppressed_at IS NOT NULL), + 'providerFailedCount', count(*) FILTER (WHERE d.provider_failed_at IS NOT NULL), + 'acceptedUnverifiedCount', count(*) FILTER (WHERE d.status='accepted' AND d.provider_sent_at IS NULL AND d.provider_delivered_at IS NULL AND d.provider_delayed_at IS NULL AND d.provider_bounced_at IS NULL AND d.provider_complained_at IS NULL AND d.provider_suppressed_at IS NULL AND d.provider_failed_at IS NULL) ) FROM survey_launches l JOIN survey_email_deliveries d ON d.launch_id=l.id WHERE l.survey_id=s.id GROUP BY l.id,l.created_at ORDER BY l.created_at DESC LIMIT 1) AS latest_launch, COUNT(r.respondent_id) AS number_of_respondents, COALESCE(jsonb_array_length(s.questions->'elements'), 0) AS number_of_questions diff --git a/api/test/email-delivery.test.js b/api/test/email-delivery.test.js index f260062..f67d2c6 100644 --- a/api/test/email-delivery.test.js +++ b/api/test/email-delivery.test.js @@ -1,7 +1,7 @@ 'use strict'; const test = require('node:test'); const assert = require('node:assert/strict'); -const { renderInvitation, buildInvitationPayload, payloadHash, ResendProvider, classifyProviderError, ProviderError } = require('../email'); +const { renderInvitation, buildInvitationPayload, payloadHash, ResendProvider, classifyProviderError, ProviderError, reserveProviderRateOnClient } = require('../email'); const { evaluateReadiness, aggregateSelect, fingerprint, launchSurvey, transitionSurvey } = require('../lifecycle'); const { DeliveryWorker, isOutsideProviderIdempotencyWindow, canRetryAmbiguous } = require('../email-worker'); @@ -47,6 +47,15 @@ test('resolved provider error objects fail and retry classification is conservat assert.equal(classifyProviderError(new ProviderError('timeout',{uncertain:true})), 'ambiguous'); }); +test('retained provider-boundary clients reserve rate capacity without reconnecting', async () => { + const calls=[]; + const client={async query(sql,values=[]){calls.push({sql,values});if(/SELECT count/.test(sql))return {rows:[{count:0}]};return {rowCount:1,rows:[]};}}; + assert.equal(await reserveProviderRateOnClient(client,'test',1),true); + assert.match(calls[0].sql,/BEGIN/); + assert.ok(calls.some(({values})=>values.includes('email-rate-budget:test'))); + assert.match(calls.at(-1).sql,/COMMIT/); +}); + test('readiness validates the entire audience and exact normalized template coverage', () => { const survey = { lifecycle_status:'draft', archived_at:null, questions:{elements:[{name:'q1',type:'text'}]} }; const good = evaluateReadiness(survey, { recipients:[{respondent_id:1,contact_info:'A@example.com',uuid:'token',lang:'English'}], templates:[{lang:' english ',text:'Welcome'}] }, {SURVEY_URL:'https://survey.test',RESEND_API_KEY:'key'}); @@ -106,6 +115,7 @@ test('transactional launch locks control then survey, snapshots all work, activa const calls=[]; const client={release(){},async query(sql,values=[]){calls.push({sql,values}); if (/SELECT \* FROM email_worker_control/.test(sql)) return {rows:[{claiming_enabled:true,minimum_release:''}]}; + if (/SELECT \* FROM email_sending_control/.test(sql)) return {rows:[{sending_enabled:true,minimum_release:'release-pinned-by-sending'}]}; if (/SELECT s\.\*, om\.role/.test(sql)) return {rows:[{id:surveyId,name:'Survey A',organization_id:orgId,role:'editor',lifecycle_status:'draft',archived_at:null,questions:{elements:[{name:'q1',type:'text'}]}}]}; if (/SELECT respondent_id/.test(sql)) return {rows:[{respondent_id:7,name:'Person',contact_info:'person@example.com',uuid:'secret-token',lang:'English'}]}; if (/SELECT lang,text FROM email/.test(sql)) return {rows:[{lang:'English',text:'Please participate'}]}; @@ -116,7 +126,9 @@ test('transactional launch locks control then survey, snapshots all work, activa }}; const result=await launchSurvey({connect:async()=>client},{id:9,isPlatformAdmin:false},surveyId,{kind:'initial',idempotencyKey:'33333333-3333-4333-8333-333333333333'},{NODE_ENV:'test',SURVEY_URL:'https://survey.test',RESEND_API_KEY:'key',SURVEY_DELIVERY_V2_ENABLED:'true'}); assert.equal(result.status,'queued');assert.equal(result.target_count,1); - assert.match(calls[1].sql,/FOR SHARE/);assert.match(calls[2].sql,/FOR UPDATE OF s/); + assert.match(calls[1].sql,/FOR SHARE/);assert.match(calls[2].sql,/email_sending_control/);assert.match(calls[3].sql,/FOR UPDATE OF s/); + const heartbeatCall=calls.find(({sql})=>/SELECT 1 FROM email_worker_heartbeats/.test(sql)); + assert.equal(heartbeatCall.values[2],'release-pinned-by-sending'); assert.equal(calls.some(({sql})=>/INSERT INTO survey_email_deliveries/.test(sql)),true); assert.equal(calls.filter(({sql})=>/INSERT INTO audit_events/.test(sql)).length,2); assert.match(calls.at(-1).sql,/COMMIT/); @@ -134,6 +146,12 @@ test('close atomically cancels queued work, fences leased work, and writes stric assert.match(calls.at(-1).sql,/COMMIT/); }); +test('Phase 2 payload tags carry only non-secret environment and delivery correlation', () => { + const payload=buildInvitationPayload({to:'a@example.com',bodyText:'Welcome',surveyBaseUrl:'https://survey.test',surveyName:'S',token:'secret-token',language:'en',deliveryId:'11111111-1111-4111-8111-111111111111',environment:'staging'}); + assert.deepEqual(payload.tags,[{name:'app',value:'network_survey'},{name:'environment',value:'staging'},{name:'delivery_id',value:'11111111-1111-4111-8111-111111111111'}]); + assert.equal(JSON.stringify(payload.tags).includes('secret-token'),false); +}); + test('payload hash changes for token, template, or address without persisting rendered token separately', () => { const base={to:'a@example.com',bodyText:'Welcome',surveyBaseUrl:'https://survey.test',surveyName:'S',token:'one',language:'en'}; const first=payloadHash(buildInvitationPayload(base)); diff --git a/api/test/webhooks.test.js b/api/test/webhooks.test.js new file mode 100644 index 0000000..ee5a6f6 --- /dev/null +++ b/api/test/webhooks.test.js @@ -0,0 +1,225 @@ +'use strict'; + +const test = require('node:test'); +const assert = require('node:assert/strict'); +const { Webhook } = require('standardwebhooks'); +const { + MAX_WEBHOOK_BYTES, + ResendWebhookIngress, + WebhookError, + extractMetadata, + verifyResendWebhook, +} = require('../webhooks'); +const { + WebhookWorker, + canaryAddress, + canaryTags, + effectiveProviderOutcome, + normalizeAddress, + shouldApplySuppressionEvent, + validateKnownEvent, +} = require('../webhook-worker'); + +function signedFixture(event, { secret = `whsec_${Buffer.from('fixture-secret-32-bytes-long!!!!').toString('base64')}`, date = new Date(), id = 'evt_fixture_1' } = {}) { + const rawBody = Buffer.from(JSON.stringify(event)); + const webhook = new Webhook(secret); + return { + rawBody, + secret, + headers: { + 'svix-id': id, + 'svix-timestamp': String(Math.floor(date.getTime() / 1000)), + 'svix-signature': webhook.sign(id, date, rawBody), + }, + }; +} + +const deliveryEvent = (type = 'email.delivered') => ({ + type, + created_at: new Date().toISOString(), + data: { + email_id: 'email_123', + to: ['person@example.test'], + tags: { app: 'network_survey', environment: 'test', delivery_id: '11111111-1111-4111-8111-111111111111' }, + }, +}); + +test('Resend SDK helper verifies exact raw bytes and rejects mutation, stale time, and missing headers', () => { + const fixture = signedFixture(deliveryEvent()); + const result = verifyResendWebhook({ + rawBody: fixture.rawBody, headers: fixture.headers, primarySecret: fixture.secret, + }); + assert.equal(result.event.type, 'email.delivered'); + assert.equal(result.verifiedWithPrevious, false); + + const mutated = Buffer.from(fixture.rawBody); + mutated[mutated.length - 2] ^= 1; + assert.throws(() => verifyResendWebhook({ rawBody: mutated, headers: fixture.headers, primarySecret: fixture.secret }), + (error) => error instanceof WebhookError && error.code === 'invalid_signature'); + const stale = signedFixture(deliveryEvent(), { date: new Date(Date.now() - 6 * 60000) }); + assert.throws(() => verifyResendWebhook({ rawBody: stale.rawBody, headers: stale.headers, primarySecret: stale.secret }), + (error) => error.code === 'invalid_signature'); + assert.throws(() => verifyResendWebhook({ rawBody: fixture.rawBody, headers: {}, primarySecret: fixture.secret }), + (error) => error.status === 400); +}); + +test('verification supports bounded previous-secret overlap without exposing either secret', () => { + const fixture = signedFixture(deliveryEvent(), { secret: `whsec_${Buffer.from('previous-secret-32-bytes-long!!!').toString('base64')}` }); + const result = verifyResendWebhook({ + rawBody: fixture.rawBody, + headers: fixture.headers, + primarySecret: `whsec_${Buffer.from('different-secret-32-bytes-long!!').toString('base64')}`, + previousSecret: fixture.secret, + }); + assert.equal(result.verifiedWithPrevious, true); +}); + +test('metadata accepts additive fields while bounding correlation fields', () => { + const metadata = extractMetadata({ ...deliveryEvent(), future_addition: { value: true } }); + assert.equal(metadata.providerMessageId, 'email_123'); + assert.equal(metadata.appTag, 'network_survey'); + assert.throws(() => extractMetadata({ type: 'x'.repeat(129), created_at: new Date().toISOString() }), /event type/i); +}); + +test('ingress inserts verified JSONB bytes once with v1_7 names and acknowledges a duplicate', async () => { + const calls = []; + const client = { + release() {}, + async query(sql, values = []) { + calls.push({ sql, values }); + if (/INSERT INTO email_webhook_events/.test(sql)) return { rowCount: 0, rows: [] }; + return { rowCount: 0, rows: [] }; + }, + }; + const event = deliveryEvent(); + const rawBody = Buffer.from(JSON.stringify(event)); + const ingress = new ResendWebhookIngress({ + pool: { connect: async () => client }, + env: { + NODE_ENV: 'test', RESEND_PROVIDER_ACCOUNT_SCOPE: 'resend-team', + RESEND_WEBHOOK_INGEST_ENABLED: 'true', RESEND_WEBHOOK_SECRET: 'secret', + }, + resend: { webhooks: { verify: () => event } }, + clock: () => new Date('2026-08-05T00:00:00Z'), + }); + const result = await ingress.ingest(rawBody, { + 'svix-id': 'evt_1', 'svix-timestamp': '1785888000', 'svix-signature': 'v1,bounded', + }); + assert.equal(result.duplicate, true); + const insert = calls.find(({ sql }) => /INSERT INTO email_webhook_events/.test(sql)); + assert.match(insert.sql, /event_created_at/); + assert.match(insert.sql, /raw_payload/); + assert.match(insert.sql, /payload_size_bytes/); + assert.match(insert.sql, /\$10::jsonb/); + assert.equal(insert.values[10], rawBody.length); + assert.equal(calls.at(-1).sql, 'COMMIT'); +}); + +test('ingress is default-off, bounds raw requests, and validates rather than truncates scope', async () => { + const pool = { connect: async () => { throw new Error('must not connect'); } }; + const disabled = new ResendWebhookIngress({ pool, env: { NODE_ENV: 'test', RESEND_PROVIDER_ACCOUNT_SCOPE: 'scope' } }); + await assert.rejects(() => disabled.ingest(Buffer.from('{}'), {}), (error) => error.code === 'ingest_disabled'); + assert.throws(() => new ResendWebhookIngress({ pool, env: { NODE_ENV: 'test', RESEND_PROVIDER_ACCOUNT_SCOPE: 'x'.repeat(129) } }), /128/); + const enabled = new ResendWebhookIngress({ pool, env: { NODE_ENV: 'test', RESEND_PROVIDER_ACCOUNT_SCOPE: 'scope', RESEND_WEBHOOK_INGEST_ENABLED: 'true' } }); + await assert.rejects(() => enabled.ingest(Buffer.alloc(MAX_WEBHOOK_BYTES + 1), {}), (error) => error.status === 413); +}); + +test('provider suppression payloads accept documented nullable source IDs', () => { + assert.equal(validateKnownEvent('suppression.removed',{data:{email:'person@example.com',source_id:null}}),null); +}); + +test('provider outcome precedence is independent and suppression ordering is deterministic', () => { + assert.equal(effectiveProviderOutcome({ provider_delivered_at: 'x', provider_bounced_at: 'y' }), 'bounced'); + assert.equal(effectiveProviderOutcome({ provider_delivered_at: 'x', provider_complained_at: 'y' }), 'complained'); + const current = { state_occurrence_at: '2026-08-05T00:00:00Z', state_event_svix_id: 'evt_b', provider_active: false }; + assert.equal(shouldApplySuppressionEvent(current, { occurredAt: '2026-08-04T00:00:00Z', eventId: 'evt_z', active: true }), false); + assert.equal(shouldApplySuppressionEvent(current, { occurredAt: '2026-08-05T00:00:00Z', eventId: 'evt_a', active: true }), true, 'equal-time adverse add wins'); + assert.equal(shouldApplySuppressionEvent({ ...current, provider_active: true }, { occurredAt: current.state_occurrence_at, eventId: 'evt_z', active: false }), false, 'equal-time removal loses'); + assert.equal(normalizeAddress(' Person@Example.COM '), 'person@example.com'); +}); + +test('delivery projection preserves earliest facts and only event-specific evidence resolves dispatch', async () => { + const queries = []; + const client = { async query(sql, values = []) { queries.push({ sql, values }); return { rowCount: 1, rows: [] }; } }; + const worker = new WebhookWorker({ pool: {}, env: { NODE_ENV: 'test', RESEND_PROVIDER_ACCOUNT_SCOPE: 'scope' } }); + const delivery = { id: 'd', status: 'uncertain', respondent_id: 4, survey_id: 's' }; + const event = { event_type: 'email.delivered', event_created_at: '2026-08-05T00:00:00Z' }; + await worker.projectDelivery(client, event, deliveryEvent('email.delivered'), delivery); + assert.match(queries[0].sql, /LEAST\(provider_delivered_at,\$2\)/); + assert.ok(queries.some(({ sql }) => /SET status='accepted'/.test(sql))); + assert.ok(queries.some(({ sql }) => /UPDATE respondent SET email_sent=true/.test(sql))); + + queries.length = 0; + await worker.projectDelivery(client, { ...event, event_type: 'email.failed' }, deliveryEvent('email.failed'), delivery); + assert.match(queries[0].sql, /provider_failed_at/); + assert.equal(queries.some(({ sql }) => /SET status='accepted'/.test(sql)), false); +}); + +test('claim, replay, purge, and canary primitives use fenced v1_7 contracts', async () => { + const calls = []; + const client = { + release() {}, + async query(sql, values = []) { + calls.push({ sql, values }); + if (/worker_control/.test(sql)) return { rows: [{ claiming_enabled: true, processing_enabled: true, minimum_release: '' }] }; + if (/SELECT \* FROM email_webhook_events/.test(sql)) return { rows: [{ id: 7 }] }; + if (/UPDATE email_webhook_events SET status='leased'/.test(sql)) return { rows: [{ id: 7, lease_token: values[2] }] }; + if (/payload_expires_at/.test(sql) && /SELECT id,status/.test(sql)) return { rows: [{ id: 7, status: 'processed' }] }; + return { rowCount: 1, rows: [{ id: 7, replay_count: 1 }] }; + }, + }; + const pool = { connect: async () => client, query: client.query.bind(client) }; + const worker = new WebhookWorker({ pool, env: { NODE_ENV: 'test', RESEND_PROVIDER_ACCOUNT_SCOPE: 'scope' }, instanceId: 'worker' }); + const claimed = await worker.claim(); + assert.equal(claimed.id, 7); + assert.ok(calls.some(({ sql }) => /FOR UPDATE SKIP LOCKED/.test(sql) && /NULLS FIRST/.test(sql))); + assert.ok(calls.some(({ sql }) => /next_attempt_at=NULL/.test(sql) && /lease_token/.test(sql))); + await worker.replay(7, 'operator@example.test', 'corrected projector deployment'); + const replayCall=calls.find(({ sql }) => /last_replayed_by_actor/.test(sql)); + assert.match(replayCall.sql,/status IN \('processed','ignored','dead_letter','unmatched'\)/); + assert.match(replayCall.sql,/last_replay_reason/); + assert.doesNotMatch(replayCall.sql,/processing_attempt_count=0/); + await worker.purgeExpired(500); + assert.ok(calls.some(({ sql, values }) => /ANY\(\$1::bigint\[\]\)/.test(sql) && values[0][0] === 7)); + await worker.claimCanary(); + const canaryClaim=calls.find(({sql})=>/UPDATE email_webhook_canary_state SET status='leased'/.test(sql)); + assert.match(canaryClaim.sql,/CASE WHEN status='idle' THEN \$3 ELSE COALESCE\(canary_token,\$3\) END/); + assert.match(canaryClaim.sql,/provider_message_id=NULL/); + assert.match(canaryClaim.sql,/correlated_webhook_event_id=NULL/); + assert.match(canaryClaim.sql,/completed_at=NULL/); + assert.equal(canaryAddress('Staging'), 'delivered+webhook-canary-staging@resend.dev'); + assert.deepEqual(canaryTags('test', 'token').map(({ name }) => name), ['app', 'environment', 'canary']); +}); + +test('canary projection is monotonic and historical replay cannot refresh health', async()=>{ + const calls=[]; + const client={async query(sql,values=[]){calls.push({sql,values});if(/SELECT \* FROM email_webhook_canary_state/.test(sql))return {rows:[{environment:'test',canary_token:'11111111-1111-4111-8111-111111111111'}]};return {rowCount:1,rows:[]};}}; + const worker=new WebhookWorker({pool:{},env:{NODE_ENV:'test',RESEND_PROVIDER_ACCOUNT_SCOPE:'scope'}}); + const payload={data:{email_id:'provider-1',tags:[{name:'canary',value:'11111111-1111-4111-8111-111111111111'}]}}; + await worker.projectCanaryEvent(client,{event_type:'email.sent',event_created_at:'2026-08-05T00:00:00Z',replay_count:0,id:1},payload); + assert.match(calls.find(({sql})=>/SET status='awaiting_webhook'/.test(sql)).sql,/delivered_at IS NULL/); + calls.length=0; + await worker.projectCanaryEvent(client,{event_type:'email.delivered',event_created_at:'2026-08-05T00:01:00Z',replay_count:1,id:2},payload); + assert.equal(calls.some(({sql})=>/^UPDATE email_webhook_canary_state/.test(sql.trim())),false); +}); + +test('reason-keyed suppression writes receiving-environment audit and keeps removals fail closed', async () => { + const calls = []; + const client = { async query(sql, values = []) { + calls.push({ sql, values }); + if (/SELECT \* FROM email_suppressions/.test(sql)) return { rows: [] }; + return { rowCount: 1, rows: [] }; + } }; + const worker = new WebhookWorker({ pool: {}, env: { NODE_ENV: 'test', RESEND_PROVIDER_ACCOUNT_SCOPE: 'scope' } }); + await worker.upsertSuppression(client, { + address: ' PERSON@example.com ', reason: 'provider_suppression', active: false, + event: { id: 9, svix_id: 'evt_remove', event_created_at: '2026-08-05T00:00:00Z' }, + payload: { data: { email: 'person@example.com', source_id: 'sup_1' } }, + }); + const upsert = calls.find(({ sql }) => /INSERT INTO email_suppressions/.test(sql)); + assert.match(upsert.sql, /locally_overridden_at=NULL/); + const audit = calls.find(({ sql }) => /INSERT INTO email_suppression_audit/.test(sql)); + assert.match(audit.sql, /receiving_environment/); + assert.ok(audit.values.includes('test')); + assert.ok(audit.values.includes('provider_remove')); +}); diff --git a/api/webhook-worker.js b/api/webhook-worker.js new file mode 100644 index 0000000..d1cee54 --- /dev/null +++ b/api/webhook-worker.js @@ -0,0 +1,823 @@ +'use strict'; + +const crypto = require('crypto'); +const os = require('os'); +const fs = require('fs'); +const dotenvFlow = require('dotenv-flow'); +const { Pool } = require('pg'); +const { ResendProvider, reserveProviderRateOnClient, DEFAULT_SENDER } = require('./email'); +const { SELECTED_EVENT_TYPES } = require('./webhooks'); +const { emitMetrics } = require('./email-metrics'); + +dotenvFlow.config(); + +function createPool(env = process.env) { + return new Pool({ + user: env.DB_USER, + password: env.DB_PASSWORD, + host: env.DB_HOST, + port: env.DB_PORT, + database: env.DB_NAME || 'ONA', + ssl: env.DB_SSL === 'true' ? { + ca: env.DB_SSL_CA ? fs.readFileSync(env.DB_SSL_CA, 'utf8') : undefined, + rejectUnauthorized: Boolean(env.DB_SSL_CA), + } : undefined, + }); +} + +const EMAIL_EVENTS = new Set([ + 'email.sent', 'email.delivered', 'email.delivery_delayed', 'email.bounced', + 'email.complained', 'email.failed', 'email.suppressed', +]); +const ACCEPTANCE_EVENTS = new Set([ + 'email.sent', 'email.delivered', 'email.delivery_delayed', 'email.bounced', + 'email.complained', 'email.suppressed', +]); +const PROVIDER_COLUMNS = Object.freeze({ + 'email.sent': 'provider_sent_at', + 'email.delivered': 'provider_delivered_at', + 'email.delivery_delayed': 'provider_delayed_at', + 'email.bounced': 'provider_bounced_at', + 'email.complained': 'provider_complained_at', + 'email.failed': 'provider_failed_at', + 'email.suppressed': 'provider_suppressed_at', +}); +const SUPPRESSION_REASONS = Object.freeze({ + 'email.bounced': 'permanent_bounce', + 'email.complained': 'complaint', + 'email.suppressed': 'provider_suppression', + 'suppression.added': 'provider_suppression', + 'suppression.removed': 'provider_suppression', +}); +const TERMINAL_EVENT_STATES = new Set(['processed', 'ignored', 'dead_letter']); +const UUID_PATTERN = /^[0-9a-f]{8}-[0-9a-f]{4}-[1-5][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/i; + +const sleep = (milliseconds) => new Promise((resolve) => setTimeout(resolve, milliseconds)); +const bounded = (value, length = 500) => String(value || '').replace(/[\r\n\t]+/g, ' ').slice(0, length); +const normalizeAddress = (value) => typeof value === 'string' ? value.trim().toLowerCase() : ''; +const validAddress = (value) => { + const normalized = normalizeAddress(value); + return normalized.length <= 320 && normalized.indexOf('@') > 0; +}; + +function eventPayload(row) { + if (Buffer.isBuffer(row.raw_payload)) return JSON.parse(row.raw_payload.toString('utf8')); + if (row.raw_payload && typeof row.raw_payload === 'object') return row.raw_payload; + if (typeof row.raw_payload !== 'string') throw new Error('payload_unavailable'); + return JSON.parse(row.raw_payload); +} + +function eventTags(payload) { + const tags = payload?.data?.tags; + return tags && typeof tags === 'object' && !Array.isArray(tags) ? tags : {}; +} + +function effectiveProviderOutcome(delivery) { + if (delivery?.provider_complained_at) return 'complained'; + if (delivery?.provider_bounced_at) return 'bounced'; + if (delivery?.provider_suppressed_at) return 'suppressed'; + if (delivery?.provider_failed_at) return 'failed'; + if (delivery?.provider_delivered_at) return 'delivered'; + if (delivery?.provider_delayed_at) return 'delayed'; + if (delivery?.provider_sent_at || delivery?.dispatch_accepted_at) return 'accepted'; + return null; +} + +/** Equal-time adverse adds beat removals; IDs deterministically order like events. */ +function shouldApplySuppressionEvent(current, incoming) { + if (!current) return true; + const oldTime = new Date(current.state_occurrence_at ?? current.source_occurred_at).getTime(); + const newTime = new Date(incoming.occurredAt).getTime(); + if (newTime !== oldTime) return newTime > oldTime; + const oldActive = Boolean(current.provider_active); + if (Boolean(incoming.active) !== oldActive) return Boolean(incoming.active); + return String(incoming.eventId) > String(current.state_event_svix_id ?? ''); +} + +function validateKnownEvent(type, payload) { + const data = payload?.data; + if (!data || typeof data !== 'object' || Array.isArray(data)) return 'missing_event_data'; + if (EMAIL_EVENTS.has(type) && (typeof data.email_id !== 'string' || data.email_id.length < 1 || data.email_id.length > 256)) { + return 'invalid_provider_message_id'; + } + if ((type === 'suppression.added' || type === 'suppression.removed') && !validAddress(data.email)) { + return 'invalid_suppression_address'; + } + if (data.source_id != null && (typeof data.source_id !== 'string' || data.source_id.length > 255)) { + return 'invalid_suppression_source_id'; + } + return null; +} + +function canaryAddress(environment) { + const safe = String(environment || 'local').toLowerCase().replace(/[^a-z0-9_-]/g, '-').slice(0, 64); + return `delivered+webhook-canary-${safe}@resend.dev`; +} + +function canaryTags(environment, canaryId) { + return [ + { name: 'app', value: 'network_survey' }, + { name: 'environment', value: String(environment).slice(0, 256) }, + { name: 'canary', value: String(canaryId).slice(0, 256) }, + ]; +} + +class WebhookWorker { + constructor({ pool, provider, env = process.env, clock = () => new Date(), random = Math.random, sleepFn = sleep, instanceId } = {}) { + if (!pool) throw new Error('Webhook worker pool is required'); + this.pool = pool; + this.provider = provider; + this.env = env; + this.clock = clock; + this.random = random; + this.sleep = sleepFn; + this.environment = String(env.EMAIL_WORKER_ENV || env.APP_ENV || env.NODE_ENV || 'local'); + this.providerAccountScope = String(env.RESEND_PROVIDER_ACCOUNT_SCOPE || ''); + if (!this.environment || this.environment !== this.environment.trim() || this.environment.length > 64) { + throw new Error('Webhook worker environment must be 1-64 trimmed characters'); + } + if (!this.providerAccountScope || this.providerAccountScope !== this.providerAccountScope.trim() || this.providerAccountScope.length > 128) { + throw new Error('Webhook worker provider account scope must be 1-128 trimmed characters'); + } + this.release = env.RELEASE_REVISION || env.REVISION || 'local'; + this.instanceId = instanceId || `${env.DEPLOYMENT_ID || 'local'}/${os.hostname()}/${process.pid}/${crypto.randomUUID()}`; + this.leaseSeconds = Math.max(20, Number(env.RESEND_WEBHOOK_LEASE_SECONDS || 60)); + this.maxAttempts = Math.max(1, Number(env.RESEND_WEBHOOK_MAX_ATTEMPTS || 12)); + this.maxAgeHours = Math.max(1, Number(env.RESEND_WEBHOOK_MAX_AGE_HOURS || 72)); + this.unmatchedDays = Math.max(1, Number(env.RESEND_WEBHOOK_UNMATCHED_DAYS || 7)); + this.stopped = false; + this.processing = false; + this.lastError = null; + } + + backoff(attempt) { + const cap = Math.min(3600000, 1000 * (2 ** Math.min(Number(attempt) || 1, 12))); + return Math.floor(this.random() * cap); + } + + async heartbeat() { + await this.pool.query( + `INSERT INTO email_webhook_worker_heartbeats + (environment,worker_instance,release_revision,enabled,claiming,processing,heartbeat_at,last_error_code,last_error,started_at) + VALUES($1,$2,$3,true,$4,$4,now(),$5,$6,now()) + ON CONFLICT(environment,worker_instance) DO UPDATE SET + release_revision=excluded.release_revision,enabled=true,claiming=excluded.claiming, + processing=excluded.processing,heartbeat_at=now(),last_error_code=excluded.last_error_code, + last_error=excluded.last_error`, + [this.environment, this.instanceId, this.release, this.processing, + this.lastError ? 'worker_error' : null, this.lastError ? bounded(this.lastError) : null] + ); + } + + async control() { + const result = await this.pool.query( + 'SELECT processing_enabled,minimum_release FROM email_webhook_worker_control WHERE environment=$1', + [this.environment] + ); + const row = result.rows[0]; + return Boolean(row?.processing_enabled && (!row.minimum_release || row.minimum_release === this.release)); + } + + async claim() { + const client = await this.pool.connect(); + try { + await client.query('BEGIN'); + const control = (await client.query( + 'SELECT claiming_enabled,processing_enabled,minimum_release FROM email_webhook_worker_control WHERE environment=$1 FOR SHARE', + [this.environment] + )).rows[0]; + if (!control?.claiming_enabled || !control.processing_enabled || (control.minimum_release && control.minimum_release !== this.release)) { + await client.query('COMMIT'); + return null; + } + const selected = (await client.query( + `SELECT * FROM email_webhook_events + WHERE provider_account_scope=$1 AND + ((status IN ('pending','retry_wait','unmatched') AND next_attempt_at<=now()) + OR (status='leased' AND lease_expires_at<=now())) + ORDER BY next_attempt_at NULLS FIRST,received_at + FOR UPDATE SKIP LOCKED LIMIT 1`, + [this.providerAccountScope] + )).rows[0]; + if (!selected) { + await client.query('COMMIT'); + return null; + } + const token = crypto.randomUUID(); + const claimed = (await client.query( + `UPDATE email_webhook_events SET status='leased',lease_owner=$2,lease_token=$3, + lease_expires_at=now()+($4::text||' seconds')::interval,next_attempt_at=NULL, + processing_attempt_count=processing_attempt_count+1,updated_at=now() + WHERE id=$1 RETURNING *`, + [selected.id, this.instanceId, token, this.leaseSeconds] + )).rows[0]; + await client.query('COMMIT'); + return claimed; + } catch (error) { + await client.query('ROLLBACK').catch(() => {}); + throw error; + } finally { + client.release(); + } + } + + async correlate(client, event, payload) { + const providerId = payload?.data?.email_id || event.provider_message_id; + const tags = eventTags(payload); + const tag = tags.app === 'network_survey' && tags.environment === this.environment + ? (tags.delivery_id || event.delivery_tag) : null; + let byProvider = null; + let byTag = null; + if (providerId) { + byProvider = (await client.query( + 'SELECT * FROM survey_email_deliveries WHERE provider_message_id=$1 FOR UPDATE', [providerId] + )).rows[0] || null; + } + if (tag && UUID_PATTERN.test(tag)) { + byTag = (await client.query('SELECT * FROM survey_email_deliveries WHERE id=$1 FOR UPDATE', [tag])).rows[0] || null; + } + if (byProvider && byTag && byProvider.id !== byTag.id) throw new Error('correlation_conflict'); + const delivery = byProvider || byTag; + if (delivery?.provider_message_id && providerId && delivery.provider_message_id !== providerId) { + throw new Error('provider_message_id_conflict'); + } + return delivery; + } + + async finishEvent(client, event, status, { code = null, message = null, deliveryId = null } = {}) { + const result = await client.query( + `UPDATE email_webhook_events SET status=$3,correlated_delivery_id=COALESCE($4,correlated_delivery_id), + lease_owner=NULL,lease_token=NULL,lease_expires_at=NULL,next_attempt_at=NULL,unmatched_since_at=NULL, + last_error_code=$5,last_error_message=$6, + processed_at=CASE WHEN $3 IN ('processed','ignored') THEN now() ELSE processed_at END, + dead_lettered_at=CASE WHEN $3='dead_letter' THEN now() ELSE dead_lettered_at END,updated_at=now() + WHERE id=$1 AND status='leased' AND lease_token=$2`, + [event.id, event.lease_token, status, deliveryId, code, message ? bounded(message) : null] + ); + if (result.rowCount !== 1) throw new Error('stale_event_lease'); + } + + async upsertSuppression(client, { address, reason, active, event, payload }) { + const normalized = normalizeAddress(address); + if (!validAddress(address)) throw new Error('invalid_suppression_address'); + await client.query('SELECT pg_advisory_xact_lock(hashtextextended($1,0))', [`email-suppression-boundary:${this.providerAccountScope}:${normalized}`]); + const sourceId = payload?.data?.source_id || payload?.data?.id || null; + const current = (await client.query( + `SELECT * FROM email_suppressions + WHERE provider_account_scope=$1 AND normalized_address=$2 AND reason=$3 FOR UPDATE`, + [this.providerAccountScope, normalized, reason] + )).rows[0]; + const incoming = { occurredAt: event.event_created_at, eventId: event.svix_id, active }; + if (!shouldApplySuppressionEvent(current, incoming)) { + await client.query( + `INSERT INTO email_suppression_audit + (provider_account_scope,normalized_address,reason,receiving_environment,action,cause_version, + provider_active,occurrence_at,source_webhook_event_id,source_event_svix_id,provider_suppression_id,detail) + VALUES($1,$2,$3,$4,'stale_event_ignored',$5,$6,$7,$8,$9,$10,'{}'::jsonb)`, + [this.providerAccountScope, normalized, reason, this.environment, current.cause_version, + current.provider_active, event.event_created_at, event.id, event.svix_id, sourceId] + ); + return { normalized, applied: false }; + } + const version = Number(current?.cause_version || 0) + 1; + await client.query( + `INSERT INTO email_suppressions + (provider_account_scope,normalized_address,reason,receiving_environment,provider_active, + provider_suppression_id,source_webhook_event_id,source_occurred_at,state_occurrence_at, + state_event_svix_id,state_is_adverse,last_add_occurrence_at,last_add_event_svix_id, + last_remove_occurrence_at,last_remove_event_svix_id,cause_version,created_at,updated_at) + VALUES($1,$2,$3,$4,$5::boolean,$6,$7,$8::timestamptz,$8::timestamptz,$9::text,$5::boolean, + CASE WHEN $5::boolean THEN $8::timestamptz ELSE NULL END,CASE WHEN $5::boolean THEN $9::text ELSE NULL END, + CASE WHEN $5::boolean THEN NULL ELSE $8::timestamptz END,CASE WHEN $5::boolean THEN NULL ELSE $9::text END,$10,now(),now()) + ON CONFLICT(provider_account_scope,normalized_address,reason) DO UPDATE SET + receiving_environment=excluded.receiving_environment,provider_active=excluded.provider_active, + provider_suppression_id=excluded.provider_suppression_id, + source_webhook_event_id=excluded.source_webhook_event_id,source_occurred_at=excluded.source_occurred_at, + state_occurrence_at=excluded.state_occurrence_at,state_event_svix_id=excluded.state_event_svix_id, + state_is_adverse=excluded.state_is_adverse, + last_add_occurrence_at=COALESCE(excluded.last_add_occurrence_at,email_suppressions.last_add_occurrence_at), + last_add_event_svix_id=COALESCE(excluded.last_add_event_svix_id,email_suppressions.last_add_event_svix_id), + last_remove_occurrence_at=COALESCE(excluded.last_remove_occurrence_at,email_suppressions.last_remove_occurrence_at), + last_remove_event_svix_id=COALESCE(excluded.last_remove_event_svix_id,email_suppressions.last_remove_event_svix_id), + cause_version=excluded.cause_version,locally_overridden_at=NULL,override_cause_version=NULL, + locally_overridden_by_actor=NULL,updated_at=now()`, + [this.providerAccountScope, normalized, reason, this.environment, active, sourceId, + event.id, event.event_created_at, event.svix_id, version] + ); + await client.query( + `INSERT INTO email_suppression_audit + (provider_account_scope,normalized_address,reason,receiving_environment,action,cause_version, + provider_active,occurrence_at,source_webhook_event_id,source_event_svix_id,provider_suppression_id,detail) + VALUES($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,'{}'::jsonb)`, + [this.providerAccountScope, normalized, reason, this.environment, + active ? 'adverse_add' : 'provider_remove', version, active, + event.event_created_at, event.id, event.svix_id, sourceId] + ); + return { normalized, applied: true }; + } + + async projectDelivery(client, event, payload, delivery) { + const type = event.event_type; + const providerId = payload.data.email_id; + const occurrenceColumn = PROVIDER_COLUMNS[type]; + if (occurrenceColumn) { + await client.query( + `UPDATE survey_email_deliveries SET ${occurrenceColumn}=CASE + WHEN ${occurrenceColumn} IS NULL THEN $2 ELSE LEAST(${occurrenceColumn},$2) END, + provider_message_id=COALESCE(provider_message_id,$3),updated_at=now() WHERE id=$1`, + [delivery.id, event.event_created_at, providerId] + ); + } + + if (ACCEPTANCE_EVENTS.has(type)) { + if (delivery.status === 'leased') { + await client.query( + `UPDATE survey_email_attempts SET finished_at=COALESCE(finished_at,now()), + outcome=CASE WHEN provider_started_at IS NOT NULL THEN 'accepted' ELSE 'cancelled' END, + provider_message_id=COALESCE(provider_message_id,$2), + error_message=CASE WHEN provider_started_at IS NULL THEN 'provider_acceptance_reconciled' ELSE error_message END + WHERE delivery_id=$1 AND lease_token=$3 AND outcome='in_progress'`, + [delivery.id, providerId, delivery.lease_token] + ); + } + await client.query( + `UPDATE survey_email_deliveries SET status='accepted',provider_message_id=COALESCE(provider_message_id,$2), + dispatch_accepted_at=COALESCE(dispatch_accepted_at,now()),dispatch_failed_at=NULL,lease_owner=NULL,lease_token=NULL, + lease_expires_at=NULL,next_attempt_at=now(),updated_at=now() + WHERE id=$1 AND status IN ('pending','retry_wait','leased','uncertain','failed','cancelled','accepted')`, + [delivery.id, providerId] + ); + await client.query( + 'UPDATE respondent SET email_sent=true WHERE respondent_id=$1 AND survey_id=$2', + [delivery.respondent_id, delivery.survey_id] + ); + } + + const reason = SUPPRESSION_REASONS[type]; + let suppression = null; + if (reason) { + suppression = await this.upsertSuppression(client, { + address: delivery.to_address, reason, active: true, event, payload, + }); + } + return suppression?.normalized || null; + } + + async projectCanaryEvent(client, event, payload) { + const tags = eventTags(payload); + const token = UUID_PATTERN.test(tags.canary || '') ? tags.canary : null; + const providerId = payload?.data?.email_id; + if (!token && !providerId) return false; + const canary = (await client.query( + `SELECT * FROM email_webhook_canary_state WHERE environment=$1 AND + (($2::uuid IS NOT NULL AND canary_token=$2::uuid) OR ($3::text IS NOT NULL AND provider_message_id=$3)) + FOR UPDATE`, + [this.environment, token, providerId] + )).rows[0]; + if (!canary) return false; + if (Number(event.replay_count || 0) > 0) return true; + if (event.event_type === 'email.delivered') { + await client.query( + `UPDATE email_webhook_canary_state SET status='idle',provider_message_id=COALESCE(provider_message_id,$2), + sent_at=COALESCE(sent_at,$3),delivered_at=LEAST(COALESCE(delivered_at,$3),$3), + correlated_webhook_event_id=$4,completed_at=now(),last_success_at=now(), + next_run_at=now()+interval '6 hours',lease_owner=NULL,lease_token=NULL,lease_expires_at=NULL,updated_at=now() + WHERE environment=$1`, + [this.environment, providerId, event.event_created_at, event.id] + ); + } else if (event.event_type === 'email.sent') { + await client.query( + `UPDATE email_webhook_canary_state SET status='awaiting_webhook', + provider_message_id=COALESCE(provider_message_id,$2),sent_at=LEAST(COALESCE(sent_at,$3),$3), + lease_owner=NULL,lease_token=NULL,lease_expires_at=NULL,updated_at=now() + WHERE environment=$1 AND delivered_at IS NULL`, + [this.environment, providerId, event.event_created_at] + ); + } else if (['email.bounced','email.complained','email.failed','email.suppressed'].includes(event.event_type)) { + await client.query( + `UPDATE email_webhook_canary_state SET status='failed',last_error_code=$2, + last_error_message='Provider canary produced an adverse event',next_run_at=now()+interval '1 hour', + lease_owner=NULL,lease_token=NULL,lease_expires_at=NULL,updated_at=now() WHERE environment=$1`, + [this.environment, event.event_type.slice(0, 64)] + ); + } + return true; + } + + async process(event) { + let payload; + try { payload = eventPayload(event); } catch (error) { return this.deadLetter(event, 'malformed_payload', error.message); } + const client = await this.pool.connect(); + let reconcileAddress = null; + try { + await client.query('BEGIN'); + const fenced = (await client.query( + `SELECT * FROM email_webhook_events + WHERE id=$1 AND status='leased' AND lease_token=$2 FOR UPDATE`, + [event.id, event.lease_token] + )).rows[0]; + if (!fenced) { + await client.query('COMMIT'); + return { action: 'stale' }; + } + event = { ...event, ...fenced }; + const type = event.event_type; + if (!SELECTED_EVENT_TYPES.has(type)) { + await this.finishEvent(client, event, 'ignored', { code: 'unsupported_event_type' }); + await client.query('COMMIT'); + return { action: 'ignored' }; + } + const malformed = validateKnownEvent(type, payload); + if (malformed) { + await this.finishEvent(client, event, 'dead_letter', { code: malformed, message: malformed }); + await client.query('COMMIT'); + return { action: 'dead_letter' }; + } + + const tags = eventTags(payload); + if (EMAIL_EVENTS.has(type) && tags.app === 'network_survey' && tags.environment && tags.environment !== this.environment) { + await this.finishEvent(client, event, 'ignored', { code: 'foreign_environment' }); + await client.query('COMMIT'); + return { action: 'ignored' }; + } + + if (EMAIL_EVENTS.has(type) && await this.projectCanaryEvent(client, event, payload)) { + await this.finishEvent(client, event, 'processed'); + await client.query('COMMIT'); + } else if (EMAIL_EVENTS.has(type) && UUID_PATTERN.test(tags.canary || '')) { + await this.finishEvent(client,event,'ignored',{code:'stale_canary'}); + await client.query('COMMIT'); + return {action:'ignored'}; + } else if (type === 'suppression.added' || type === 'suppression.removed') { + const normalized = normalizeAddress(payload.data.email); + const reasons = [type === 'suppression.added' && payload.data.origin === 'bounce' ? 'permanent_bounce' + : type === 'suppression.added' && payload.data.origin === 'complaint' ? 'complaint' : 'provider_suppression']; + for (const reason of reasons) await this.upsertSuppression(client, { + address: normalized, + reason, + active: type === 'suppression.added', + event, + payload, + }); + reconcileAddress = normalized; + await this.finishEvent(client, event, 'processed'); + await client.query('COMMIT'); + } else { + const delivery = await this.correlate(client, event, payload); + if (!delivery) { + const adverseReason=SUPPRESSION_REASONS[type]; + const recipients=Array.isArray(payload.data.to)?payload.data.to:[]; + if(adverseReason&&recipients.length===1&&validAddress(recipients[0])){ + const suppression=await this.upsertSuppression(client,{address:recipients[0],reason:adverseReason,active:true,event,payload}); + await this.finishEvent(client,event,'processed'); + await client.query('COMMIT'); + await this.reconcileAddress(suppression.normalized); + return {action:'processed'}; + } + if (tags.app !== 'network_survey') { + await this.finishEvent(client,event,'ignored',{code:'non_survey_email'}); + await client.query('COMMIT'); + return {action:'ignored'}; + } + await client.query('ROLLBACK'); + return this.markUnmatched(event); + } + reconcileAddress = await this.projectDelivery(client, event, payload, delivery); + await this.finishEvent(client, event, 'processed', { deliveryId: delivery.id }); + await client.query('COMMIT'); + } + } catch (error) { + await client.query('ROLLBACK').catch(() => {}); + if (['correlation_conflict', 'provider_message_id_conflict', 'invalid_suppression_address'].includes(error.message)) { + return this.deadLetter(event, error.message, error.message); + } + if (error.message === 'stale_event_lease') return { action: 'stale' }; + return this.retry(event, 'projection_error', error.message); + } finally { + client.release(); + } + if (reconcileAddress) await this.reconcileAddress(reconcileAddress).catch((error) => { + this.lastError = `suppression_reconcile:${bounded(error.message, 200)}`; + emitMetrics({ environment:this.environment,release:this.release,metrics:{ SuppressionReconciliationFailureCount:1 } }); + }); + return { action: 'processed' }; + } + + async retry(event, code, message) { + const ageExceeded = new Date(event.received_at).getTime() <= this.clock().getTime() - this.maxAgeHours * 3600000; + const exhausted = Number(event.processing_attempt_count || 0) >= this.maxAttempts || ageExceeded; + const status = exhausted ? 'dead_letter' : 'retry_wait'; + const delay = this.backoff(event.processing_attempt_count); + const result = await this.pool.query( + `UPDATE email_webhook_events SET status=$3,next_attempt_at=CASE WHEN $3='retry_wait' + THEN now()+($4::text||' milliseconds')::interval ELSE NULL END,unmatched_since_at=NULL, + lease_owner=NULL,lease_token=NULL,lease_expires_at=NULL,last_error_code=$5,last_error_message=$6, + dead_lettered_at=CASE WHEN $3='dead_letter' THEN now() ELSE NULL END,processed_at=NULL,updated_at=now() + WHERE id=$1 AND status='leased' AND lease_token=$2`, + [event.id, event.lease_token, status, delay, code, bounded(message)] + ); + return { action: result.rowCount ? status : 'stale' }; + } + + async markUnmatched(event) { + const expired = new Date(event.received_at).getTime() <= this.clock().getTime() - this.unmatchedDays * 86400000; + const status = expired ? 'dead_letter' : 'unmatched'; + const result = await this.pool.query( + `UPDATE email_webhook_events SET status=$3,next_attempt_at=CASE WHEN $3='unmatched' + THEN now()+($4::text||' milliseconds')::interval ELSE NULL END, + unmatched_since_at=CASE WHEN $3='unmatched' THEN COALESCE(unmatched_since_at,now()) ELSE NULL END, + lease_owner=NULL,lease_token=NULL,lease_expires_at=NULL,last_error_code='delivery_unmatched', + last_error_message=NULL,processed_at=NULL, + dead_lettered_at=CASE WHEN $3='dead_letter' THEN now() ELSE NULL END,updated_at=now() + WHERE id=$1 AND status='leased' AND lease_token=$2`, + [event.id, event.lease_token, status, this.backoff(event.processing_attempt_count)] + ); + return { action: result.rowCount ? status : 'stale' }; + } + + async deadLetter(event, code, message) { + const result = await this.pool.query( + `UPDATE email_webhook_events SET status='dead_letter',lease_owner=NULL,lease_token=NULL, + lease_expires_at=NULL,next_attempt_at=NULL,unmatched_since_at=NULL,processed_at=NULL, + last_error_code=$3,last_error_message=$4,dead_lettered_at=now(),updated_at=now() + WHERE id=$1 AND status='leased' AND lease_token=$2`, + [event.id, event.lease_token, code, bounded(message)] + ); + return { action: result.rowCount ? 'dead_letter' : 'stale' }; + } + + async reconcileAddress(address) { + const normalized = normalizeAddress(address); + if (!normalized) return { cancelled: 0, fenced: 0 }; + const client = await this.pool.connect(); + try { + await client.query('BEGIN'); + // Preserve universal lock order: delivery rows are locked before the address boundary. + const candidates = await client.query( + `SELECT id,status FROM survey_email_deliveries WHERE lower(btrim(to_address))=$1 + AND status IN ('pending','retry_wait','leased') ORDER BY id FOR UPDATE`, [normalized] + ); + await client.query('SELECT pg_advisory_xact_lock(hashtextextended($1,0))', [`email-suppression-boundary:${this.providerAccountScope}:${normalized}`]); + const active = (await client.query( + `SELECT EXISTS(SELECT 1 FROM email_suppressions WHERE provider_account_scope=$1 + AND normalized_address=$2 AND (provider_active OR locally_overridden_at IS NULL + OR override_cause_version IS DISTINCT FROM cause_version)) AS suppressed`, + [this.providerAccountScope, normalized] + )).rows[0]?.suppressed; + if (!active) { + await client.query('COMMIT'); + return { cancelled: 0, fenced: 0 }; + } + const pendingIds = candidates.rows.filter((row) => row.status !== 'leased').map((row) => row.id); + const leasedIds = candidates.rows.filter((row) => row.status === 'leased').map((row) => row.id); + const cancelled = pendingIds.length ? await client.query( + `UPDATE survey_email_deliveries SET status='cancelled',provider_suppressed_at=COALESCE(provider_suppressed_at,now()), + last_error_code='suppressed',lease_owner=NULL,lease_token=NULL,lease_expires_at=NULL,updated_at=now() + WHERE id=ANY($1::uuid[]) AND status IN ('pending','retry_wait')`, [pendingIds] + ) : { rowCount: 0 }; + const fenced = leasedIds.length ? await client.query( + `UPDATE survey_email_deliveries SET cancellation_requested_at=COALESCE(cancellation_requested_at,now()), + provider_suppressed_at=COALESCE(provider_suppressed_at,now()),last_error_code='suppressed',updated_at=now() + WHERE id=ANY($1::uuid[]) AND status='leased'`, [leasedIds] + ) : { rowCount: 0 }; + await client.query('COMMIT'); + return { cancelled: cancelled.rowCount, fenced: fenced.rowCount }; + } catch (error) { + await client.query('ROLLBACK').catch(() => {}); + throw error; + } finally { client.release(); } + } + + async reconcileUnmatched({ providerMessageId = null, deliveryId = null } = {}) { + if (!providerMessageId && !deliveryId) throw new Error('A provider message ID or delivery ID is required'); + const result = await this.pool.query( + `UPDATE email_webhook_events SET next_attempt_at=now(),updated_at=now() + WHERE provider_account_scope=$1 AND status='unmatched' + AND (($2::text IS NOT NULL AND provider_message_id=$2) + OR ($3::text IS NOT NULL AND delivery_tag=$3))`, + [this.providerAccountScope, providerMessageId, deliveryId] + ); + return result.rowCount; + } + + async replay(eventId, actor, reason) { + if (!actor || !reason) throw new Error('Replay actor and reason are required'); + const result = await this.pool.query( + `UPDATE email_webhook_events SET status='pending',next_attempt_at=now(),unmatched_since_at=NULL, + lease_owner=NULL,lease_token=NULL,lease_expires_at=NULL,last_error_code=NULL,last_error_message=NULL, + processed_at=NULL,dead_lettered_at=NULL, + replay_count=replay_count+1,last_replayed_at=now(),last_replayed_by_actor=$2,last_replay_reason=$3,updated_at=now() + WHERE id=$1 AND status IN ('processed','ignored','dead_letter','unmatched') AND raw_payload IS NOT NULL + RETURNING id,replay_count`, [eventId, String(actor).slice(0, 200), String(reason).slice(0,500)] + ); + if (!result.rows[0]) throw new Error('Event is not replayable'); + return result.rows[0]; + } + + async purgeExpired(limit = 100) { + const batch = Math.max(1, Math.min(100, Number(limit) || 100)); + const client = await this.pool.connect(); + try { + await client.query('BEGIN'); + const rows = await client.query( + `SELECT id,status FROM email_webhook_events WHERE raw_payload IS NOT NULL AND payload_expires_at<=now() + ORDER BY payload_expires_at FOR UPDATE SKIP LOCKED LIMIT $1`, [batch] + ); + if (rows.rows.length) { + await client.query( + `UPDATE email_webhook_events SET raw_payload=NULL, + status=CASE WHEN status IN ('processed','ignored','dead_letter') THEN status ELSE 'dead_letter' END, + last_error_code=CASE WHEN status IN ('processed','ignored','dead_letter') THEN last_error_code ELSE 'payload_expired' END, + dead_lettered_at=CASE WHEN status IN ('processed','ignored','dead_letter') THEN dead_lettered_at ELSE now() END, + processed_at=CASE WHEN status IN ('processed','ignored') THEN processed_at ELSE NULL END, + next_attempt_at=NULL,unmatched_since_at=NULL,lease_owner=NULL,lease_token=NULL,lease_expires_at=NULL,updated_at=now() + WHERE id=ANY($1::bigint[])`, [rows.rows.map((row) => row.id)] + ); + } + await client.query('COMMIT'); + return rows.rows.length; + } catch (error) { + await client.query('ROLLBACK').catch(() => {}); + throw error; + } finally { client.release(); } + } + + async claimCanary() { + const token = crypto.randomUUID(); + await this.pool.query( + `INSERT INTO email_webhook_canary_state + (environment,provider_account_scope,status,scheduled_at,next_run_at) + VALUES($1,$2,'idle',now(),now()) ON CONFLICT(environment) DO NOTHING`, + [this.environment, this.providerAccountScope] + ); + await this.pool.query(`UPDATE email_webhook_canary_state SET status='retry_wait',next_run_at=now(),canary_token=NULL,last_error_code=COALESCE(last_error_code,'webhook_timeout'),updated_at=now() WHERE environment=$1 AND ((status='awaiting_webhook' AND sent_at{}); + this.lastError=bounded(error.message); + } finally { + if (boundaryLocked) await client.query(`SELECT pg_advisory_unlock(hashtextextended($1,0))`, [`email-provider-boundary:${this.environment}`]).catch(()=>{}); + client.release(); + } + } + + async emitOperationalMetrics() { + const row = (await this.pool.query(`SELECT + COALESCE((SELECT extract(epoch FROM now()-min(received_at)) FROM email_webhook_events WHERE receiving_environment=$1 AND status IN ('pending','retry_wait','leased')),0)::float8 AS pending_age, + COALESCE((SELECT extract(epoch FROM now()-min(unmatched_since_at)) FROM email_webhook_events WHERE receiving_environment=$1 AND status='unmatched'),0)::float8 AS unmatched_age, + (SELECT count(*)::int FROM email_webhook_events WHERE receiving_environment=$1 AND status='dead_letter') AS dead_count, + (SELECT count(*)::int FROM survey_email_deliveries WHERE status='uncertain') AS uncertain_count, + COALESCE((SELECT CASE WHEN last_success_at IS NULL THEN extract(epoch FROM now()-created_at) ELSE extract(epoch FROM now()-last_success_at) END FROM email_webhook_canary_state WHERE environment=$1),0)::float8 AS canary_age, + COALESCE((SELECT (NOT claiming_enabled AND reason ILIKE '%quota%')::int FROM email_worker_control WHERE environment=$1),0)::int AS quota_disabled`, [this.environment])).rows[0]; + emitMetrics({ namespace:this.env.WEBHOOK_METRIC_NAMESPACE,environment:this.environment,release:this.release,metrics:{ + WebhookWorkerHeartbeat:1, + OldestPendingEventAgeSeconds:Number(row.pending_age||0), + OldestUnmatchedEventAgeSeconds:Number(row.unmatched_age||0), + DeadLetterCount:Number(row.dead_count||0), + UncertainDeliveryCount:Number(row.uncertain_count||0), + QuotaClaimingDisabled:Number(row.quota_disabled||0), + WebhookCanaryAgeSeconds:Number(row.canary_age||0), + }}); + } + + async processOne() { + const event = await this.claim(); + if (event) { await this.process(event); return true; } + if (this.provider) { + const canary = await this.claimCanary(); + if (canary) { await this.processCanary(canary); return true; } + } + return false; + } + + async run() { + const heartbeatMs = Math.max(3000, Number(this.env.RESEND_WEBHOOK_HEARTBEAT_MS || 10000)); + const timer = setInterval(() => this.heartbeat().catch((error) => { this.lastError = error.message; }), heartbeatMs); + const maintenance = setInterval(() => { + this.emitOperationalMetrics().catch((error) => { this.lastError = error.message; }); + this.purgeExpired(100).catch((error) => { + this.lastError = error.message; + emitMetrics({ environment:this.environment,release:this.release,metrics:{ PayloadPurgeFailureCount:1 } }); + }); + }, 60000); + try { + await this.emitOperationalMetrics().catch((error) => { this.lastError = error.message; }); + while (!this.stopped) { + this.processing = await this.control(); + await this.heartbeat(); + if (!this.processing) { await this.sleep(1000); continue; } + if (!await this.processOne()) await this.sleep(Number(this.env.RESEND_WEBHOOK_IDLE_MS || 750)); + } + } finally { + clearInterval(timer); + clearInterval(maintenance); + this.processing = false; + await this.heartbeat().catch(() => {}); + } + } + + stop() { this.stopped = true; } +} + +async function main() { + const pool = createPool(); + const provider = process.env.RESEND_API_KEY || process.env.RESEND_KEY + ? new ResendProvider({ apiKey:process.env.RESEND_API_KEY || process.env.RESEND_KEY }) + : null; + const worker = new WebhookWorker({ pool,provider }); + const stop = () => worker.stop(); + process.on('SIGTERM',stop); + process.on('SIGINT',stop); + try { await worker.run(); } finally { await pool.end(); } +} +if (require.main === module) main().catch((error) => { console.error('Webhook worker failed:',bounded(error.message)); process.exit(1); }); + +module.exports = { + ACCEPTANCE_EVENTS, + createPool, + EMAIL_EVENTS, + PROVIDER_COLUMNS, + SUPPRESSION_REASONS, + TERMINAL_EVENT_STATES, + WebhookWorker, + canaryAddress, + canaryTags, + effectiveProviderOutcome, + eventPayload, + normalizeAddress, + shouldApplySuppressionEvent, + validateKnownEvent, + validAddress, +}; diff --git a/api/webhooks.js b/api/webhooks.js new file mode 100644 index 0000000..d073f30 --- /dev/null +++ b/api/webhooks.js @@ -0,0 +1,215 @@ +'use strict'; + +const { Resend } = require('resend'); +const { emitMetrics } = require('./email-metrics'); + +const MAX_WEBHOOK_BYTES = 256 * 1024; +const RAW_PAYLOAD_RETENTION_DAYS = 30; +const MAX_EVENT_TYPE_LENGTH = 128; +const MAX_PROVIDER_ID_LENGTH = 255; +const MAX_TAG_LENGTH = 256; +const SELECTED_EVENT_TYPES = new Set([ + 'email.sent', + 'email.delivered', + 'email.delivery_delayed', + 'email.bounced', + 'email.complained', + 'email.failed', + 'email.suppressed', + 'suppression.added', + 'suppression.removed', +]); + +class WebhookError extends Error { + constructor(code, status, message = code) { + super(message); + this.name = 'WebhookError'; + this.code = code; + this.status = status; + } +} + +function exactTrue(value) { + return value === true || value === 'true'; +} + +function headerValue(headers, name) { + const value = headers?.[name] ?? headers?.[name.toLowerCase()] ?? headers?.[name.toUpperCase()]; + return Array.isArray(value) ? value[0] : value; +} + +function signatureHeaders(headers) { + const values = { + id: headerValue(headers, 'svix-id'), + timestamp: headerValue(headers, 'svix-timestamp'), + signature: headerValue(headers, 'svix-signature'), + }; + for (const [name, value] of Object.entries(values)) { + if (typeof value !== 'string' || value.length === 0 || value.length > 2048) { + throw new WebhookError(`invalid_${name}_header`, 400, 'Invalid webhook signature headers'); + } + } + if (values.id.length > MAX_PROVIDER_ID_LENGTH) { + throw new WebhookError('invalid_event_id', 400, 'Invalid webhook event ID'); + } + return values; +} + +function rawPayload(rawBody, maxBytes = MAX_WEBHOOK_BYTES) { + if (!Buffer.isBuffer(rawBody)) throw new WebhookError('raw_body_required', 400, 'Raw webhook body required'); + if (rawBody.length > maxBytes) throw new WebhookError('payload_too_large', 413, 'Webhook payload too large'); + if (rawBody.length === 0) throw new WebhookError('empty_payload', 400, 'Webhook payload required'); + return rawBody; +} + +function boundedString(value, max) { + return typeof value === 'string' && value.length > 0 && value.length <= max ? value : null; +} + +function extractMetadata(event) { + if (!event || typeof event !== 'object' || Array.isArray(event)) { + throw new WebhookError('invalid_payload', 400, 'Invalid webhook payload'); + } + const eventType = boundedString(event.type, MAX_EVENT_TYPE_LENGTH); + if (!eventType) throw new WebhookError('invalid_event_type', 400, 'Invalid webhook event type'); + const eventTime = new Date(event.created_at); + if (typeof event.created_at !== 'string' || !Number.isFinite(eventTime.getTime())) { + throw new WebhookError('invalid_event_time', 400, 'Invalid webhook event time'); + } + const data = event.data && typeof event.data === 'object' && !Array.isArray(event.data) ? event.data : {}; + const providerMessageId = boundedString(data.email_id, MAX_PROVIDER_ID_LENGTH); + const tags = data.tags && typeof data.tags === 'object' && !Array.isArray(data.tags) ? data.tags : {}; + const deliveryTag = boundedString(tags.delivery_id, MAX_TAG_LENGTH); + return { + eventType, + eventTime: eventTime.toISOString(), + providerMessageId, + deliveryTag, + appTag: boundedString(tags.app, MAX_TAG_LENGTH), + environmentTag: boundedString(tags.environment, MAX_TAG_LENGTH), + }; +} + +/** Verify exact payload bytes through the public Resend SDK API. */ +function verifyResendWebhook({ rawBody, headers, primarySecret, previousSecret, resend } = {}) { + const body = rawPayload(rawBody); + const verifiedHeaders = signatureHeaders(headers); + if (!primarySecret) throw new WebhookError('webhook_secret_unavailable', 503, 'Webhook verifier unavailable'); + // The SDK constructor requires a non-empty API key even though webhook verification performs no API call. + const sdk = resend || new Resend('webhook-verification-only'); + const verify = (secret) => sdk.webhooks.verify({ + payload: body.toString('utf8'), + headers: verifiedHeaders, + webhookSecret: secret, + }); + try { + return { event: verify(primarySecret), verifiedWithPrevious: false }; + } catch (primaryError) { + if (previousSecret) { + try { + return { event: verify(previousSecret), verifiedWithPrevious: true }; + } catch { /* report one bounded result below */ } + } + throw new WebhookError('invalid_signature', 400, 'Invalid webhook signature'); + } +} + +class ResendWebhookIngress { + constructor({ pool, env = process.env, resend, clock = () => new Date(), maxBytes = MAX_WEBHOOK_BYTES } = {}) { + if (!pool) throw new Error('Webhook ingress pool is required'); + this.pool = pool; + this.env = env; + this.resend = resend; + this.clock = clock; + this.maxBytes = maxBytes; + this.retentionDays = Math.max(1, Math.min(365, Number(env.WEBHOOK_PAYLOAD_RETENTION_DAYS || RAW_PAYLOAD_RETENTION_DAYS))); + this.environment = String(env.EMAIL_WORKER_ENV || env.APP_ENV || env.NODE_ENV || 'local'); + this.providerAccountScope = String(env.RESEND_PROVIDER_ACCOUNT_SCOPE || ''); + if (!this.environment || this.environment !== this.environment.trim() || this.environment.length > 64) { + throw new Error('Receiving environment must be 1-64 trimmed characters'); + } + if (this.providerAccountScope && (this.providerAccountScope !== this.providerAccountScope.trim() || this.providerAccountScope.length > 128)) { + throw new Error('Provider account scope must be at most 128 trimmed characters'); + } + } + + enabled() { + return exactTrue(this.env.RESEND_WEBHOOK_INGEST_ENABLED); + } + + async ingest(rawBody, headers) { + if (!this.enabled()) throw new WebhookError('ingest_disabled', 503, 'Webhook ingestion disabled'); + rawPayload(rawBody, this.maxBytes); + if (!this.providerAccountScope) throw new WebhookError('provider_scope_unavailable', 503, 'Webhook ingress unavailable'); + + const verified = verifyResendWebhook({ + rawBody, + headers, + primarySecret: this.env.RESEND_WEBHOOK_SECRET, + previousSecret: this.env.RESEND_WEBHOOK_PREVIOUS_SECRET, + resend: this.resend, + }); + const metadata = extractMetadata(verified.event); + const svixId = signatureHeaders(headers).id; + const receivedAt = this.clock(); + const expiresAt = new Date(receivedAt.getTime() + this.retentionDays * 86400000); + + const client = await this.pool.connect(); + try { + await client.query('BEGIN'); + const inserted = await client.query( + `INSERT INTO email_webhook_events + (provider_account_scope,receiving_environment,svix_id,event_type,event_created_at, + provider_message_id,delivery_tag,app_tag,environment_tag,raw_payload,payload_size_bytes, + received_at,payload_expires_at,status,next_attempt_at) + VALUES($1,$2,$3,$4,$5,$6,$7,$8,$9,$10::jsonb,$11,$12,$13,'pending',$12) + ON CONFLICT(provider_account_scope,svix_id) DO NOTHING + RETURNING id`, + [this.providerAccountScope, this.environment, svixId, metadata.eventType, metadata.eventTime, + metadata.providerMessageId, metadata.deliveryTag, metadata.appTag, metadata.environmentTag, + rawBody.toString('utf8'), rawBody.length, receivedAt, expiresAt] + ); + await client.query('COMMIT'); + return { + accepted: true, + duplicate: inserted.rowCount === 0, + selected: SELECTED_EVENT_TYPES.has(metadata.eventType), + verifiedWithPrevious: verified.verifiedWithPrevious, + }; + } catch (error) { + await client.query('ROLLBACK').catch(() => {}); + throw new WebhookError('database_unavailable', 503, 'Webhook could not be stored'); + } finally { + client.release(); + } + } +} + +function createResendWebhookHandler(options) { + const ingress = options instanceof ResendWebhookIngress ? options : new ResendWebhookIngress(options); + return async function resendWebhookHandler(req, res) { + try { + const result = await ingress.ingest(req.body, req.headers); + res.status(200).json({ received: true, duplicate: result.duplicate }); + } catch (error) { + const status = error instanceof WebhookError ? error.status : 503; + if (error?.code === 'invalid_signature' || /^invalid_(id|timestamp|signature)_header$/.test(error?.code || '')) { + emitMetrics({ environment:ingress.environment, release:ingress.env.RELEASE_REVISION, metrics:{ InvalidSignatureCount:1 } }); + } + res.status(status).json({ error: status === 400 ? 'invalid_webhook' : status === 413 ? 'payload_too_large' : 'webhook_unavailable' }); + } + }; +} + +module.exports = { + MAX_WEBHOOK_BYTES, + RAW_PAYLOAD_RETENTION_DAYS, + SELECTED_EVENT_TYPES, + WebhookError, + ResendWebhookIngress, + createResendWebhookHandler, + exactTrue, + extractMetadata, + signatureHeaders, + verifyResendWebhook, +}; diff --git a/dashboard/src/components/RespondentTable.js b/dashboard/src/components/RespondentTable.js index ec78886..ab1d0f5 100644 --- a/dashboard/src/components/RespondentTable.js +++ b/dashboard/src/components/RespondentTable.js @@ -9,6 +9,7 @@ import SaveIcon from '@mui/icons-material/Save'; import DeleteIcon from '@mui/icons-material/Delete'; import TableMenuCell from './TableMenuCell'; import { LANGUAGES } from '@network-survey/frontend-shared'; +import { formatDateTime, providerOutcome, providerOutcomeLabel, providerOutcomeTimestamp } from './surveyLifecycle'; const TEMPLATE_DATA = [ 'First,Last,Email,Respondent,Location,Level,Gender,Race,Manager,VP,Business Group,Business Group - 1,Business Group - 2,Language', @@ -91,10 +92,23 @@ const RespondentTable = ({ rows, surveyName, onRespondentsUpdate, readOnly = fal valueGetter: (_, row) => row.responseStatus || row.response_status || row.status || 'Not started' }, { - field: 'emailStatus', - headerName: 'Email status', - width: 140, - valueGetter: (_, row) => row.emailStatus || row.email_status || 'Not queued' + field: 'dispatchStatus', + headerName: 'Dispatch status', + width: 150, + valueGetter: (_, row) => row.dispatchStatus || row.dispatch_status || row.emailStatus || row.email_status || 'Not queued' + }, + { + field: 'providerOutcome', + headerName: 'Provider outcome', + width: 175, + valueGetter: (_, row) => providerOutcomeLabel(providerOutcome(row)) + }, + { + field: 'providerOutcomeAt', + headerName: 'Provider outcome time', + width: 190, + valueGetter: (_, row) => providerOutcomeTimestamp(row), + valueFormatter: (value) => formatDateTime(value) }, { field: 'lastEmailAttempt', diff --git a/dashboard/src/components/RespondentTable.test.js b/dashboard/src/components/RespondentTable.test.js new file mode 100644 index 0000000..842f9cb --- /dev/null +++ b/dashboard/src/components/RespondentTable.test.js @@ -0,0 +1,45 @@ +import React from 'react'; +import { render, screen } from '@testing-library/react'; +import { expect, test, vi } from 'vitest'; +import RespondentTable from './RespondentTable'; +import { formatDateTime } from './surveyLifecycle'; + +vi.mock('@network-survey/frontend-shared', () => ({ LANGUAGES: [{ label: 'English' }] })); +vi.mock('../api/axios', () => ({ default: { get: vi.fn(), post: vi.fn(), delete: vi.fn() } })); + +vi.mock('@mui/x-data-grid', () => ({ + GridToolbar: () => null, + DataGrid: ({ rows, columns }) => ( +
+ {columns.filter((column) => ['dispatchStatus', 'providerOutcome', 'providerOutcomeAt'].includes(column.field)).map((column) => ( +
+ {rows.map((row) => { + const value = column.valueGetter ? column.valueGetter(undefined, row) : row[column.field]; + return {column.valueFormatter ? column.valueFormatter(value) : value}; + })} +
+ ))} +
+ ), +})); + +vi.mock('./TableUploadButton', () => ({ default: () => null })); +vi.mock('./AddRowButton', () => ({ default: () => null })); +vi.mock('./TableMenuCell', () => ({ default: () => null })); + +test('presents dispatch and provider outcomes as separate respondent fields', async () => { + const deliveredAt = '2026-01-02T03:04:05Z'; + render(); + + expect(await screen.findByRole('region', { name: 'Dispatch status' })).toHaveTextContent('accepted'); + const outcomes = screen.getByRole('region', { name: 'Provider outcome' }); + expect(outcomes).toHaveTextContent('Delivered'); + expect(outcomes).toHaveTextContent('Accepted / unverified'); + expect(screen.getByRole('region', { name: 'Provider outcome time' })).toHaveTextContent(formatDateTime(deliveredAt)); +}); diff --git a/dashboard/src/components/SurveyLifecyclePanel.js b/dashboard/src/components/SurveyLifecyclePanel.js index 9a2480b..18875e5 100644 --- a/dashboard/src/components/SurveyLifecyclePanel.js +++ b/dashboard/src/components/SurveyLifecyclePanel.js @@ -7,8 +7,8 @@ import RefreshIcon from '@mui/icons-material/Refresh'; import LockIcon from '@mui/icons-material/Lock'; import api from '../api/axios'; import { - formatDateTime, isLaunchRunning, launchCounts, launchStatus, - lifecycleLabel, lifecycleStatus, surveyId, + formatDateTime, isLaunchRunning, launchCounts, launchStatus, providerCounts, + lifecycleLabel, lifecycleStatus, shouldPollLaunch, surveyId, } from './surveyLifecycle'; export const LifecycleChip = ({ status, size = 'small' }) => { @@ -67,7 +67,9 @@ const SurveyLifecyclePanel = ({ survey, onSurveyRefresh, refreshToken = 0 }) => timer = setTimeout(poll, Math.min(30000, 3000 * (2 ** Math.min(failures - 1, 4)))); } else { failures = 0; - timer = setTimeout(poll, next.some(isLaunchRunning) ? 3000 : 30000); + if (next.some((launch) => shouldPollLaunch(launch))) { + timer = setTimeout(poll, next.some(isLaunchRunning) ? 3000 : 30000); + } } }; poll(); @@ -82,12 +84,15 @@ const SurveyLifecyclePanel = ({ survey, onSurveyRefresh, refreshToken = 0 }) => const visibleLaunches = launchSurveyId === id ? launches : []; const latest = visibleLaunches[0] || survey.latestLaunch || survey.latest_launch; const counts = launchCounts(latest); + const outcomes = providerCounts(latest); const terminal = counts.accepted + counts.failed + counts.uncertain + counts.cancelled; const progress = counts.target ? Math.min(100, (terminal / counts.target) * 100) : 0; const hasIssue = counts.failed + counts.uncertain + counts.cancelled > 0; + const hasProviderIssue = outcomes.bounced + outcomes.complained + outcomes.suppressed + outcomes.providerFailed > 0; const summary = latest ? `${statusText(launchStatus(latest))}: ${terminal} of ${counts.target} finished; ${counts.pending} pending, ${counts.leased} sending, ${counts.retryWait} waiting to retry, ${counts.accepted} accepted, ${counts.failed} failed, ${counts.uncertain} uncertain, ${counts.cancelled} cancelled.` : 'No invitation launch history.'; + const providerSummary = `${outcomes.delivered} delivered, ${outcomes.delayed} delayed, ${outcomes.bounced} bounced, ${outcomes.complained} complained, ${outcomes.suppressed} suppressed, ${outcomes.providerFailed} provider failed, ${outcomes.acceptedUnverified} accepted / unverified.`; const manualRefresh = () => setManualRefreshToken((value) => value + 1); @@ -113,11 +118,18 @@ const SurveyLifecyclePanel = ({ survey, onSurveyRefresh, refreshToken = 0 }) => Latest invitation dispatch {summary} - Email status reflects dispatch acceptance, not mailbox delivery. + + Provider outcomes + {providerSummary} + + Dispatch acceptance is separate from mailbox delivery. Accepted / unverified means no mailbox outcome has been recorded yet. Terminal launches refresh automatically for seven days; older history uses manual refresh. } {hasIssue && Some invitations were not accepted. Failed, uncertain, and cancelled messages remain visible in history. } + {hasProviderIssue && + Some accepted invitations have an adverse provider outcome. Provider outcomes do not change dispatch progress. + } {lifecycleStatus(survey) !== 'draft' && } sx={{ mt: 2 }}> Questions, respondents, notification templates, and survey design are read-only while this survey is {lifecycleStatus(survey)}. } @@ -125,9 +137,9 @@ const SurveyLifecyclePanel = ({ survey, onSurveyRefresh, refreshToken = 0 }) => {visibleLaunches.length > 0 && Launch history - CreatedStatusIn progressAccepted / targetFailedUncertainCancelled - {visibleLaunches.map((launch, index) => { const rowCounts = launchCounts(launch); return ( - {formatDateTime(launch.createdAt || launch.created_at)}{statusText(launchStatus(launch))}{rowCounts.pending} pending · {rowCounts.leased} sending · {rowCounts.retryWait} retrying{rowCounts.accepted} / {rowCounts.target}{rowCounts.failed}{rowCounts.uncertain}{rowCounts.cancelled} + CreatedDispatch statusIn progressAccepted / targetFailedUncertainCancelledProvider outcomes + {visibleLaunches.map((launch, index) => { const rowCounts = launchCounts(launch); const rowOutcomes = providerCounts(launch); return ( + {formatDateTime(launch.createdAt || launch.created_at)}{statusText(launchStatus(launch))}{rowCounts.pending} pending · {rowCounts.leased} sending · {rowCounts.retryWait} retrying{rowCounts.accepted} / {rowCounts.target}{rowCounts.failed}{rowCounts.uncertain}{rowCounts.cancelled}{rowOutcomes.delivered} delivered · {rowOutcomes.delayed} delayed · {rowOutcomes.bounced} bounced · {rowOutcomes.complained} complained · {rowOutcomes.suppressed} suppressed · {rowOutcomes.providerFailed} provider failed · {rowOutcomes.acceptedUnverified} accepted / unverified ); })}
} diff --git a/dashboard/src/components/SurveyLifecyclePanel.test.js b/dashboard/src/components/SurveyLifecyclePanel.test.js index f54571f..d94448c 100644 --- a/dashboard/src/components/SurveyLifecyclePanel.test.js +++ b/dashboard/src/components/SurveyLifecyclePanel.test.js @@ -3,7 +3,10 @@ import { act, render, screen } from '@testing-library/react'; import { afterEach, beforeEach, expect, test, vi } from 'vitest'; import api from '../api/axios'; import SurveyLifecyclePanel from './SurveyLifecyclePanel'; -import { launchCounts, launchStatus } from './surveyLifecycle'; +import { + launchCounts, launchStatus, providerCounts, providerOutcome, providerOutcomeLabel, + providerOutcomeTimestamp, providerTimestamps, shouldPollLaunch, +} from './surveyLifecycle'; vi.mock('../api/axios', () => ({ default: { get: vi.fn() } })); @@ -24,6 +27,56 @@ test('normalizes the real snake_case launch aggregate contract', () => { expect(launchStatus({ targetCount: 42, acceptedCount: 42 })).toBe('completed'); }); +test('normalizes additive provider counts, timestamps, and old accepted deliveries', () => { + expect(providerCounts({ + accepted_count: '5', + provider_outcome_counts: { + sent_count: '4', delivered_count: '2', delayed_count: 1, bounced_count: '1', + complained_count: 1, suppressed_count: 0, provider_failed_count: '1', accepted_unverified_count: '1', + }, + })).toEqual({ + sent: 4, delivered: 2, delayed: 1, bounced: 1, complained: 1, + suppressed: 0, providerFailed: 1, acceptedUnverified: 1, + }); + expect(providerCounts({ acceptedCount: 3 }).acceptedUnverified).toBe(3); + + const delivery = { + dispatch_status: 'accepted', + provider_timestamps: { delivered_at: '2026-01-02T03:04:05Z' }, + provider_complained_at: '2026-01-03T03:04:05Z', + }; + expect(providerTimestamps(delivery)).toMatchObject({ + delivered: '2026-01-02T03:04:05Z', complained: '2026-01-03T03:04:05Z', + }); + expect(providerOutcome(delivery)).toBe('complained'); + expect(providerOutcomeTimestamp(delivery)).toBe('2026-01-03T03:04:05Z'); + expect(providerOutcome({ emailStatus: 'legacy_assumed_accepted' })).toBe('accepted_unverified'); + expect(providerOutcomeLabel('accepted_unverified')).toBe('Accepted / unverified'); +}); + +test('keeps terminal provider reconciliation polling bounded', () => { + const now = Date.parse('2026-02-01T00:00:00Z'); + expect(shouldPollLaunch({ status: 'processing', created_at: '2020-01-01T00:00:00Z' }, now)).toBe(true); + expect(shouldPollLaunch({ status: 'completed', created_at: '2026-01-31T00:00:00Z' }, now)).toBe(true); + expect(shouldPollLaunch({ status: 'completed', created_at: '2026-01-01T00:00:00Z' }, now)).toBe(false); +}); + +test('renders provider outcomes separately without changing dispatch arithmetic', async () => { + api.get.mockResolvedValue({ data: { launches: [{ + id: 'launch-1', status: 'completed', created_at: new Date().toISOString(), + target_count: 3, accepted_count: 3, + provider_outcome_counts: { delivered_count: 2, complained_count: 1, accepted_unverified_count: 1 }, + }] } }); + render(); + + expect(await screen.findByRole('status')).toHaveTextContent('3 of 3 finished'); + const providerSummary = screen.getByRole('region', { name: 'Provider outcome summary' }); + expect(providerSummary).toHaveTextContent('2 delivered'); + expect(providerSummary).toHaveTextContent('1 complained'); + expect(providerSummary).toHaveTextContent('1 accepted / unverified'); + expect(providerSummary).not.toHaveTextContent('3 delivered'); +}); + test('does not render prior survey history while the next survey is loading', async () => { const second = deferred(); api.get @@ -39,6 +92,20 @@ test('does not render prior survey history while the next survey is loading', as }); }); +test('stops automatic terminal polling after the reconciliation horizon', async () => { + vi.useFakeTimers(); + vi.setSystemTime(new Date('2026-02-01T00:00:00Z')); + api.get.mockResolvedValue({ data: { launches: [{ + id: 'old', status: 'completed', created_at: '2026-01-01T00:00:00Z', + target_count: 1, accepted_count: 1, + }] } }); + render(); + await act(async () => { await Promise.resolve(); await Promise.resolve(); }); + expect(api.get).toHaveBeenCalledTimes(1); + await act(async () => { await vi.advanceTimersByTimeAsync(60000); }); + expect(api.get).toHaveBeenCalledTimes(1); +}); + test('retries launch history with backoff after a transient failure', async () => { vi.useFakeTimers(); api.get.mockRejectedValueOnce({ response: { status: 503 } }).mockResolvedValueOnce({ data: { launches: [] } }); diff --git a/dashboard/src/components/SurveyTable.js b/dashboard/src/components/SurveyTable.js index dc4598c..a1abd60 100644 --- a/dashboard/src/components/SurveyTable.js +++ b/dashboard/src/components/SurveyTable.js @@ -2,7 +2,7 @@ import React, { useMemo } from 'react'; import { DataGrid, GridToolbar } from '@mui/x-data-grid'; import MenuCell from './SurveyTableMenuCell'; import { LifecycleChip } from './SurveyLifecyclePanel'; -import { launchCounts, lifecycleStatus } from './surveyLifecycle'; +import { launchCounts, lifecycleStatus, providerCounts } from './surveyLifecycle'; const SurveyTable = ({ rows, @@ -41,6 +41,18 @@ const SurveyTable = ({ return `${counts.accepted} accepted / ${counts.target}${counts.failed ? ` · ${counts.failed} failed` : ''}${counts.uncertain ? ` · ${counts.uncertain} uncertain` : ''}`; }, }, + { + field: 'providerSummary', + headerName: 'Provider outcomes', + width: 310, + sortable: false, + renderCell: ({ row }) => { + const latest = row.latestLaunch || row.latest_launch; + if (!latest) return 'No provider outcomes'; + const counts = providerCounts(latest); + return `${counts.delivered} delivered · ${counts.delayed} delayed · ${counts.bounced} bounced · ${counts.complained} complained · ${counts.suppressed} suppressed · ${counts.providerFailed} provider failed · ${counts.acceptedUnverified} accepted / unverified`; + }, + }, { field: 'date', headerName: 'Creation Date', width: 170 }, { field: 'actions', diff --git a/dashboard/src/components/SurveyTable.test.js b/dashboard/src/components/SurveyTable.test.js new file mode 100644 index 0000000..3aca2ff --- /dev/null +++ b/dashboard/src/components/SurveyTable.test.js @@ -0,0 +1,36 @@ +import React from 'react'; +import { render, screen } from '@testing-library/react'; +import { expect, test, vi } from 'vitest'; +import SurveyTable from './SurveyTable'; + +vi.mock('../api/axios', () => ({ default: { get: vi.fn() } })); + +vi.mock('@mui/x-data-grid', () => ({ + GridToolbar: () => null, + DataGrid: ({ rows, columns }) => ( +
+ {columns.filter((column) => ['invitationSummary', 'providerSummary'].includes(column.field)).map((column) => ( +
+ {rows.map((row) => {column.renderCell({ row })})} +
+ ))} +
+ ), +})); + +vi.mock('./SurveyTableMenuCell', () => ({ default: () => null })); + +test('keeps survey dispatch acceptance separate from provider outcomes', () => { + render( {}} />); + + expect(screen.getByRole('region', { name: 'Invitation dispatch' })).toHaveTextContent('4 accepted / 4'); + const provider = screen.getByRole('region', { name: 'Provider outcomes' }); + expect(provider).toHaveTextContent('2 delivered'); + expect(provider).toHaveTextContent('1 bounced'); + expect(provider).toHaveTextContent('2 accepted / unverified'); +}); diff --git a/dashboard/src/components/surveyLifecycle.js b/dashboard/src/components/surveyLifecycle.js index a63faa6..87537ce 100644 --- a/dashboard/src/components/surveyLifecycle.js +++ b/dashboard/src/components/surveyLifecycle.js @@ -34,6 +34,109 @@ export const launchCounts = (launch) => { }; }; +const finiteCount = (value) => { + const parsed = Number(value ?? 0); + return Number.isFinite(parsed) ? parsed : 0; +}; + +const firstValue = (source, keys) => keys.map((key) => source?.[key]).find((value) => value !== undefined && value !== null); + +export const providerCounts = (launch) => { + const nested = launch?.providerOutcomeCounts || launch?.provider_outcome_counts + || launch?.providerCounts || launch?.provider_counts || launch?.providerOutcomes || launch?.provider_outcomes; + const sources = [nested, launch?.counts, launch?.dispatchCounts, launch?.dispatch_counts, launch].filter(Boolean); + const aliases = { + sent: ['sent', 'sentCount', 'sent_count', 'providerSent', 'provider_sent', 'providerSentCount', 'provider_sent_count'], + delivered: ['delivered', 'deliveredCount', 'delivered_count', 'providerDelivered', 'provider_delivered', 'providerDeliveredCount', 'provider_delivered_count'], + delayed: ['delayed', 'delayedCount', 'delayed_count', 'providerDelayed', 'provider_delayed', 'providerDelayedCount', 'provider_delayed_count'], + bounced: ['bounced', 'bouncedCount', 'bounced_count', 'providerBounced', 'provider_bounced', 'providerBouncedCount', 'provider_bounced_count'], + complained: ['complained', 'complainedCount', 'complained_count', 'providerComplained', 'provider_complained', 'providerComplainedCount', 'provider_complained_count'], + suppressed: ['suppressed', 'suppressedCount', 'suppressed_count', 'providerSuppressed', 'provider_suppressed', 'providerSuppressedCount', 'provider_suppressed_count'], + providerFailed: ['providerFailed', 'provider_failed', 'providerFailedCount', 'provider_failed_count', 'failedProvider', 'failed_provider'], + acceptedUnverified: ['acceptedUnverified', 'accepted_unverified', 'acceptedUnverifiedCount', 'accepted_unverified_count', 'unverifiedAccepted', 'unverified_accepted', 'unverifiedAcceptedCount', 'unverified_accepted_count'], + }; + const raw = (keys) => sources.map((source) => firstValue(source, keys)).find((value) => value !== undefined && value !== null); + const hasProviderData = Boolean(nested) || Object.values(aliases).flat().some((key) => sources.some((source) => source?.[key] !== undefined && source?.[key] !== null)); + const acceptedUnverified = raw(aliases.acceptedUnverified); + const nestedFailed = nested && firstValue(nested, ['failed', 'failedCount', 'failed_count']); + + return { + sent: finiteCount(raw(aliases.sent)), + delivered: finiteCount(raw(aliases.delivered)), + delayed: finiteCount(raw(aliases.delayed)), + bounced: finiteCount(raw(aliases.bounced)), + complained: finiteCount(raw(aliases.complained)), + suppressed: finiteCount(raw(aliases.suppressed)), + providerFailed: finiteCount(nestedFailed ?? raw(aliases.providerFailed)), + acceptedUnverified: acceptedUnverified === undefined + ? (hasProviderData ? 0 : launchCounts(launch).accepted) + : finiteCount(acceptedUnverified), + }; +}; + +const timestampAliases = { + sent: ['sentAt', 'sent_at', 'providerSentAt', 'provider_sent_at'], + delivered: ['deliveredAt', 'delivered_at', 'providerDeliveredAt', 'provider_delivered_at'], + delayed: ['delayedAt', 'delayed_at', 'providerDelayedAt', 'provider_delayed_at'], + bounced: ['bouncedAt', 'bounced_at', 'providerBouncedAt', 'provider_bounced_at'], + complained: ['complainedAt', 'complained_at', 'providerComplainedAt', 'provider_complained_at'], + suppressed: ['suppressedAt', 'suppressed_at', 'providerSuppressedAt', 'provider_suppressed_at'], + providerFailed: ['providerFailedAt', 'provider_failed_at', 'failedAt', 'failed_at'], +}; + +export const providerTimestamps = (delivery) => { + const nested = delivery?.providerTimestamps || delivery?.provider_timestamps + || delivery?.providerOutcomeTimestamps || delivery?.provider_outcome_timestamps || {}; + const sources = [nested, delivery || {}]; + return Object.fromEntries(Object.entries(timestampAliases).map(([outcome, keys]) => [ + outcome, + sources.map((source) => firstValue(source, keys)).find((value) => value !== undefined && value !== null) || null, + ])); +}; + +const normalizeStatus = (value) => String(value || '').trim().toLowerCase().replace(/[\s-]+/g, '_'); + +export const providerOutcome = (delivery) => { + const rawOutcome = delivery?.providerOutcome || delivery?.provider_outcome; + const explicit = normalizeStatus(typeof rawOutcome === 'object' + ? (rawOutcome?.outcome || rawOutcome?.status || rawOutcome?.effectiveOutcome || rawOutcome?.effective_outcome) + : rawOutcome); + const aliases = { + complaint: 'complained', email_complained: 'complained', bounce: 'bounced', email_bounced: 'bounced', + delivery_delayed: 'delayed', email_delivery_delayed: 'delayed', email_delivered: 'delivered', + failed: 'provider_failed', provider_failure: 'provider_failed', email_failed: 'provider_failed', + provider_suppressed: 'suppressed', email_suppressed: 'suppressed', sent: 'sent', email_sent: 'sent', + accepted: 'accepted_unverified', legacy_assumed_accepted: 'accepted_unverified', + none: 'none', not_queued: 'none', unverified: 'accepted_unverified', + }; + if (explicit) return aliases[explicit] || explicit; + + const timestamps = providerTimestamps(delivery); + const timestampOutcome = [ + ['complained', timestamps.complained], ['bounced', timestamps.bounced], + ['suppressed', timestamps.suppressed], ['provider_failed', timestamps.providerFailed], + ['delivered', timestamps.delivered], ['delayed', timestamps.delayed], ['sent', timestamps.sent], + ].find(([, value]) => value)?.[0]; + if (timestampOutcome) return timestampOutcome; + + const dispatch = normalizeStatus(delivery?.dispatchStatus || delivery?.dispatch_status || delivery?.emailStatus || delivery?.email_status || delivery?.status); + return ['accepted', 'legacy_assumed_accepted'].includes(dispatch) ? 'accepted_unverified' : 'none'; +}; + +export const providerOutcomeLabel = (outcome) => ({ + complained: 'Complained', bounced: 'Bounced', suppressed: 'Suppressed', + provider_failed: 'Provider failed', delivered: 'Delivered', delayed: 'Delayed', + sent: 'Provider accepted', accepted_unverified: 'Accepted / unverified', none: 'No provider outcome', +}[normalizeStatus(outcome)] || String(outcome || 'No provider outcome').replaceAll('_', ' ')); + +export const providerOutcomeTimestamp = (delivery, outcome = providerOutcome(delivery)) => { + const key = normalizeStatus(outcome) === 'provider_failed' ? 'providerFailed' : normalizeStatus(outcome); + const rawOutcome = delivery?.providerOutcome || delivery?.provider_outcome; + return providerTimestamps(delivery)[key] + || (typeof rawOutcome === 'object' && (rawOutcome.occurredAt || rawOutcome.occurred_at || rawOutcome.timestamp)) + || delivery?.providerOutcomeAt || delivery?.provider_outcome_at || null; +}; + export const launchStatus = (launch) => { const explicit = launch?.status || launch?.launchStatus; if (explicit) return String(explicit).toLowerCase(); @@ -46,6 +149,17 @@ export const launchStatus = (launch) => { }; export const isLaunchRunning = (launch) => ['queued', 'processing'].includes(launchStatus(launch)); +export const PROVIDER_RECONCILIATION_HORIZON_MS = 7 * 24 * 60 * 60 * 1000; + +export const shouldPollLaunch = (launch, now = Date.now()) => { + if (!launch) return false; + if (isLaunchRunning(launch)) return true; + const value = launch.providerUpdatedAt || launch.provider_updated_at || launch.finishedAt + || launch.finished_at || launch.updatedAt || launch.updated_at || launch.createdAt || launch.created_at; + const occurredAt = new Date(value).getTime(); + return Number.isFinite(occurredAt) && occurredAt >= now - PROVIDER_RECONCILIATION_HORIZON_MS; +}; + export const formatDateTime = (value) => { if (!value) return '—'; const date = new Date(value); diff --git a/db/changelogs/master-changelog.xml b/db/changelogs/master-changelog.xml index bcf25a1..0f64d83 100644 --- a/db/changelogs/master-changelog.xml +++ b/db/changelogs/master-changelog.xml @@ -14,4 +14,5 @@ + diff --git a/db/changelogs/v1_7_email_webhook_delivery_truth.sql b/db/changelogs/v1_7_email_webhook_delivery_truth.sql new file mode 100644 index 0000000..c76b25c --- /dev/null +++ b/db/changelogs/v1_7_email_webhook_delivery_truth.sql @@ -0,0 +1,571 @@ +--liquibase formatted sql + +--changeset cladvisors:email-webhook-delivery-truth-preflight-1 splitStatements:false +--comment Refuse a partial Phase 1 foundation before installing the additive webhook schema. +SET LOCAL lock_timeout = '5s'; +SET LOCAL statement_timeout = '5min'; +DO $$ +DECLARE + required_column TEXT; +BEGIN + IF current_setting('server_version_num')::INTEGER < 120000 THEN + RAISE EXCEPTION 'Phase 2 email webhook schema requires PostgreSQL 12 or newer'; + END IF; + + IF to_regclass('survey_email_deliveries') IS NULL + OR to_regclass('survey_email_attempts') IS NULL + OR to_regclass('email_worker_control') IS NULL THEN + RAISE EXCEPTION 'Phase 2 email webhook schema requires the Phase 1 email delivery schema'; + END IF; + + FOREACH required_column IN ARRAY ARRAY[ + 'id', 'launch_id', 'to_address', 'status', 'provider_message_id', + 'provider_delivered_at', 'provider_delayed_at', 'provider_bounced_at', + 'provider_complained_at', 'provider_suppressed_at', 'provider_failed_at' + ] LOOP + IF NOT EXISTS ( + SELECT 1 + FROM pg_attribute + WHERE attrelid = 'survey_email_deliveries'::regclass + AND attname = required_column + AND attnum > 0 + AND NOT attisdropped + ) THEN + RAISE EXCEPTION 'Phase 2 preflight failed: survey_email_deliveries.% is missing', required_column; + END IF; + END LOOP; + + IF EXISTS ( + SELECT provider_message_id + FROM survey_email_deliveries + WHERE provider_message_id IS NOT NULL + GROUP BY provider_message_id + HAVING count(*) > 1 + ) THEN + RAISE EXCEPTION 'Phase 2 preflight failed: duplicate provider message IDs prevent deterministic correlation'; + END IF; + + IF NOT EXISTS ( + SELECT 1 + FROM pg_index + WHERE indexrelid = to_regclass('delivery_provider_message_unique') + AND indrelid = 'survey_email_deliveries'::regclass + AND indisunique AND indisvalid AND indisready + ) THEN + RAISE EXCEPTION 'Phase 2 preflight failed: the valid Phase 1 provider-message unique index is missing'; + END IF; +END $$; + +--changeset cladvisors:email-webhook-delivery-projection-1 splitStatements:false +--comment Add only the missing independent provider occurrence used by the Phase 2 projection. +SET LOCAL lock_timeout = '5s'; +SET LOCAL statement_timeout = '5min'; +ALTER TABLE survey_email_deliveries ADD COLUMN IF NOT EXISTS provider_sent_at TIMESTAMPTZ; + +--changeset cladvisors:email-webhook-delivery-projection-indexes-1 runInTransaction:false +--comment Concurrent builds avoid blocking the existing Phase 1 delivery table and self-heal an invalid interrupted build. +SET lock_timeout = '5s'; +SET statement_timeout = '5min'; +DROP INDEX CONCURRENTLY IF EXISTS delivery_launch_provider_sent; +CREATE INDEX CONCURRENTLY delivery_launch_provider_sent + ON survey_email_deliveries(launch_id, provider_sent_at) + WHERE provider_sent_at IS NOT NULL; +DROP INDEX CONCURRENTLY IF EXISTS delivery_launch_provider_delivered; +CREATE INDEX CONCURRENTLY delivery_launch_provider_delivered + ON survey_email_deliveries(launch_id, provider_delivered_at) + WHERE provider_delivered_at IS NOT NULL; +DROP INDEX CONCURRENTLY IF EXISTS delivery_launch_provider_delayed; +CREATE INDEX CONCURRENTLY delivery_launch_provider_delayed + ON survey_email_deliveries(launch_id, provider_delayed_at) + WHERE provider_delayed_at IS NOT NULL; +DROP INDEX CONCURRENTLY IF EXISTS delivery_launch_provider_bounced; +CREATE INDEX CONCURRENTLY delivery_launch_provider_bounced + ON survey_email_deliveries(launch_id, provider_bounced_at) + WHERE provider_bounced_at IS NOT NULL; +DROP INDEX CONCURRENTLY IF EXISTS delivery_launch_provider_complained; +CREATE INDEX CONCURRENTLY delivery_launch_provider_complained + ON survey_email_deliveries(launch_id, provider_complained_at) + WHERE provider_complained_at IS NOT NULL; +DROP INDEX CONCURRENTLY IF EXISTS delivery_launch_provider_suppressed; +CREATE INDEX CONCURRENTLY delivery_launch_provider_suppressed + ON survey_email_deliveries(launch_id, provider_suppressed_at) + WHERE provider_suppressed_at IS NOT NULL; +DROP INDEX CONCURRENTLY IF EXISTS delivery_launch_provider_failed; +CREATE INDEX CONCURRENTLY delivery_launch_provider_failed + ON survey_email_deliveries(launch_id, provider_failed_at) + WHERE provider_failed_at IS NOT NULL; +RESET lock_timeout; +RESET statement_timeout; + +--changeset cladvisors:email-webhook-fenced-inbox-1 splitStatements:false +--comment Store verified raw events before acknowledgement; lease fields fence every projector finalization. +SET LOCAL lock_timeout = '5s'; +SET LOCAL statement_timeout = '5min'; +CREATE TABLE IF NOT EXISTS email_webhook_events ( + id BIGSERIAL PRIMARY KEY, + provider_account_scope VARCHAR(128) NOT NULL, + receiving_environment VARCHAR(64) NOT NULL, + svix_id VARCHAR(255) NOT NULL, + event_type VARCHAR(128) NOT NULL, + event_created_at TIMESTAMPTZ NOT NULL, + provider_message_id VARCHAR(255), + delivery_tag VARCHAR(256), + app_tag VARCHAR(256), + environment_tag VARCHAR(256), + raw_payload JSONB, + payload_size_bytes INTEGER NOT NULL, + received_at TIMESTAMPTZ NOT NULL DEFAULT now(), + payload_expires_at TIMESTAMPTZ NOT NULL, + status TEXT NOT NULL DEFAULT 'pending', + processing_attempt_count INTEGER NOT NULL DEFAULT 0, + next_attempt_at TIMESTAMPTZ DEFAULT now(), + unmatched_since_at TIMESTAMPTZ, + lease_owner VARCHAR(255), + lease_token UUID, + lease_expires_at TIMESTAMPTZ, + last_error_code VARCHAR(64), + last_error_message VARCHAR(500), + correlated_delivery_id UUID REFERENCES survey_email_deliveries(id) ON DELETE RESTRICT, + processed_at TIMESTAMPTZ, + dead_lettered_at TIMESTAMPTZ, + replay_count INTEGER NOT NULL DEFAULT 0, + last_replayed_at TIMESTAMPTZ, + last_replayed_by_actor VARCHAR(255), + last_replay_reason VARCHAR(500), + created_at TIMESTAMPTZ NOT NULL DEFAULT now(), + updated_at TIMESTAMPTZ NOT NULL DEFAULT now(), + CONSTRAINT email_webhook_event_provider_svix_unique UNIQUE(provider_account_scope, svix_id), + CONSTRAINT email_webhook_event_status_check CHECK ( + status IN ('pending','leased','retry_wait','processed','unmatched','ignored','dead_letter') + ), + CONSTRAINT email_webhook_event_scope_check CHECK ( + provider_account_scope = btrim(provider_account_scope) AND provider_account_scope <> '' + ), + CONSTRAINT email_webhook_event_environment_check CHECK ( + receiving_environment = btrim(receiving_environment) AND receiving_environment <> '' + ), + CONSTRAINT email_webhook_event_type_check CHECK (event_type = btrim(event_type) AND event_type <> ''), + CONSTRAINT email_webhook_event_payload_check CHECK ( + payload_size_bytes BETWEEN 2 AND 262144 + AND payload_expires_at >= received_at + ), + CONSTRAINT email_webhook_event_attempt_check CHECK (processing_attempt_count >= 0 AND replay_count >= 0), + CONSTRAINT email_webhook_event_lease_check CHECK ( + (status = 'leased' AND lease_owner IS NOT NULL AND lease_token IS NOT NULL AND lease_expires_at IS NOT NULL) + OR + (status <> 'leased' AND lease_owner IS NULL AND lease_token IS NULL AND lease_expires_at IS NULL) + ), + CONSTRAINT email_webhook_event_schedule_check CHECK ( + (status IN ('pending','retry_wait','unmatched') AND next_attempt_at IS NOT NULL) + OR + (status NOT IN ('pending','retry_wait','unmatched') AND next_attempt_at IS NULL) + ), + CONSTRAINT email_webhook_event_terminal_check CHECK ( + (status IN ('processed','ignored') AND processed_at IS NOT NULL AND dead_lettered_at IS NULL) + OR (status = 'dead_letter' AND dead_lettered_at IS NOT NULL) + OR (status NOT IN ('processed','ignored','dead_letter') AND processed_at IS NULL AND dead_lettered_at IS NULL) + ), + CONSTRAINT email_webhook_event_unmatched_check CHECK ( + (status = 'unmatched' AND unmatched_since_at IS NOT NULL) + OR status <> 'unmatched' + ), + CONSTRAINT email_webhook_event_replay_audit_check CHECK ( + (replay_count = 0 AND last_replayed_at IS NULL AND last_replayed_by_actor IS NULL AND last_replay_reason IS NULL) + OR (replay_count > 0 AND last_replayed_at IS NOT NULL AND last_replayed_by_actor IS NOT NULL AND last_replay_reason IS NOT NULL) + ) +); +COMMENT ON COLUMN email_webhook_events.raw_payload IS + 'Verified JSON payload, limited to a 256 KiB request; set to NULL after payload_expires_at without deleting deduplication metadata.'; +COMMENT ON COLUMN email_webhook_events.delivery_tag IS + 'Bounded untrusted tag text; cast to UUID only after projector validation.'; + +CREATE INDEX IF NOT EXISTS email_webhook_event_due_work + ON email_webhook_events(next_attempt_at, id) + WHERE status IN ('pending','retry_wait','unmatched'); +CREATE INDEX IF NOT EXISTS email_webhook_event_expired_lease + ON email_webhook_events(lease_expires_at, id) + WHERE status = 'leased'; +CREATE INDEX IF NOT EXISTS email_webhook_event_provider_message + ON email_webhook_events(provider_account_scope, provider_message_id, event_created_at) + WHERE provider_message_id IS NOT NULL; +CREATE INDEX IF NOT EXISTS email_webhook_event_delivery_tag + ON email_webhook_events(delivery_tag, event_created_at) + WHERE delivery_tag IS NOT NULL; +CREATE INDEX IF NOT EXISTS email_webhook_event_correlated_delivery + ON email_webhook_events(correlated_delivery_id, event_created_at) + WHERE correlated_delivery_id IS NOT NULL; +CREATE INDEX IF NOT EXISTS email_webhook_event_payload_retention + ON email_webhook_events(payload_expires_at, id) + WHERE raw_payload IS NOT NULL; +CREATE INDEX IF NOT EXISTS email_webhook_event_dead_letter + ON email_webhook_events(dead_lettered_at DESC) + WHERE status = 'dead_letter'; + +--changeset cladvisors:email-reason-keyed-suppression-1 splitStatements:false +--comment Keep each suppression cause independent and fail closed after provider removal until the exact cause version is overridden. +SET LOCAL lock_timeout = '5s'; +SET LOCAL statement_timeout = '5min'; +CREATE TABLE IF NOT EXISTS email_suppressions ( + provider_account_scope VARCHAR(128) NOT NULL, + normalized_address VARCHAR(320) NOT NULL, + reason TEXT NOT NULL, + receiving_environment VARCHAR(64) NOT NULL, + provider_active BOOLEAN NOT NULL DEFAULT true, + provider_suppression_id VARCHAR(255), + source_webhook_event_id BIGINT NOT NULL REFERENCES email_webhook_events(id) ON DELETE RESTRICT, + source_occurred_at TIMESTAMPTZ NOT NULL, + state_occurrence_at TIMESTAMPTZ NOT NULL, + state_event_svix_id VARCHAR(255) NOT NULL, + state_is_adverse BOOLEAN NOT NULL, + last_add_occurrence_at TIMESTAMPTZ, + last_add_event_svix_id VARCHAR(255), + last_remove_occurrence_at TIMESTAMPTZ, + last_remove_event_svix_id VARCHAR(255), + cause_version BIGINT NOT NULL DEFAULT 1, + locally_overridden_at TIMESTAMPTZ, + override_cause_version BIGINT, + locally_overridden_by_actor VARCHAR(255), + created_at TIMESTAMPTZ NOT NULL DEFAULT now(), + updated_at TIMESTAMPTZ NOT NULL DEFAULT now(), + PRIMARY KEY(provider_account_scope, normalized_address, reason), + CONSTRAINT email_suppression_reason_check CHECK ( + reason IN ('permanent_bounce','complaint','provider_suppression') + ), + CONSTRAINT email_suppression_scope_check CHECK ( + provider_account_scope = btrim(provider_account_scope) AND provider_account_scope <> '' + ), + CONSTRAINT email_suppression_environment_check CHECK ( + receiving_environment = btrim(receiving_environment) AND receiving_environment <> '' + ), + CONSTRAINT email_suppression_address_check CHECK ( + normalized_address = lower(btrim(normalized_address)) + AND normalized_address <> '' + AND position('@' IN normalized_address) > 1 + ), + CONSTRAINT email_suppression_version_check CHECK (cause_version > 0), + CONSTRAINT email_suppression_state_check CHECK (provider_active = state_is_adverse), + CONSTRAINT email_suppression_add_check CHECK ( + (last_add_occurrence_at IS NULL) = (last_add_event_svix_id IS NULL) + AND (NOT provider_active OR last_add_occurrence_at IS NOT NULL) + ), + CONSTRAINT email_suppression_remove_check CHECK ( + (last_remove_occurrence_at IS NULL) = (last_remove_event_svix_id IS NULL) + ), + CONSTRAINT email_suppression_override_check CHECK ( + (locally_overridden_at IS NULL AND override_cause_version IS NULL AND locally_overridden_by_actor IS NULL) + OR + (locally_overridden_at IS NOT NULL AND override_cause_version = cause_version + AND locally_overridden_by_actor IS NOT NULL AND provider_active = false) + ) +); +COMMENT ON TABLE email_suppressions IS + 'One independently versioned cause per provider account and normalized address; effective blocking is provider_active OR no exact-version local override.'; + +CREATE INDEX IF NOT EXISTS email_suppression_effective_lookup + ON email_suppressions(provider_account_scope, normalized_address) + WHERE provider_active OR locally_overridden_at IS NULL; +CREATE INDEX IF NOT EXISTS email_suppression_source_event + ON email_suppressions(source_webhook_event_id); +CREATE INDEX IF NOT EXISTS email_suppression_updated + ON email_suppressions(updated_at DESC); + +CREATE TABLE IF NOT EXISTS email_suppression_audit ( + id BIGSERIAL PRIMARY KEY, + provider_account_scope VARCHAR(128) NOT NULL, + normalized_address VARCHAR(320) NOT NULL, + reason TEXT NOT NULL, + receiving_environment VARCHAR(64) NOT NULL, + action TEXT NOT NULL, + cause_version BIGINT NOT NULL, + provider_active BOOLEAN NOT NULL, + occurrence_at TIMESTAMPTZ NOT NULL, + source_webhook_event_id BIGINT REFERENCES email_webhook_events(id) ON DELETE RESTRICT, + source_event_svix_id VARCHAR(255), + provider_suppression_id VARCHAR(255), + actor VARCHAR(255), + detail JSONB NOT NULL DEFAULT '{}'::jsonb, + recorded_at TIMESTAMPTZ NOT NULL DEFAULT now(), + CONSTRAINT email_suppression_audit_reason_check CHECK ( + reason IN ('permanent_bounce','complaint','provider_suppression') + ), + CONSTRAINT email_suppression_audit_action_check CHECK ( + action IN ('adverse_add','provider_remove','local_override','stale_event_ignored') + ), + CONSTRAINT email_suppression_audit_address_check CHECK ( + normalized_address = lower(btrim(normalized_address)) AND normalized_address <> '' + ), + CONSTRAINT email_suppression_audit_environment_check CHECK ( + receiving_environment = btrim(receiving_environment) AND receiving_environment <> '' + ), + CONSTRAINT email_suppression_audit_version_check CHECK (cause_version > 0), + CONSTRAINT email_suppression_audit_actor_check CHECK ( + action <> 'local_override' OR actor IS NOT NULL + ) +); +CREATE INDEX IF NOT EXISTS email_suppression_audit_cause + ON email_suppression_audit(provider_account_scope, normalized_address, reason, recorded_at DESC); +CREATE INDEX IF NOT EXISTS email_suppression_audit_source + ON email_suppression_audit(source_webhook_event_id) + WHERE source_webhook_event_id IS NOT NULL; + +--changeset cladvisors:email-webhook-operational-controls-1 splitStatements:false +--comment Add revision-fenced processing, suppression, registration, and global sending controls plus exact-release heartbeats. +SET LOCAL lock_timeout = '5s'; +SET LOCAL statement_timeout = '5min'; +CREATE TABLE IF NOT EXISTS email_webhook_worker_control ( + environment VARCHAR(64) PRIMARY KEY, + claiming_enabled BOOLEAN NOT NULL DEFAULT false, + processing_enabled BOOLEAN NOT NULL DEFAULT false, + minimum_release VARCHAR(255) NOT NULL DEFAULT '', + release_revision VARCHAR(255) NOT NULL DEFAULT '', + control_revision BIGINT NOT NULL DEFAULT 0 CHECK (control_revision >= 0), + updated_at TIMESTAMPTZ NOT NULL DEFAULT now(), + updated_by_actor VARCHAR(255), + reason VARCHAR(500), + CHECK (environment = btrim(environment) AND environment <> '') +); + +CREATE TABLE IF NOT EXISTS email_webhook_worker_heartbeats ( + environment VARCHAR(64) NOT NULL, + worker_instance VARCHAR(255) NOT NULL, + release_revision VARCHAR(255) NOT NULL, + enabled BOOLEAN NOT NULL, + claiming BOOLEAN NOT NULL, + processing BOOLEAN NOT NULL, + heartbeat_at TIMESTAMPTZ NOT NULL DEFAULT now(), + last_error_code VARCHAR(64), + last_error VARCHAR(500), + started_at TIMESTAMPTZ NOT NULL DEFAULT now(), + PRIMARY KEY(environment, worker_instance), + CHECK (environment = btrim(environment) AND environment <> ''), + CHECK (worker_instance = btrim(worker_instance) AND worker_instance <> ''), + CHECK (release_revision = btrim(release_revision) AND release_revision <> ''), + CHECK (heartbeat_at >= started_at) +); +CREATE INDEX IF NOT EXISTS email_webhook_worker_heartbeat_fresh + ON email_webhook_worker_heartbeats(environment, heartbeat_at DESC); +CREATE INDEX IF NOT EXISTS email_webhook_worker_release_fresh + ON email_webhook_worker_heartbeats(environment, release_revision, heartbeat_at DESC); + +CREATE TABLE IF NOT EXISTS email_suppression_control ( + environment VARCHAR(64) PRIMARY KEY, + enforcement_enabled BOOLEAN NOT NULL DEFAULT false, + activated_at TIMESTAMPTZ, + activated_by_actor VARCHAR(255), + activation_release VARCHAR(255), + minimum_release VARCHAR(255) NOT NULL DEFAULT '', + control_revision BIGINT NOT NULL DEFAULT 0 CHECK (control_revision >= 0), + updated_at TIMESTAMPTZ NOT NULL DEFAULT now(), + updated_by_actor VARCHAR(255), + reason VARCHAR(500), + CHECK (environment = btrim(environment) AND environment <> ''), + CHECK ( + (activated_at IS NULL AND activated_by_actor IS NULL AND activation_release IS NULL AND enforcement_enabled = false) + OR + (activated_at IS NOT NULL AND activated_by_actor IS NOT NULL AND activation_release IS NOT NULL) + ) +); + +CREATE TABLE IF NOT EXISTS email_sending_control ( + environment VARCHAR(64) PRIMARY KEY, + sending_enabled BOOLEAN NOT NULL DEFAULT false, + minimum_release VARCHAR(255) NOT NULL DEFAULT '', + control_revision BIGINT NOT NULL DEFAULT 0 CHECK (control_revision >= 0), + updated_at TIMESTAMPTZ NOT NULL DEFAULT now(), + updated_by_actor VARCHAR(255), + reason VARCHAR(500), + CHECK (environment = btrim(environment) AND environment <> '') +); + +CREATE TABLE IF NOT EXISTS email_webhook_registration_control ( + environment VARCHAR(64) PRIMARY KEY, + provider_account_scope VARCHAR(128) NOT NULL, + endpoint_id VARCHAR(255), + endpoint_state TEXT NOT NULL DEFAULT 'unregistered', + previous_endpoint_id VARCHAR(255), + previous_endpoint_state TEXT, + previous_secret_parameter_version BIGINT, + endpoint_url_hash VARCHAR(128), + event_set_hash VARCHAR(128) NOT NULL, + operation_id UUID NOT NULL, + pre_operation_endpoint_ids JSONB NOT NULL DEFAULT '[]'::jsonb, + secret_parameter_version BIGINT, + ingestion_required BOOLEAN NOT NULL DEFAULT false, + minimum_ingest_release VARCHAR(255) NOT NULL DEFAULT '', + control_revision BIGINT NOT NULL DEFAULT 0 CHECK (control_revision >= 0), + updated_at TIMESTAMPTZ NOT NULL DEFAULT now(), + updated_by_actor VARCHAR(255) NOT NULL, + reason VARCHAR(500), + CHECK (environment = btrim(environment) AND environment <> ''), + CHECK (provider_account_scope = btrim(provider_account_scope) AND provider_account_scope <> ''), + CHECK (endpoint_state IN ('unregistered','bootstrap_disabled','enable_pending','enabled','disabled','deleted','recovery_required','rotation_required','rotation_disabled')), + CHECK (previous_endpoint_state IS NULL OR previous_endpoint_state IN ('enabled','disabled')), + CHECK (jsonb_typeof(pre_operation_endpoint_ids) = 'array'), + CHECK (secret_parameter_version IS NULL OR secret_parameter_version > 0), + CHECK (previous_secret_parameter_version IS NULL OR previous_secret_parameter_version > 0), + CHECK (endpoint_state IN ('unregistered','deleted','recovery_required') OR endpoint_id IS NOT NULL), + CHECK (NOT ingestion_required OR endpoint_state IN ('bootstrap_disabled','enable_pending','enabled','disabled','rotation_required','rotation_disabled')) +); + +CREATE TABLE IF NOT EXISTS email_control_audit ( + id BIGSERIAL PRIMARY KEY, + environment VARCHAR(64) NOT NULL, + control_name TEXT NOT NULL, + control_revision BIGINT NOT NULL CHECK (control_revision >= 0), + operation_id UUID, + previous_value JSONB, + new_value JSONB NOT NULL, + actor VARCHAR(255) NOT NULL, + reason VARCHAR(500) NOT NULL, + recorded_at TIMESTAMPTZ NOT NULL DEFAULT now(), + CHECK (environment = btrim(environment) AND environment <> ''), + CHECK (control_name IN ('webhook_processing','suppression_enforcement','email_sending','webhook_registration')), + CHECK (previous_value IS NULL OR jsonb_typeof(previous_value) = 'object'), + CHECK (jsonb_typeof(new_value) = 'object'), + UNIQUE(environment, control_name, control_revision) +); +CREATE INDEX IF NOT EXISTS email_control_audit_recorded + ON email_control_audit(environment, recorded_at DESC); + +INSERT INTO email_webhook_worker_control(environment, claiming_enabled, processing_enabled) +VALUES ('local',false,false),('test',false,false),('staging',false,false),('prod',false,false) +ON CONFLICT (environment) DO NOTHING; +INSERT INTO email_suppression_control(environment, enforcement_enabled) +VALUES ('local',false),('test',false),('staging',false),('prod',false) +ON CONFLICT (environment) DO NOTHING; +INSERT INTO email_sending_control(environment, sending_enabled) +VALUES ('local',true),('test',true),('staging',false),('prod',false) +ON CONFLICT (environment) DO NOTHING; + +--changeset cladvisors:email-webhook-canary-state-1 splitStatements:false +--comment A singleton per environment schedules and fences the six-hour real-provider webhook canary. +SET LOCAL lock_timeout = '5s'; +SET LOCAL statement_timeout = '5min'; +CREATE TABLE IF NOT EXISTS email_webhook_canary_state ( + environment VARCHAR(64) PRIMARY KEY, + provider_account_scope VARCHAR(128) NOT NULL, + status TEXT NOT NULL DEFAULT 'idle', + canary_token UUID, + scheduled_at TIMESTAMPTZ NOT NULL DEFAULT now(), + next_run_at TIMESTAMPTZ NOT NULL DEFAULT now(), + attempt_count INTEGER NOT NULL DEFAULT 0, + lease_owner VARCHAR(255), + lease_token UUID, + lease_expires_at TIMESTAMPTZ, + provider_started_at TIMESTAMPTZ, + provider_message_id VARCHAR(255), + sent_at TIMESTAMPTZ, + delivered_at TIMESTAMPTZ, + correlated_webhook_event_id BIGINT REFERENCES email_webhook_events(id) ON DELETE RESTRICT, + completed_at TIMESTAMPTZ, + last_success_at TIMESTAMPTZ, + last_error_code VARCHAR(64), + last_error_message VARCHAR(500), + created_at TIMESTAMPTZ NOT NULL DEFAULT now(), + updated_at TIMESTAMPTZ NOT NULL DEFAULT now(), + CHECK (environment = btrim(environment) AND environment <> ''), + CHECK (provider_account_scope = btrim(provider_account_scope) AND provider_account_scope <> ''), + CHECK (status IN ('idle','leased','sending','awaiting_webhook','retry_wait','failed')), + CHECK (attempt_count >= 0), + CHECK ( + (status IN ('leased','sending') AND lease_owner IS NOT NULL AND lease_token IS NOT NULL AND lease_expires_at IS NOT NULL) + OR + (status NOT IN ('leased','sending') AND lease_owner IS NULL AND lease_token IS NULL AND lease_expires_at IS NULL) + ), + CHECK (provider_started_at IS NULL OR status IN ('sending','awaiting_webhook','retry_wait','failed','idle')), + CHECK (delivered_at IS NULL OR sent_at IS NOT NULL), + CHECK (completed_at IS NULL OR delivered_at IS NOT NULL), + CHECK (correlated_webhook_event_id IS NULL OR canary_token IS NOT NULL) +); +CREATE UNIQUE INDEX IF NOT EXISTS email_webhook_canary_token_unique + ON email_webhook_canary_state(canary_token) + WHERE canary_token IS NOT NULL; +CREATE UNIQUE INDEX IF NOT EXISTS email_webhook_canary_provider_message_unique + ON email_webhook_canary_state(provider_message_id) + WHERE provider_message_id IS NOT NULL; +CREATE INDEX IF NOT EXISTS email_webhook_canary_due + ON email_webhook_canary_state(next_run_at, environment) + WHERE status IN ('idle','retry_wait'); +CREATE INDEX IF NOT EXISTS email_webhook_canary_expired_lease + ON email_webhook_canary_state(lease_expires_at, environment) + WHERE status IN ('leased','sending'); +CREATE INDEX IF NOT EXISTS email_webhook_canary_last_success + ON email_webhook_canary_state(environment, last_success_at DESC); + +--changeset cladvisors:email-phase2-append-only-guards-1 splitStatements:false +--comment Enforce immutable operator and suppression history at the database boundary. +SET LOCAL lock_timeout = '5s'; +SET LOCAL statement_timeout = '5min'; +CREATE OR REPLACE FUNCTION reject_email_phase2_audit_mutation() +RETURNS trigger +LANGUAGE plpgsql +AS $$ +BEGIN + RAISE EXCEPTION '% is append-only; % is not permitted', TG_TABLE_NAME, TG_OP; +END; +$$; + +DROP TRIGGER IF EXISTS email_suppression_audit_append_only ON email_suppression_audit; +CREATE TRIGGER email_suppression_audit_append_only +BEFORE UPDATE OR DELETE ON email_suppression_audit +FOR EACH ROW EXECUTE FUNCTION reject_email_phase2_audit_mutation(); + +DROP TRIGGER IF EXISTS email_control_audit_append_only ON email_control_audit; +CREATE TRIGGER email_control_audit_append_only +BEFORE UPDATE OR DELETE ON email_control_audit +FOR EACH ROW EXECUTE FUNCTION reject_email_phase2_audit_mutation(); + +CREATE OR REPLACE FUNCTION enforce_suppression_control_latch() +RETURNS trigger +LANGUAGE plpgsql +AS $$ +BEGIN + IF OLD.enforcement_enabled + AND NOT NEW.enforcement_enabled + AND EXISTS ( + SELECT 1 FROM email_sending_control + WHERE environment = OLD.environment AND sending_enabled + ) THEN + RAISE EXCEPTION 'cannot disable suppression enforcement while email sending is enabled for %', OLD.environment; + END IF; + + IF OLD.activated_at IS NOT NULL AND ( + NEW.activated_at IS DISTINCT FROM OLD.activated_at + OR NEW.activated_by_actor IS DISTINCT FROM OLD.activated_by_actor + OR NEW.activation_release IS DISTINCT FROM OLD.activation_release + ) THEN + RAISE EXCEPTION 'suppression activation evidence is immutable for %', OLD.environment; + END IF; + RETURN NEW; +END; +$$; + +DROP TRIGGER IF EXISTS email_suppression_control_latch ON email_suppression_control; +CREATE TRIGGER email_suppression_control_latch +BEFORE UPDATE ON email_suppression_control +FOR EACH ROW EXECUTE FUNCTION enforce_suppression_control_latch(); + +CREATE OR REPLACE FUNCTION enforce_email_sending_suppression_latch() +RETURNS trigger +LANGUAGE plpgsql +AS $$ +BEGIN + IF NEW.sending_enabled + AND NOT OLD.sending_enabled + AND EXISTS ( + SELECT 1 + FROM email_suppression_control + WHERE environment = NEW.environment + AND activated_at IS NOT NULL + AND NOT enforcement_enabled + ) THEN + RAISE EXCEPTION 'cannot enable email sending without activated suppression enforcement for %', NEW.environment; + END IF; + RETURN NEW; +END; +$$; + +DROP TRIGGER IF EXISTS email_sending_suppression_latch ON email_sending_control; +CREATE TRIGGER email_sending_suppression_latch +BEFORE UPDATE ON email_sending_control +FOR EACH ROW EXECUTE FUNCTION enforce_email_sending_suppression_latch(); diff --git a/docs/plan/phase-2-webhook-investigation.md b/docs/plan/phase-2-webhook-investigation.md new file mode 100644 index 0000000..e42772b --- /dev/null +++ b/docs/plan/phase-2-webhook-investigation.md @@ -0,0 +1,491 @@ +# Phase 2 Resend Webhook and Suppression Investigation + +Status: implementation planning complete; staging registration authorized; Resend upgrade required before production registration +Parent plan: `docs/plan/survey-lifecycle-email-delivery.md` +Scope: provider delivery truth, durable webhook ingestion/projection, suppression, reconciliation, dashboard presentation, and operations. Open/click engagement tracking is excluded. + +## 1. Executive conclusion + +The Phase 1 design is a suitable foundation for Phase 2. It already provides stable delivery UUIDs, unique provider message IDs, independent provider-outcome timestamps, durable worker patterns, dispatch/provider separation, and provider-boundary fencing. + +Phase 2 can proceed additively. It must not reinterpret `accepted` as `delivered`, alter dispatch progress arithmetic, resend mail while replaying a webhook, or make the webhook signing secret a prerequisite for Phase 1 operation. + +A secure API inspection confirmed that staging and production use distinct API keys belonging to the same Resend team (both keys return the same API-key set) and that the team currently has zero webhook registrations. No secret values or identifiers were printed or retained. The team is currently on Resend Free, which permits one webhook endpoint. The approved decision is to use that endpoint for staging validation, then upgrade before production so staging and production can retain separate endpoints and secrets. + +## 2. Verified provider contracts + +### Delivery guarantees + +Resend documents: + +- HTTPS JSON webhook requests. +- At-least-once delivery; duplicates are possible. +- `svix-id` is the durable deduplication identity. +- Delivery order is not guaranteed. +- Event `created_at` should be used for event ordering, not arrival time. +- Automatic attempts occur immediately, then after 5 seconds, 5 minutes, 30 minutes, 2 hours, 5 hours, 10 hours, and another 10 hours. +- Failed and successful messages can be manually replayed from the Resend dashboard. +- Continued endpoint failures can cause Resend to disable the endpoint. + +Therefore the receiver must acknowledge only after a verified event is durably inserted. A valid duplicate returns `200`. A database failure returns non-2xx so Resend retries. + +### Signature verification + +Resend requires the exact raw payload and these headers: + +- `svix-id` +- `svix-timestamp` +- `svix-signature` + +The repository pins `resend` 6.18.1. Its documented `resend.webhooks.verify()` implementation uses `standardwebhooks` 1.0.0, the same underlying verifier used by the current Svix package. The standard verifier rejects timestamps more than five minutes in the past or future. + +Decision: + +- Use the pinned Resend SDK's public `webhooks.verify()` API. +- Do not add the broader `svix` client solely for verification. +- Contract-test valid, mutated, stale, future, missing-header, and wrong-secret fixtures. +- Mount `express.raw({ type: 'application/json', limit: '256kb' })` for the webhook route before the global `express.json()` middleware. Resend publishes no maximum; 256 KiB is a conservative local bound for the selected metadata-only events and must be validated against captured staging payloads before production. +- Never log raw payloads, signing headers, full recipient addresses, or secrets. + +### Selected event set + +Subscribe only to: + +1. `email.sent` +2. `email.delivered` +3. `email.delivery_delayed` +4. `email.bounced` +5. `email.complained` +6. `email.failed` +7. `email.suppressed` +8. `suppression.added` +9. `suppression.removed` + +Do not subscribe to `email.opened`, `email.clicked`, inbound-email, contact, domain, or scheduled-email events. + +`email.sent` is useful even though Phase 1 records API acceptance: it can reconcile a delivery whose local provider call ended as uncertain. `email.failed` is a provider failure outcome associated with an email but does not itself prove prior API acceptance; it must not trigger an automatic resend. + +### Payload and correlation + +Outbound email events include: + +- top-level `type` and `created_at` +- `data.email_id`, which corresponds to the Resend send result ID +- `data.to` +- event-specific details +- optional `data.tags` as an object; legacy and non-survey sends may have no tags + +Send-time tags are an array of `{ name, value }`; webhook tags are returned as an object. Resend permits up to 75 tags, with names and values restricted to ASCII letters, numbers, underscores, and dashes and bounded to 256 characters. + +Decision: add these non-secret tags to every Phase 2 survey delivery: + +- `app=network_survey` +- `environment=` +- `delivery_id=` + +Correlation order: + +1. `data.email_id` to unique `survey_email_deliveries.provider_message_id` +2. `data.tags.delivery_id` to delivery UUID +3. otherwise retain as unmatched for reconciliation + +Never correlate by recipient, subject, RFC message ID, or timestamps. Validate that a tag-correlated delivery belongs to the expected application/environment and that the provider email ID does not conflict with another delivery. + +Staging and production endpoints on a shared Resend team may receive the same account-level event stream. Environment tags allow an endpoint to durably acknowledge but ignore foreign-environment survey email events. Account-level `suppression.added` events must still be applied in every sending environment that shares the provider suppression list. This behavior must be confirmed with a real shared-team staging registration. + +## 3. State model and event semantics + +### Dispatch remains separate + +The existing mutually exclusive dispatch dimension remains: + +- pending +- leased/sending +- retrying +- accepted +- failed +- uncertain +- cancelled + +Provider projections are an independent dimension. They never enter the dispatch-progress denominator or sum. + +Acceptance evidence is event-specific. A correlated `email.sent` explicitly proves that the API request succeeded. Correlated delivered, delayed, bounced, complained, or email-suppressed events prove that Resend processed that specific email far enough to establish acceptance. `email.failed` does not safely prove API acceptance because Resend documents API-key, domain, recipient, and quota causes; it records provider failure but leaves a locally uncertain dispatch uncertain. Account-level `suppression.added` and `suppression.removed` never prove anything about a delivery. When acceptance is proven, append-only attempt history preserves the earlier local uncertainty. + +### Provider occurrence timestamps + +Keep each provider fact independently. For repeated events, store the earliest verified provider occurrence with `LEAST(existing, event_created_at)`. Preserve `received_at` in the inbox separately. Never erase a provider fact because an older or lower-precedence event arrives later. + +Recommended effective presentation precedence: + +1. complained +2. bounced +3. suppressed +4. provider failed +5. delivered +6. delayed +7. sent/dispatch accepted +8. current dispatch state + +Precedence is presentation only; all timestamps remain queryable. A complaint may legitimately coexist with delivered because complaint occurs after delivery. + +### Event-specific effects + +- `email.sent`: prove provider acceptance; no delivery claim. +- `email.delivered`: set delivered occurrence. +- `email.delivery_delayed`: set delayed occurrence; later delivery remains visible as delivered. +- `email.bounced`: set bounce occurrence and activate local permanent-bounce suppression. +- `email.complained`: set complaint occurrence and activate local complaint suppression. +- `email.failed`: set provider-failed occurrence; preserve an already accepted dispatch, leave an uncertain dispatch uncertain, and do not automatically resend. +- `email.suppressed`: set provider-suppressed occurrence and activate local provider suppression. +- `suppression.added`: activate account-level local suppression using `data.email`, `origin`, `source_id`, and event time. +- `suppression.removed`: record provider removal, but remain locally fail-closed until an audited platform-operator override. This prevents an out-of-order or accidental provider removal from silently re-enabling mail. + +Unknown verified event types or additive fields are durably stored and acknowledged, then marked ignored. A malformed known event is dead-lettered rather than causing provider retries forever after durable receipt. + +## 4. Additive database design + +Create a new changelog after v1.6; never modify the already-deployable v1.6 changeset. + +### `email_webhook_events` + +Required fields: + +- provider account scope and receiving environment +- unique `svix_id` +- event type and provider event time +- provider message ID and delivery tag when available +- bounded raw JSON payload, received time, and `payload_expires_at` +- status: pending, leased, retry_wait, processed, unmatched, ignored, dead_letter +- processing attempt count and next attempt +- lease owner, random lease token, lease expiry +- bounded error code/message +- correlated delivery ID when found +- processed/dead-letter timestamps +- replay count, last replayed time, and audited replay actor where applicable + +Required constraints/indexes: + +- unique provider + `svix_id` +- due-work and expired-lease partial indexes +- provider-message-ID and delivery-tag reconciliation indexes +- retention deadline index +- state/lease consistency checks +- bounded diagnostic columns + +### `email_suppressions` + +Suppression is provider-account/address scoped, not organization scoped, because a shared Resend team suppresses the address for all sends. + +Store independent live causes keyed by `(provider_account_scope, normalized_address, reason)`, where reason is permanent bounce, complaint, or provider suppression. Effective suppression is `EXISTS` any active, non-overridden cause; one cause can never clear another. + +Required fields per cause: + +- stable provider-account scope identifier, configured identically in every environment sharing the Resend team; receiving environment is audit metadata, not suppression identity +- normalized address and reason +- provider-active state +- source webhook event/provider suppression ID +- source occurrence time +- last add/remove occurrence time and event ID +- locally overridden time, cause-version, and platform-operator actor +- created/updated timestamps + +Use event occurrence time to prevent an older out-of-order event from replacing newer state for the same cause; at equal timestamps, adverse add wins over removal and event ID breaks ties between equal event types. Every newer adverse event clears an override for that cause. A provider removal changes only the provider-suppression cause and records provider-active=false; permanent-bounce and complaint causes remain blocked. Local enforcement for a removed cause remains active until a platform operator overrides that exact latest cause version; an override is rejected while provider-active=true and is automatically invalidated by a later adverse event. Preserve immutable suppression audit history in a companion history table or append-only audit table. + +Address normalization must match delivery normalization used by Phase 1: trim and lowercase. Do not attempt correlation or suppression from unverified arbitrary recipient data. + +### Existing delivery changes + +The Phase 1 table already has provider timestamps. Add only what is needed for precise projection, such as a provider-sent occurrence if retained and indexes for provider outcome aggregation. Do not overwrite old accepted deliveries that have null provider outcomes; present them as accepted/unverified. + +## 5. Processing and reconciliation + +### Ingress transaction + +1. Enforce the ingestion gate. Before registration, disabled returns `503`; after registration, ingestion is treated as an always-on compatibility capability and the endpoint must be disabled/removed at Resend before this gate is turned off. +2. Read the raw body with the 256 KiB limit. +3. Verify signature and five-minute timestamp tolerance. +4. Validate bounded top-level metadata without rejecting additive unknown fields. +5. Insert by `svix-id` in one short transaction. +6. Return `200` after insertion or when the verified ID already exists. +7. Return `400` for invalid signatures/headers, `413` for oversized input, and `503` for database unavailability. + +The HTTP handler performs no delivery projection and no provider call. + +### Fenced projector + +Use the existing delivery-worker pattern: + +1. Claim due inbox rows with `FOR UPDATE SKIP LOCKED`. +2. Assign owner, random lease token, expiry, and increment attempt count. +3. Commit before processing. +4. Correlate by provider ID then delivery tag. +5. Project under a delivery row lock, fenced by current event lease token. +6. When event-specific evidence proves acceptance, transition any pending/retry/uncertain/failed/cancelled delivery to accepted, preserve cancellation evidence, set provider ID and dispatch acceptance, clear retry/lease fields, and dual-write `Respondent.email_sent`. For a leased delivery, finish an in-progress attempt as accepted only when `provider_started_at` is set; otherwise finish that pre-boundary attempt as cancelled with `provider_acceptance_reconciled`. The sender's later lease-token finalizer then safely affects zero rows. `email.failed` alone performs no such transition. +7. Mark processed only in the same transaction as projection and the respondent dual-write. +8. A stale worker cannot project or finalize after lease loss. + +Recommendation: run a dedicated `ona-email-webhook-worker` PM2 process with its own control and heartbeat namespace. This isolates projection from provider-send latency and permits pausing webhook projection without stopping Phase 1 dispatch. Deployment and rollback must verify both worker heartbeats once Phase 2 is enabled. + +### Unmatched events + +Unmatched is not immediately a processing failure: + +- Retry with bounded exponential backoff and jitter. +- Wake likely matches when the delivery worker stores a provider message ID. +- Retry for seven days, then move to dead letter. +- Never correlate by PII heuristics. +- Foreign-environment events with explicit valid tags are processed as ignored, not unmatched. +- Keep dead-letter metadata after raw payload expiry. + +Recommended transient processing policy: maximum 12 attempts or 72 hours for ordinary projector failures. Unmatched correlation uses the separate seven-day horizon. + +### Replay + +Provider replay may deliver the same `svix-id`, so it tests ingress durability and deduplication but does not force a second projection. Local replay is an audited operator action that resets processing state for the original stored event and creates a new processing attempt. Replay never creates a delivery or sends an email. + +Initially expose replay as a platform-operator command, not a tenant-facing HTTP endpoint. + +## 6. Suppression/send race fencing + +A plain suppression query immediately before send is insufficient: suppression can commit between that query and provider invocation. + +Add a deterministic address-level advisory boundary lock shared by suppression projection and the delivery worker. + +Universal sender/projector lock order: + +1. delivery-worker global provider-boundary lock when crossing a send boundary +2. worker control row when sending +3. Survey provider-boundary advisory lock when sending +4. Survey row when sending +5. delivery row +6. normalized-address advisory lock when suppression state is read or changed +7. suppression check +8. provider-boundary marker commit +9. provider invocation + +A projector that already holds a delivery row may then acquire the address lock. No path may hold an address lock and then acquire a delivery, Survey, control, or global-boundary lock. The sender uses session-scoped `pg_advisory_lock` for the address on the same retained connection, commits the provider-boundary marker while retaining that lock, invokes `provider.send()` so network I/O has begun, then explicitly unlocks in `finally`. A transaction-scoped address lock is forbidden because commit would reopen the suppression-check/send race. + +Suppression upsert acquires only the normalized-address lock, writes suppression, and commits. It must not hold that lock while later acquiring Survey/global locks. This avoids reverse-order deadlocks. + +Linearization: + +- If suppression obtains the address lock first, a later worker sees suppression and does not send. +- If the worker obtains it first, the provider boundary commits first; the message is honestly recorded as already crossed when suppression arrives. + +For a correlated delivery event such as bounce, complaint, or email suppression, delivery outcome projection, the reason-keyed suppression upsert, event lease fencing, and event completion commit atomically under delivery-row then address-lock order. For account-level `suppression.added`/`removed` events without a delivery, the event transaction acquires only the address lock and commits the cause plus event completion. It never acquires a delivery afterward. + +After an account-level suppression commit, or in a separate post-commit reconciliation pass for correlated events, reconciliation: + +- cancels pending/retry-wait work for the address +- marks leased pre-call work cancellation-requested so its final check cancels it +- sets the suppression provider projection +- never rewrites accepted/uncertain evidence as unsent + +Launch readiness should report suppressed recipients separately. A launch with no eligible recipients returns `422` and creates no launch. Partially suppressed audiences require an explicit product decision before Phase 3; for Phase 2 initial-launch compatibility, readiness should block rather than silently change the immutable audience. + +## 7. API and dashboard contract + +Add fields without renaming existing dispatch fields: + +- provider outcome counts on launch/history/latest-launch responses +- provider occurrence timestamps and effective provider outcome on delivery details +- separate `dispatchStatus` and `providerOutcome` on targets +- retain legacy `emailStatus` temporarily for compatibility + +Dashboard behavior: + +- dispatch progress remains unchanged +- show delivered, delayed, bounced, complained, suppressed, and provider-failed as a separate summary +- never label accepted as delivered +- use text/icons as well as color +- continue low-frequency terminal polling for a documented reconciliation horizon, then require manual refresh +- display old null-outcome rows as accepted/unverified + +Freeze the current launch replay contract: a newly created launch returns `202`; an exact replay returns `200` with the existing launch. + +## 8. Configuration, infrastructure, and rollout + +### Independent gates + +Add three exact-true, default-off controls; do not reuse `SURVEY_DELIVERY_V2_ENABLED`: + +1. `RESEND_WEBHOOK_INGEST_ENABLED` is release configuration. Disabled returns `503`, never `2xx`. It may be false before registration, but after registration it is an always-on compatibility capability: disable/remove the Resend endpoint first, then turn it off by redeploying. +2. `RESEND_WEBHOOK_PROCESSING_ENABLED` is stored in a new `email_webhook_worker_control` row and changed by a revision-fenced audited operator script. Processing may pause while ingestion continues. +3. `EMAIL_SUPPRESSION_ENFORCEMENT_ENABLED` is a database latch with activation time/actor/release. Before first activation it may be false. After activation it cannot be cleared by normal tooling while any application mail path remains enabled; compatible releases must continue enforcing known suppression. + +Add dedicated webhook control and heartbeat tables containing environment, claiming/processing state, minimum release, release revision, worker instance, freshness, bounded last error, update actor/reason, and startup time. Deployment health requires a fresh heartbeat for the exact deployment instance even when processing is intentionally paused. Operator changes use the same revision/fresh-heartbeat safeguards as Phase 1 claiming control. + +Also add one audited environment-wide `EMAIL_SENDING_ENABLED` database control checked by the delivery worker and every synchronous demo/account/invite/reset provider path under the shared provider-boundary fence. It defaults off in hosted environments until explicitly enabled. Break-glass can therefore stop all application-originated provider calls, not only survey delivery claiming. + +### Secrets and IAM + +Add per-environment `RESEND_WEBHOOK_SECRET_PARAMETER` and optional `RESEND_WEBHOOK_PREVIOUS_SECRET_PARAMETER` paths and permit the EC2 role to read only those environment-specific SSM parameters. Do not expose secret values through Terraform state, workflow output, logs, or chat. The verifier tries the primary secret and then the previous secret; it emits only a bounded `verified_with_previous` metric. + +Rotation procedure: + +1. copy the current primary secret to the previous-secret SSM parameter in process memory without printing it; +2. create the replacement registration, immediately set it disabled, and recover its secret through the documented retrieve/list API if the script is interrupted; +3. write the replacement secret to primary SSM, deploy, and prove fixtures for both loaded secrets; +4. enable the replacement endpoint while leaving the old endpoint enabled for a 36-hour overlap, exceeding Resend's documented automatic retry schedule; +5. disable/delete the old endpoint, clear previous-secret SSM, and redeploy. + +Creation uses the real endpoint while ingestion returns non-2xx until the secret is loaded, so a narrow create/disable race is retryable rather than silently acknowledged. + +If Resend supports an in-place signing-secret rotation by implementation time, the same primary/previous overlap applies without a second endpoint. Endpoint-count limits make provider-plan confirmation necessary first. + +### Registration ownership + +Implement a dedicated idempotent operator script, rather than a Terraform resource that could store the signing secret in Terraform state. The script: + +- requires explicit environment, expected public URL, provider-account scope, and exact event-set hash; +- snapshots the complete pre-operation endpoint-ID set and records it with an operation ID before provider creation; +- creates only the selected environment registration, immediately disables it during bootstrap, and retrieves its signing secret through the documented get/list API on recovery; +- on recovery, computes the post-minus-pre endpoint set and accepts exactly one matching newly created endpoint; if multiple candidates exist it disables all candidates and requires an audited operator selection rather than guessing; +- writes the signing secret directly to SSM and never stdout/stderr, then reconciles provider registration, SSM version, and control record on every rerun; crash-point tests cover every boundary from pre-snapshot through final control commit; +- stores non-secret endpoint ID, URL hash, event-set hash, provider-account scope, actor, and update time in an audited control record; +- supports status, disable, and delete as separate confirmation-required operations; +- refuses deletion while unprocessed inbox events remain unless break-glass is explicitly audited. + +Provider endpoint ID is non-secret; the signing secret exists only in SSM/runtime memory. + +### Payload retention + +Recommended default: + +- raw webhook payload: 30 days, configurable +- processed/dead-letter metadata and provider projections: retained with survey delivery history +- a daily webhook-worker maintenance task claims batches of at most 100 expired rows with `FOR UPDATE SKIP LOCKED`; nonterminal expired rows are first dead-lettered with `payload_expired`, then raw payload is nulled in the same transaction +- purge clears raw payload for every expired row; it never deletes deduplication identity, audit metadata, or projections; failures retry on the next maintenance pass and raise an alarm + +Resend pricing currently advertises 30-day provider data retention on Free/Pro/Scale, but does not document a guaranteed webhook replay window. Local retention must therefore be authoritative for the configured period. + +### Monitoring + +Phase 2 adds CloudWatch log groups/retention and the CloudWatch agent on the API host. Every 60 seconds the webhook worker queries current gauges and emits CloudWatch Embedded Metric Format records with stable Environment-only alarm dimensions; release revision is a non-dimension diagnostic property and may also be emitted in a separate non-alarm metric stream. The agent forwards them. Missing heartbeat/gauge data is treated as breaching. API/workers also emit bounded counter records. No address, subject, raw payload, signature, or secret appears in these records. + +Initial alarm defaults: + +- webhook heartbeat absent for two consecutive 60-second periods: alarm +- oldest pending/retry event over 5 minutes for two periods: alarm +- unmatched event over 1 hour for two periods: warning; over 24 hours for one period: alarm +- dead-letter count at least 1 for one period: alarm +- invalid-signature count over 10 in 5 minutes: alarm +- suppression reconciliation or purge failure at least 1 for one period: alarm +- existing uncertain delivery count at least 1 and quota-triggered claiming disable for one period: alarm +- controlled provider canary absent for 18 hours: alarm + +Use an environment-specific SNS topic supplied by Terraform variable for notification routing. Staging may initially use an operations email subscription; production routing must be explicitly approved before production enablement. + +Organic webhook silence is not meaningful for a low-volume application. A webhook-worker-owned database singleton schedules and leases one canary every 6 hours per environment, reserves shared provider rate, and sends to `delivered+webhook-canary-@resend.dev` with app/environment/canary tags. The correlated real event completes the canary row; lease/send/project failures emit the canary metric. A locally signed fixture tests only local verification and is not a registration/routing canary. + +### Rollout order + +1. Apply additive schema. +2. Deploy route, projector, tags, controls, and suppression checks with Phase 2 gates off; startup does not require a webhook secret while ingestion is off. +3. Verify API and both worker heartbeats. +4. Use the currently available Free-plan endpoint for registration bootstrap: create the staging endpoint, immediately disable it, write its secret to SSM, and reconcile the control record. +5. Redeploy with the secret parameter and ingestion gate enabled; verify secret readability with a local signed fixture. +6. Enable the Resend endpoint only after step 5 passes. +7. Test signed ingress, duplicate IDs, mutation, stale timestamp, oversize, and DB failure. +8. Enable projection and test provider-ID/tag correlation, out-of-order events, and unmatched reconciliation. +9. Enable suppression enforcement and run the suppression-boundary race test. +10. Exercise dead-letter, local replay, raw-payload purge, and rollback. +11. Use Resend test recipients (`delivered@resend.dev`, `bounced@resend.dev`, `complained@resend.dev`, and `suppressed@resend.dev`) for controlled staging evidence. +12. Repeat for production only after endpoint capacity, alarms, and rollback floor are confirmed. + +Every Phase 2 artifact includes `deploy/CAPABILITIES.json` with `webhook_ingest`, `webhook_projection`, `suppression_enforcement`, and schema capability versions. Deployment records the minimum compatible capability/release in webhook control when registration or suppression is enabled. Remote deploy and rollback validate the marker and database minimum before changing the `current` symlink. + +After registration, normal rollback may target only an artifact with compatible raw ingestion. After suppression activation, normal rollback may target only an artifact with compatible suppression enforcement. A Phase 1 artifact is rejected even if its files happen to exist. Ordered rollback is: pause projection, leave ingestion and suppression active, deploy a compatible artifact, verify exact-release API and webhook-worker heartbeat, then restore prior processing state. If no compatible artifact exists, break-glass atomically sets `EMAIL_SENDING_ENABLED=false` under the global provider-boundary fence, waits for all environment provider-boundary locks/in-flight calls to drain within a bounded timeout, verifies all worker and synchronous sends are fenced, disables the provider endpoint, and then stops API/workers. It does not run an unsafe Phase 1 binary. Additive schema is never reversed. + +## 9. Required acceptance tests + +### Ingress + +- valid exact-byte signature accepted +- one-byte mutation rejected +- stale/future timestamp rejected +- missing and malformed headers rejected +- wrong environment secret rejected +- payload over 256 KiB rejected +- duplicate `svix-id` returns 200 and creates one row +- database failure returns non-2xx and retry later succeeds + +### Projector + +- distinct workers claim distinct events +- expired lease reclaim and stale-token fencing +- provider-ID correlation +- webhook-before-provider-ID persistence correlation by delivery tag +- foreign-environment tag ignored safely +- duplicate and out-of-order events preserve all facts and precedence +- malformed known event dead-letters without corrupting delivery state +- only correlated sent/delivered/delayed/bounced/complained/email-suppressed evidence resolves uncertain dispatch; email.failed and account suppression events do not +- provider outcomes never alter dispatch progress arithmetic + +### Suppression + +- bounce, complaint, email suppression, and suppression.added activate suppression +- older add/remove cannot replace newer state +- suppression.removed remains locally blocked pending audited override +- suppression committed before the address boundary prevents provider invocation +- boundary crossed before suppression is recorded honestly +- pending/retry work cancels; stale workers cannot overwrite cancellation +- all-targets-suppressed launch creates no work +- staging and production/shared-account behavior is explicitly exercised + +### Operations/UI + +- dead-letter and audited replay +- raw payload purged while metadata/projection remains +- worker restart/reclaim +- processing pause does not stop durable ingestion +- suppression remains enforced during safe rollback +- separate provider summary is accessible and never calls accepted delivered + +## 10. Remaining decisions and external confirmations + +No policy confirmation remains before local implementation or staging registration. The user authorized the controlled staging test, and the Free plan's single currently unused endpoint is sufficient for it. + +Required before production enablement: + +1. Upgrade the shared Resend team from Free to a tier supporting at least two endpoints. +2. With staging registered, create the production endpoint disabled, then temporarily enable both to observe whether both same-team endpoints receive every matching event and validate environment-tag routing/account-level suppression behavior. +3. Finalize platform-operator principals authorized for replay and suppression override. Account emails and GitHub identities may be supplied later. + +Approved decisions: + +- Initial operational alert subscription: `bgarcia2324@gmail.com`. +- Alert recipients are Terraform/SNS configuration, allowing a different alert service later. +- Infrastructure alerts remain platform-operator notifications. Future survey-level delivery summaries may separately notify organization owners/site administrators after account email coverage and notification preferences are designed; shared infrastructure alarms must not disclose platform state to tenant owners. +- Raw webhook payload retention: 30 days. +- Controlled Resend staging routing/test-recipient activity: authorized. +- Resend endpoint strategy: staging on the current Free endpoint; upgrade before production to preserve separate staging/production endpoints. + +Recommended defaults if no contrary requirement is supplied: + +- existing Resend SDK verifier +- selected nine-event subscription +- 256 KiB raw body limit +- dedicated webhook worker +- three independent default-off gates +- 30-day raw payload retention +- 12 attempts/72 hours for processor failures +- seven-day unmatched horizon +- fail-closed suppression removal with platform-operator override +- separate staging/production endpoints and SSM secrets + +## 11. Primary sources + +- Resend webhook introduction and guarantees: https://resend.com/docs/webhooks/introduction +- Signature verification: https://resend.com/docs/webhooks/verify-webhooks-requests +- Retries and replay: https://resend.com/docs/webhooks/retries-and-replays +- Event types: https://resend.com/docs/webhooks/event-types +- Email bounce payload: https://resend.com/docs/webhooks/emails/bounced +- Email suppression payload: https://resend.com/docs/webhooks/emails/suppressed +- Suppression added: https://resend.com/docs/webhooks/suppressions/added +- Suppression removed: https://resend.com/docs/webhooks/suppressions/removed +- Tags and webhook tag projection: https://resend.com/docs/dashboard/emails/tags +- Webhook creation API: https://resend.com/docs/api-reference/webhooks/create-webhook +- Safe test recipients: https://resend.com/docs/knowledge-base/what-email-addresses-to-use-for-testing +- Resend endpoint limits/data retention: https://resend.com/pricing +- Svix verification/timestamp behavior: https://www.svix.com/guides/receiving/receive-webhooks-with-javascript-nodejs/ diff --git a/docs/runbooks/resend-webhook-operations.md b/docs/runbooks/resend-webhook-operations.md new file mode 100644 index 0000000..fa234a6 --- /dev/null +++ b/docs/runbooks/resend-webhook-operations.md @@ -0,0 +1,132 @@ +# Resend webhook operations + +Phase 2 is additive. Provider acceptance remains distinct from provider delivery truth. Never disable ingestion before disabling/removing the provider endpoint, never roll back schema, and never use replay to send email. + +## Capabilities and controls + +Every artifact must contain `deploy/CAPABILITIES.json` with versioned `webhook_ingest`, `webhook_projection`, `suppression_enforcement`, and schema capabilities. Deploy and rollback reject artifacts without the marker and require fresh exact-deployment heartbeats from both workers. + +The controls are independent and default off: + +1. `RESEND_WEBHOOK_INGEST_ENABLED` is Terraform release configuration. Disabled returns `503`. After registration, disable the Resend endpoint before deploying this false. +2. `email_webhook_worker_control.processing_enabled` pauses projection while ingestion remains durable. Change it only with `set-webhook-processing.js`. +3. `email_suppression_control.enforcement_enabled` is a one-way normal-operations latch. Activate it only with `activate-suppression-enforcement.js`. +4. `email_sending_control.sending_enabled` fences all application-originated provider calls. Hosted rows start false. Use `set-email-sending.js` for controlled enable or break-glass stop. + +Read the current `control_revision` first. All mutating commands require that expected revision and record actor/reason audit data. + +## Staging registration bootstrap + +Production registration is blocked until the shared Resend team supports two endpoints. Use the currently available endpoint for staging first. + +1. Apply Terraform with `resend_webhook_ingest_enabled=false`. Confirm the SNS subscription and CloudWatch Agent/log groups. +2. Deploy the Phase 2 artifact and verify API, delivery-worker, and webhook-worker exact-deployment heartbeats. +3. Obtain the selected event-set hash without printing any secret: + + ```bash + cd /opt/service/current/api + node ../deploy/manage-resend-webhook.js event-hash + ``` + +4. Assume a separately authenticated platform-operator AWS identity with narrowly scoped SSM write permission; the EC2 runtime role is read-only for secrets. Run bootstrap with the canonical public webhook URL, stable shared provider scope, exact hash, and operator actor. The script snapshots all endpoint IDs before create, creates only the requested endpoint, immediately disables it, writes the signing secret directly to the environment-specific SSM SecureString, and reconciles the audited registration row. It never prints the signing secret. + + ```bash + node ../deploy/manage-resend-webhook.js bootstrap \ + --environment staging \ + --url https://staging.ona.api.bennetts.work/api/webhooks/resend \ + --account-scope network-survey-resend-team \ + --event-set-hash \ + --actor + ``` + + Rerun the same command after interruption. If more than one post-snapshot candidate matches, the script disables candidates and refuses to guess. + +5. Inspect non-secret state with the same arguments and the `status` command. +6. Apply Terraform with ingestion enabled and redeploy. Prove primary and previous-secret signed fixtures locally. The previous SSM parameter is optional and may be absent outside rotation. +7. Enable the provider endpoint only after signed ingress, duplicate, stale/future timestamp, oversize, and database-failure behavior pass: + + ```bash + REVISION=$(cat /opt/service/current/REVISION) + node ../deploy/manage-resend-webhook.js enable \ + --confirm --expected-release "$REVISION" + ``` + +8. Enable projection using the exact release/deployment and control revision: + + ```bash + REVISION=$(cat /opt/service/current/REVISION) + # DEPLOYMENT_ID is visible in the exact worker heartbeat row. + EXPECTED_RELEASE_REVISION="$REVISION" EXPECTED_DEPLOYMENT_ID= \ + node ../deploy/set-webhook-processing.js true controlled-staging-rollout + ``` + +9. Activate suppression only after boundary-race tests pass: + + ```bash + EXPECTED_RELEASE_REVISION="$REVISION" \ + node ../deploy/activate-suppression-enforcement.js controlled-staging-activation + ``` + +10. Enable all application-originated mail only after suppression is active and the exact release is healthy: + + ```bash + EXPECTED_RELEASE_REVISION="$REVISION" \ + node ../deploy/set-email-sending.js true controlled-staging-email-enable + ``` + +## Rotation + +Never print or export secret values in an interactive shell. + +Endpoint-count and provider-plan capacity must be confirmed first. Under the separate platform-operator AWS identity, run crash-recoverable rotation bootstrap with the same common options used for registration: + +```bash +node ../deploy/manage-resend-webhook.js rotate +``` + +The command copies primary to previous in process memory, creates/reconciles the replacement disabled, writes its secret to primary, and durably retains the old endpoint ID. Deploy and prove fixtures signed by both loaded secrets, then enable the replacement with its exact ID/release. Keep both endpoints and both secrets for at least 36 hours. Finally retire the exact old endpoint and redeploy without the previous secret: + +```bash +node ../deploy/manage-resend-webhook.js retire-previous --confirm +``` + +The retirement command disables the old endpoint and clears the previous SecureString; deletion of the already-disabled old endpoint can follow after provider-state verification. + +## Suppression override, replay, and endpoint removal + +A provider removal remains fail-closed. Override only the exact latest inactive cause version after an audited platform decision; the address is accepted as input but never printed: + +```bash +node ../deploy/override-email-suppression.js
+``` + +The command rejects provider-active causes, stale versions, and duplicate overrides. A newer adverse event automatically invalidates the override. + +Local replay resets only the original durable inbox event; it never sends email: + +```bash +node ../deploy/replay-webhook-event.js +``` + +Disable/delete require exact endpoint-ID confirmation. Delete refuses while inbox work is nonterminal unless a nonempty audited break-glass reason is provided: + +```bash +node ../deploy/manage-resend-webhook.js disable --confirm +node ../deploy/manage-resend-webhook.js delete --confirm +``` + +## Monitoring + +Confirm the SNS email subscription. Environment-only alarm dimensions cover heartbeat, oldest pending/retry, one-hour and 24-hour unmatched age, dead letters, invalid signatures, suppression reconciliation, payload purge, uncertain-plus-quota-disable, and the 18-hour provider canary. Missing gauge data breaches. Logs/EMF must never contain addresses, subjects, payloads, signing headers, or secrets. + +## Compatible rollback + +Normal order: + +1. Record whether processing is enabled, then disable it with the revision-fenced script. +2. Leave ingestion and suppression enforcement active. +3. Redeploy only an artifact whose capability marker passes. +4. Verify exact-release API and both worker heartbeats. +5. Restore the prior processing state against the new release/control revision. + +If no compatible artifact exists, do **not** start a Phase 1 binary. Use `set-email-sending.js false ` to acquire the global provider boundary and durably disable application mail, wait for in-flight provider calls to drain, verify every worker and synchronous sender is fenced, disable the provider endpoint, then stop API/workers. Additive schema remains in place. diff --git a/docs/runbooks/survey-email-worker.md b/docs/runbooks/survey-email-worker.md index 935aade..c9e9127 100644 --- a/docs/runbooks/survey-email-worker.md +++ b/docs/runbooks/survey-email-worker.md @@ -1,5 +1,7 @@ # Survey email worker rollout and control +This runbook covers Phase 1 dispatch. After Phase 2 webhook registration or suppression activation, `docs/runbooks/resend-webhook-operations.md` supplies the stricter capability-aware deploy and rollback procedure and takes precedence. + Durable launch rows may be created only when `SURVEY_DELIVERY_V2_ENABLED=true`. Provider dispatch is independently controlled by `email_worker_control.claiming_enabled`; migrations seed hosted environments with claiming disabled. ## Deployment order @@ -43,7 +45,7 @@ Use `prod` only on the production instance. Never update the control row manuall ## Rollback -The rollback workflow validates the target artifact before disabling claiming. It refuses pre-lifecycle artifacts. If activation fails, it restores the prior symlink/processes and attempts to re-enable the prior revision. If automatic re-enable cannot verify a fresh heartbeat, claiming stays disabled for safety and must be restored with the command above. +Before Phase 2 activation, the rollback workflow validates the target artifact before disabling claiming. Once webhook registration or suppression is active, only a capability-compatible artifact is allowed: pause projection, keep ingestion and suppression active, verify both worker heartbeats, and follow `resend-webhook-operations.md`. A Phase 1 artifact must never be restored after that floor is raised. ## Ambiguous provider calls diff --git a/scripts/ci/lifecycle-worker-smoke.js b/scripts/ci/lifecycle-worker-smoke.js index 43a4823..8faed6f 100644 --- a/scripts/ci/lifecycle-worker-smoke.js +++ b/scripts/ci/lifecycle-worker-smoke.js @@ -72,9 +72,10 @@ const pool = new Pool({ await boundaryBlocker.query(`SELECT pg_advisory_unlock(hashtextextended($1,0))`, [`survey-provider-boundary:${process.env.SURVEY_ID}`]); boundaryBlocker.release(); await providerStarted; + assert.equal(closeResolved,false,'close must remain fenced for the full provider invocation'); + acceptProviderRequest({ id: 'ci-provider-message-id' }); const closed = await closeRequest; assert.equal(closed.lifecycleStatus, 'closed'); - acceptProviderRequest({ id: 'ci-provider-message-id' }); await processing; const delivery = (await pool.query( @@ -86,7 +87,8 @@ const pool = new Pool({ [process.env.SURVEY_ID] )).rows[0]; assert.equal(delivery.status, 'accepted'); - assert.ok(delivery.cancellation_requested_at); + // Close waited for the provider invocation. Depending on finalizer scheduling, + // it either observed an accepted delivery or recorded a cancellation request. assert.equal(delivery.provider_message_id, 'ci-provider-message-id'); assert.ok(attempt.provider_started_at); assert.equal(attempt.outcome, 'accepted'); diff --git a/scripts/ci/webhook-worker-smoke.js b/scripts/ci/webhook-worker-smoke.js new file mode 100644 index 0000000..f73a575 --- /dev/null +++ b/scripts/ci/webhook-worker-smoke.js @@ -0,0 +1,156 @@ +'use strict'; + +const assert = require('node:assert/strict'); +const crypto = require('node:crypto'); +const path = require('node:path'); +const { createRequire } = require('node:module'); + +const apiRequire = createRequire(path.resolve(process.cwd(), 'api/package.json')); +const { Pool } = apiRequire('pg'); +const { Webhook } = apiRequire('standardwebhooks'); +const lifecycle = require('../../api/lifecycle'); +const { DeliveryWorker } = require('../../api/email-worker'); +const { ResendWebhookIngress } = require('../../api/webhooks'); +const { WebhookWorker, effectiveProviderOutcome } = require('../../api/webhook-worker'); + +const pool = new Pool({ + user: process.env.DB_USER, + password: process.env.DB_PASSWORD, + host: process.env.DB_HOST, + port: Number(process.env.DB_PORT), + database: process.env.DB_NAME, +}); +const environment = 'test'; +const providerAccountScope = 'ci-shared-resend-team'; +const secret = `whsec_${Buffer.from('ci-webhook-secret-ci-webhook-secret').toString('base64')}`; +const signer = new Webhook(secret); +let eventSequence = 0; + +function signedEvent(type, data, createdAt = new Date()) { + eventSequence += 1; + const id = `msg_ci_${String(eventSequence).padStart(4, '0')}`; + const payload = JSON.stringify({ type, created_at: createdAt.toISOString(), data }); + return { + rawBody: Buffer.from(payload), + headers: { + 'svix-id': id, + 'svix-timestamp': String(Math.floor(createdAt.getTime() / 1000)), + 'svix-signature': signer.sign(id, createdAt, payload), + }, + }; +} + +const sleep = (milliseconds) => new Promise((resolve) => setTimeout(resolve, milliseconds)); +async function waitForAdvisoryWaiters(minimum) { + for (let attempt = 0; attempt < 50; attempt += 1) { + const count = Number((await pool.query(`SELECT count(*)::int AS count FROM pg_locks WHERE locktype='advisory' AND NOT granted`)).rows[0].count); + if (count >= minimum) return; + await sleep(20); + } + throw new Error(`Timed out waiting for ${minimum} advisory lock waiters`); +} + +(async () => { + const actor = (await pool.query("SELECT id FROM users WHERE username='ci-smoke'")).rows[0]; + assert.ok(actor, 'API smoke user must exist'); + const survey = (await pool.query(`INSERT INTO survey(name,title,creation_date,questions,organization_id,lifecycle_status) + SELECT 'CIWebhookSurvey','Webhook',now(),'{"elements":[{"type":"text","name":"question_1"}]}'::jsonb,organization_id,'draft' + FROM survey WHERE name='CISmokeSurvey' RETURNING *`)).rows[0]; + await pool.query(`INSERT INTO respondent(name,contact_info,survey_name,survey_id,can_respond,uuid,lang,email_sent) + VALUES('Webhook Person','webhook-person@example.test',$1,$2,true,'ci-webhook-person-token','English',false)`, [survey.name,survey.id]); + await pool.query(`INSERT INTO email(survey_name,survey_id,lang,text) VALUES($1,$2,'English','Webhook delivery truth')`, [survey.name,survey.id]); + await pool.query(`UPDATE email_worker_control SET claiming_enabled=true,minimum_release='' WHERE environment=$1`, [environment]); + await pool.query(`UPDATE email_sending_control SET sending_enabled=true,minimum_release='' WHERE environment=$1`, [environment]); + await pool.query(`INSERT INTO email_worker_heartbeats(environment,worker_instance,release_revision,enabled,claiming,heartbeat_at) + VALUES($1,'ci-phase2','local',true,true,now()) ON CONFLICT(environment,worker_instance) DO UPDATE SET heartbeat_at=now(),enabled=true,claiming=true`, [environment]); + const launch = await lifecycle.launchSurvey(pool, actor, survey.id, { + kind: 'initial', idempotencyKey: '77777777-7777-4777-8777-777777777777', + }, { + NODE_ENV:'test', EMAIL_WORKER_ENV:environment, SURVEY_URL:process.env.SURVEY_URL, + RESEND_API_KEY:'ci-key', SURVEY_DELIVERY_V2_ENABLED:'true', RESEND_PROVIDER_ACCOUNT_SCOPE:providerAccountScope, + }); + const delivery = (await pool.query('SELECT * FROM survey_email_deliveries WHERE launch_id=$1', [launch.id])).rows[0]; + assert.ok(delivery); + + await pool.query(`UPDATE email_webhook_worker_control SET claiming_enabled=true,processing_enabled=true,minimum_release='' WHERE environment=$1`, [environment]); + await pool.query(`UPDATE email_suppression_control SET enforcement_enabled=true,activated_at=now(),activated_by_actor='ci',activation_release='local',minimum_release='' WHERE environment=$1`, [environment]); + + const ingress = new ResendWebhookIngress({ pool, env:{ + EMAIL_WORKER_ENV:environment, RESEND_PROVIDER_ACCOUNT_SCOPE:providerAccountScope, + RESEND_WEBHOOK_INGEST_ENABLED:'true', RESEND_WEBHOOK_SECRET:secret, + }}); + const webhookWorker = new WebhookWorker({ pool, env:{ + EMAIL_WORKER_ENV:environment, RESEND_PROVIDER_ACCOUNT_SCOPE:providerAccountScope, RELEASE_REVISION:'local', + }, instanceId:'ci-webhook-worker' }); + let providerCalls = 0; + const deliveryWorker = new DeliveryWorker({ + pool, + provider:{ send:async()=>{ providerCalls += 1; return { id:'should-not-send' }; } }, + env:{ NODE_ENV:'test',EMAIL_WORKER_ENV:environment,EMAIL_RATE_BUDGET_ENV:environment,RELEASE_REVISION:'local', + SURVEY_URL:process.env.SURVEY_URL,EMAIL_RATE_PER_SECOND:'10',RESEND_PROVIDER_ACCOUNT_SCOPE:providerAccountScope }, + instanceId:'ci-delivery-worker', + }); + + const suppression = signedEvent('suppression.added', { + id:'supp_ci_1', email:delivery.to_address, origin:'manual', source_id:null, created_at:new Date().toISOString(), + }); + const inserted = await ingress.ingest(suppression.rawBody, suppression.headers); + assert.equal(inserted.duplicate, false); + const duplicate = await ingress.ingest(suppression.rawBody, suppression.headers); + assert.equal(duplicate.duplicate, true, 'same svix-id must be acknowledged once'); + + await pool.query(`UPDATE survey_email_deliveries SET status='cancelled',last_error_code='ci_phase2_isolation' WHERE id<>$1 AND status IN ('pending','retry_wait')`, [delivery.id]); + const claimed = await deliveryWorker.claim(); + assert.equal(claimed.id,delivery.id); + const blocker = await pool.connect(); + const addressKey = `email-suppression-boundary:${providerAccountScope}:${delivery.to_address}`; + await blocker.query('SELECT pg_advisory_lock(hashtextextended($1,0))', [addressKey]); + const projecting = webhookWorker.processOne(); + await waitForAdvisoryWaiters(1); + assert.equal(providerCalls, 0); + await blocker.query('SELECT pg_advisory_unlock(hashtextextended($1,0))', [addressKey]); + blocker.release(); + const projectionResult = await projecting; + assert.equal(projectionResult,true); + const suppressionCount = Number((await pool.query(`SELECT count(*)::int AS count FROM email_suppressions WHERE provider_account_scope=$1 AND normalized_address=$2 AND (provider_active OR locally_overridden_at IS NULL)`,[providerAccountScope,delivery.to_address])).rows[0].count); + assert.equal(suppressionCount,1,'suppression must commit before delivery final check'); + const sending = deliveryWorker.startProviderRequest(claimed); + await sending; + assert.equal(providerCalls, 0, 'suppression committed first must prevent provider invocation'); + const cancelled = (await pool.query('SELECT status,provider_suppressed_at FROM survey_email_deliveries WHERE id=$1', [delivery.id])).rows[0]; + assert.equal(cancelled.status, 'cancelled'); + assert.ok(cancelled.provider_suppressed_at); + + await pool.query(`UPDATE survey_email_deliveries SET status='uncertain',provider_message_id='ci-phase2-provider-id',dispatch_failed_at=now(),provider_suppressed_at=NULL WHERE id=$1`, [delivery.id]); + const deliveredEvent = signedEvent('email.delivered', { + email_id:'ci-phase2-provider-id', to:[delivery.to_address], + tags:{ app:'network_survey',environment,delivery_id:delivery.id }, + }); + await ingress.ingest(deliveredEvent.rawBody,deliveredEvent.headers); + await webhookWorker.processOne(); + const projected = (await pool.query(`SELECT d.*,r.email_sent FROM survey_email_deliveries d JOIN respondent r ON r.respondent_id=d.respondent_id WHERE d.id=$1`, [delivery.id])).rows[0]; + assert.equal(projected.status,'accepted'); + assert.equal(projected.dispatch_failed_at,null); + assert.ok(projected.dispatch_accepted_at); + assert.ok(projected.provider_delivered_at); + assert.equal(projected.email_sent,true); + assert.equal(effectiveProviderOutcome(projected),'delivered'); + + const complainedEvent = signedEvent('email.complained', { + email_id:'ci-phase2-provider-id', to:[delivery.to_address], + tags:{ app:'network_survey',environment,delivery_id:delivery.id }, + }); + await ingress.ingest(complainedEvent.rawBody,complainedEvent.headers); + await webhookWorker.processOne(); + const complained = (await pool.query('SELECT * FROM survey_email_deliveries WHERE id=$1',[delivery.id])).rows[0]; + assert.ok(complained.provider_complained_at); + assert.equal(effectiveProviderOutcome(complained),'complained'); + assert.equal((await pool.query(`SELECT count(*)::int AS count FROM email_webhook_events WHERE provider_account_scope=$1`,[providerAccountScope])).rows[0].count,3); + + await pool.end(); + console.log('Phase 2 webhook PostgreSQL smoke passed'); +})().catch(async (error) => { + console.error(error); + await pool.end().catch(() => {}); + process.exit(1); +}); diff --git a/scripts/deploy/CAPABILITIES.json b/scripts/deploy/CAPABILITIES.json new file mode 100644 index 0000000..a127141 --- /dev/null +++ b/scripts/deploy/CAPABILITIES.json @@ -0,0 +1,10 @@ +{ + "format_version": 1, + "webhook_ingest": 1, + "webhook_projection": 1, + "suppression_enforcement": 1, + "schema": { + "email_delivery": 6, + "webhook_delivery_truth": 1 + } +} diff --git a/scripts/deploy/activate-suppression-enforcement.js b/scripts/deploy/activate-suppression-enforcement.js new file mode 100644 index 0000000..d98659c --- /dev/null +++ b/scripts/deploy/activate-suppression-enforcement.js @@ -0,0 +1,17 @@ +'use strict'; + +const path = require('path'); +const fs = require('fs'); +const crypto = require('crypto'); +require(path.join(process.cwd(), 'node_modules/dotenv')).config({ path: path.join(process.cwd(), '.env.prod') }); +const { Pool } = require(path.join(process.cwd(), 'node_modules/pg')); +const [revisionArg, actorArg, ...reasonParts] = process.argv.slice(2); +if (!/^\d+$/.test(revisionArg || '') || !actorArg || !reasonParts.length) throw new Error('usage: activate-suppression-enforcement.js '); +const environment = process.env.EMAIL_WORKER_ENV; +const release = process.env.EXPECTED_RELEASE_REVISION; +if (!['staging', 'prod'].includes(environment) || !release) throw new Error('EMAIL_WORKER_ENV and EXPECTED_RELEASE_REVISION are required'); +const expected = Number(revisionArg), actor = actorArg.slice(0, 255), reason = reasonParts.join(' ').slice(0, 500); +const artifactRelease=fs.readFileSync(path.join(process.cwd(),'..','REVISION'),'utf8').trim(); +if(artifactRelease!==release)throw new Error('EXPECTED_RELEASE_REVISION does not match the current artifact'); +const pool = new Pool({ user:process.env.DB_USER,password:process.env.DB_PASSWORD,host:process.env.DB_HOST,port:process.env.DB_PORT,database:process.env.DB_NAME||'ONA',ssl:process.env.DB_SSL==='true'?{ca:process.env.DB_SSL_CA?fs.readFileSync(process.env.DB_SSL_CA,'utf8'):undefined,rejectUnauthorized:Boolean(process.env.DB_SSL_CA)}:undefined }); +(async()=>{const c=await pool.connect();try{const healthy=await c.query(`SELECT (EXISTS(SELECT 1 FROM email_worker_heartbeats WHERE environment=$1 AND release_revision=$2 AND enabled=true AND heartbeat_at>now()-interval '45 seconds') AND EXISTS(SELECT 1 FROM email_webhook_worker_heartbeats WHERE environment=$1 AND release_revision=$2 AND enabled=true AND heartbeat_at>now()-interval '45 seconds')) AS ok`,[environment,release]);if(!healthy.rows[0]?.ok)throw new Error('exact-release delivery and webhook worker heartbeats must be fresh before activation');await c.query('BEGIN');await c.query('SELECT pg_advisory_xact_lock(hashtextextended($1,0))',[`email-provider-boundary:${environment}`]);const before=(await c.query('SELECT * FROM email_suppression_control WHERE environment=$1 FOR UPDATE',[environment])).rows[0];if(!before)throw new Error('suppression control row not found');if(Number(before.control_revision)!==expected)throw new Error(`control revision changed; current revision is ${before.control_revision}`);if(before.enforcement_enabled){throw new Error('suppression enforcement is already active; it cannot be cleared by normal tooling');}const next=expected+1;const after=(await c.query(`UPDATE email_suppression_control SET enforcement_enabled=true,activated_at=COALESCE(activated_at,now()),activated_by_actor=COALESCE(activated_by_actor,$3),activation_release=COALESCE(activation_release,$4),minimum_release=$4,control_revision=$5,updated_at=now(),updated_by_actor=$3,reason=$6 WHERE environment=$1 AND control_revision=$2 RETURNING *`,[environment,expected,actor,release,next,reason])).rows[0];if(!after)throw new Error('stale suppression control revision');await c.query(`INSERT INTO email_control_audit(environment,control_name,control_revision,operation_id,previous_value,new_value,actor,reason) VALUES($1,'suppression_enforcement',$2,$3,$4,$5,$6,$7)`,[environment,next,crypto.randomUUID(),before,after,actor,reason]);await c.query('COMMIT');console.log(`suppression enforcement activated for ${environment}; control revision ${next}`);}catch(e){await c.query('ROLLBACK').catch(()=>{});throw e;}finally{c.release();await pool.end();}})().catch(e=>{console.error(e.message);process.exit(1);}); diff --git a/scripts/deploy/ecosystem.config.js b/scripts/deploy/ecosystem.config.js index cc5a6b4..4e5561c 100644 --- a/scripts/deploy/ecosystem.config.js +++ b/scripts/deploy/ecosystem.config.js @@ -7,6 +7,7 @@ module.exports = { env: { NODE_ENV: 'prod', RELEASE_REVISION: process.env.RELEASE_REVISION, + DEPLOYMENT_ID: process.env.DEPLOYMENT_ID, EMAIL_WORKER_ENV: process.env.EMAIL_WORKER_ENV, }, kill_timeout: 10000, @@ -18,6 +19,20 @@ module.exports = { env: { NODE_ENV: 'prod', RELEASE_REVISION: process.env.RELEASE_REVISION, + DEPLOYMENT_ID: process.env.DEPLOYMENT_ID, + EMAIL_WORKER_ENV: process.env.EMAIL_WORKER_ENV, + }, + kill_timeout: 30000, + }, + { + name: 'ona-email-webhook-worker', + script: 'webhook-worker.js', + cwd: '/opt/service/current/api', + env: { + NODE_ENV: 'prod', + APP_ENV: process.env.EMAIL_WORKER_ENV, + RELEASE_REVISION: process.env.RELEASE_REVISION, + DEPLOYMENT_ID: process.env.DEPLOYMENT_ID, EMAIL_WORKER_ENV: process.env.EMAIL_WORKER_ENV, }, kill_timeout: 30000, diff --git a/scripts/deploy/manage-resend-webhook.js b/scripts/deploy/manage-resend-webhook.js new file mode 100644 index 0000000..daff7f5 --- /dev/null +++ b/scripts/deploy/manage-resend-webhook.js @@ -0,0 +1,52 @@ +'use strict'; + +const path=require('path'),fs=require('fs'),crypto=require('crypto'),{spawnSync}=require('child_process'); +require(path.join(process.cwd(),'node_modules/dotenv')).config({path:path.join(process.cwd(),'.env.prod')}); +const{Pool}=require(path.join(process.cwd(),'node_modules/pg')); +const EVENTS=['email.sent','email.delivered','email.delivery_delayed','email.bounced','email.complained','email.failed','email.suppressed','suppression.added','suppression.removed']; +const EVENT_HASH=crypto.createHash('sha256').update(EVENTS.slice().sort().join('\n')).digest('hex'); +const command=process.argv[2];const options={};for(let i=3;i --environment --url --account-scope --event-set-hash --actor [--confirm ]'); +const environment=options.environment,actor=(options.actor||'').slice(0,255),scope=options['account-scope'],expectedHash=options['event-set-hash'],url=options.url; +if(!['staging','prod'].includes(environment)||!actor||!scope||!/^[-A-Za-z0-9_]{1,128}$/.test(scope)||expectedHash!==EVENT_HASH)throw new Error('environment, actor, stable account scope, and exact selected event-set hash are required'); +if(process.env.EMAIL_WORKER_ENV!==environment)throw new Error('requested environment must exactly match EMAIL_WORKER_ENV loaded on this host'); +let parsedUrl;try{parsedUrl=new URL(url);}catch(_){throw new Error('a valid expected public URL is required');}if(parsedUrl.protocol!=='https:'||parsedUrl.toString()!==url)throw new Error('expected public URL must be canonical HTTPS'); +const apiKey=process.env.RESEND_API_KEY;if(!apiKey)throw new Error('RESEND_API_KEY must be loaded in process memory'); +const secretParameter=process.env.RESEND_WEBHOOK_SECRET_PARAMETER,previousSecretParameter=process.env.RESEND_WEBHOOK_PREVIOUS_SECRET_PARAMETER;if(['bootstrap','rotate','retire-previous'].includes(command)&&!secretParameter)throw new Error('RESEND_WEBHOOK_SECRET_PARAMETER is required');if(['rotate','retire-previous'].includes(command)&&!previousSecretParameter)throw new Error('RESEND_WEBHOOK_PREVIOUS_SECRET_PARAMETER is required'); +const urlHash=crypto.createHash('sha256').update(url).digest('hex'); +const pool=new Pool({user:process.env.DB_USER,password:process.env.DB_PASSWORD,host:process.env.DB_HOST,port:process.env.DB_PORT,database:process.env.DB_NAME||'ONA',ssl:process.env.DB_SSL==='true'?{ca:process.env.DB_SSL_CA?fs.readFileSync(process.env.DB_SSL_CA,'utf8'):undefined,rejectUnauthorized:Boolean(process.env.DB_SSL_CA)}:undefined}); +async function provider(method,route,body){const response=await fetch(`https://api.resend.com${route}`,{method,headers:{Authorization:`Bearer ${apiKey}`,'Content-Type':'application/json'},body:body===undefined?undefined:JSON.stringify(body)});const text=await response.text();if(!response.ok)throw new Error(`Resend ${method} ${route} failed with HTTP ${response.status}`);return text?JSON.parse(text):{};} +function items(body){const value=body.data??body;return Array.isArray(value)?value:(Array.isArray(value?.data)?value.data:[]);}function endpointUrl(e){return e.endpoint||e.endpoint_url||e.url;}function endpointEvents(e){return (e.events||[]).slice().sort();}function matches(e){return endpointUrl(e)===url&&JSON.stringify(endpointEvents(e))===JSON.stringify(EVENTS.slice().sort());} +function putSecret(value,name=secretParameter){if(typeof value!=='string'||!value)throw new Error('provider response did not include a signing secret');const input=JSON.stringify({Name:name,Type:'SecureString',Overwrite:true,Value:value});const result=spawnSync('aws',['ssm','put-parameter','--cli-input-json','file:///dev/stdin','--query','Version','--output','text'],{input,encoding:'utf8',stdio:['pipe','pipe','pipe']});if(result.status!==0)throw new Error('failed to write webhook secret to SSM');const version=Number(result.stdout.trim());if(!Number.isSafeInteger(version)||version<1)throw new Error('SSM returned an invalid parameter version');return version;} +function getSecret(name){const r=spawnSync('aws',['ssm','get-parameter','--name',name,'--with-decryption','--query','Parameter.Value','--output','text'],{encoding:'utf8',stdio:['ignore','pipe','pipe']});if(r.status!==0||!r.stdout.trim())throw new Error('failed to read webhook secret from SSM');return r.stdout.trim();} +function deleteSecret(name){const r=spawnSync('aws',['ssm','delete-parameter','--name',name],{encoding:'utf8',stdio:['ignore','pipe','pipe']});if(r.status!==0&&!String(r.stderr).includes('ParameterNotFound'))throw new Error('failed to clear previous webhook secret from SSM');} +async function audit(client,before,after,reason){await client.query(`INSERT INTO email_control_audit(environment,control_name,control_revision,operation_id,previous_value,new_value,actor,reason) VALUES($1,'webhook_registration',$2,$3,$4,$5,$6,$7)`,[environment,after.control_revision,after.operation_id,before,after,actor,reason.slice(0,500)]);} +async function bootstrap(){const listed=items(await provider('GET','/webhooks'));let control=(await pool.query('SELECT * FROM email_webhook_registration_control WHERE environment=$1',[environment])).rows[0];if(!control){const operationId=crypto.randomUUID();const beforeIds=listed.map(e=>e.id).filter(Boolean);control=(await pool.query(`INSERT INTO email_webhook_registration_control(environment,provider_account_scope,event_set_hash,operation_id,pre_operation_endpoint_ids,endpoint_url_hash,updated_by_actor,reason,endpoint_state) VALUES($1,$2,$3,$4,$5,$6,$7,'bootstrap snapshot recorded','recovery_required') RETURNING *`,[environment,scope,EVENT_HASH,operationId,JSON.stringify(beforeIds),urlHash,actor])).rows[0];} +if(control.provider_account_scope!==scope||control.event_set_hash!==EVENT_HASH||control.endpoint_url_hash!==urlHash)throw new Error('existing registration control does not match the required account scope, URL hash, and event set'); +let endpoint=control.endpoint_id?listed.find(e=>e.id===control.endpoint_id):null;if(!endpoint){const beforeIds=new Set(control.pre_operation_endpoint_ids||[]);const candidates=listed.filter(e=>!beforeIds.has(e.id)&&matches(e));if(candidates.length>1){for(const candidate of candidates)await provider('PATCH',`/webhooks/${encodeURIComponent(candidate.id)}`,{status:'disabled'}).catch(()=>{});throw new Error('multiple post-snapshot candidates were disabled; audited operator selection is required');}endpoint=candidates[0];} +if(!endpoint){const created=await provider('POST','/webhooks',{endpoint:url,events:EVENTS});endpoint=created.data||created;if(!endpoint.id)throw new Error('provider create response did not include endpoint id');} +await provider('PATCH',`/webhooks/${encodeURIComponent(endpoint.id)}`,{status:'disabled'}); +let detail=await provider('GET',`/webhooks/${encodeURIComponent(endpoint.id)}`);detail=detail.data||detail;const signingSecret=detail.signing_secret||endpoint.signing_secret;let version=control.secret_parameter_version;if(signingSecret)version=putSecret(signingSecret);if(!version)throw new Error('signing secret could not be recovered; endpoint remains disabled');const client=await pool.connect();try{await client.query('BEGIN');const before=(await client.query('SELECT * FROM email_webhook_registration_control WHERE environment=$1 FOR UPDATE',[environment])).rows[0];const next=Number(before.control_revision)+1;const after=(await client.query(`UPDATE email_webhook_registration_control SET endpoint_id=$2,endpoint_state='bootstrap_disabled',secret_parameter_version=$3,ingestion_required=false,minimum_ingest_release='',control_revision=$4,updated_at=now(),updated_by_actor=$5,reason='disabled endpoint and SSM secret reconciled' WHERE environment=$1 RETURNING *`,[environment,endpoint.id,version,next,actor])).rows[0];await audit(client,before,after,'disabled endpoint and SSM secret reconciled');await client.query('COMMIT');console.log(`registration bootstrap reconciled for ${environment}; endpoint remains disabled; SSM version ${version}`);}catch(e){await client.query('ROLLBACK').catch(()=>{});throw e;}finally{client.release();}} +async function rotate(){ + let control=(await pool.query('SELECT * FROM email_webhook_registration_control WHERE environment=$1',[environment])).rows[0]; + if(!control?.endpoint_id||!control.ingestion_required)throw new Error('rotation requires a recorded endpoint and ingestion capability floor'); + if(control.previous_endpoint_id&&control.endpoint_state!=='rotation_required')throw new Error('a previous endpoint is already retained; retire it before another rotation'); + if(!control.previous_endpoint_id&&control.endpoint_state!=='enabled')throw new Error('a new rotation requires the current endpoint to be enabled'); + if(!control.previous_endpoint_id){const listed=items(await provider('GET','/webhooks'));const c=await pool.connect();try{await c.query('BEGIN');const before=(await c.query('SELECT * FROM email_webhook_registration_control WHERE environment=$1 FOR UPDATE',[environment])).rows[0];const next=Number(before.control_revision)+1;const after=(await c.query(`UPDATE email_webhook_registration_control SET previous_endpoint_id=endpoint_id,previous_endpoint_state=endpoint_state,endpoint_state='rotation_required',pre_operation_endpoint_ids=$2,control_revision=$3,updated_at=now(),updated_by_actor=$4,reason='rotation snapshot recorded' WHERE environment=$1 RETURNING *`,[environment,JSON.stringify(listed.map(e=>e.id).filter(Boolean)),next,actor])).rows[0];await audit(c,before,after,'overlapping-secret rotation snapshot recorded');await c.query('COMMIT');control=after;}catch(e){await c.query('ROLLBACK').catch(()=>{});throw e;}finally{c.release();}} + if(!control.previous_secret_parameter_version){const version=putSecret(getSecret(secretParameter),previousSecretParameter);const c=await pool.connect();try{await c.query('BEGIN');const before=(await c.query('SELECT * FROM email_webhook_registration_control WHERE environment=$1 FOR UPDATE',[environment])).rows[0];if(before.endpoint_state!=='rotation_required'||before.previous_secret_parameter_version)throw new Error('rotation state changed while copying previous secret');const next=Number(before.control_revision)+1;const after=(await c.query(`UPDATE email_webhook_registration_control SET previous_secret_parameter_version=$2,control_revision=$3,updated_at=now(),updated_by_actor=$4,reason='previous signing secret copied' WHERE environment=$1 RETURNING *`,[environment,version,next,actor])).rows[0];await audit(c,before,after,'previous signing secret copied for overlap');await c.query('COMMIT');control=after;}catch(e){await c.query('ROLLBACK').catch(()=>{});throw e;}finally{c.release();}} + const listed=items(await provider('GET','/webhooks'));const beforeIds=new Set(control.pre_operation_endpoint_ids||[]);let candidates=listed.filter(e=>!beforeIds.has(e.id)&&matches(e));if(candidates.length>1){for(const e of candidates)await provider('PATCH',`/webhooks/${encodeURIComponent(e.id)}`,{status:'disabled'}).catch(()=>{});throw new Error('multiple rotation candidates were disabled; operator reconciliation required');}let endpoint=candidates[0];if(!endpoint){const created=await provider('POST','/webhooks',{endpoint:url,events:EVENTS});endpoint=created.data||created;if(!endpoint.id)throw new Error('provider create response did not include endpoint id');}await provider('PATCH',`/webhooks/${encodeURIComponent(endpoint.id)}`,{status:'disabled'});let detail=await provider('GET',`/webhooks/${encodeURIComponent(endpoint.id)}`);detail=detail.data||detail;const signingSecret=detail.signing_secret||endpoint.signing_secret;if(!signingSecret)throw new Error('replacement signing secret could not be recovered; endpoint remains disabled');const version=putSecret(signingSecret); + const c=await pool.connect();try{await c.query('BEGIN');const before=(await c.query('SELECT * FROM email_webhook_registration_control WHERE environment=$1 FOR UPDATE',[environment])).rows[0];if(before.endpoint_state!=='rotation_required'||before.previous_endpoint_id!==control.previous_endpoint_id)throw new Error('rotation state changed before replacement reconciliation');const next=Number(before.control_revision)+1;const after=(await c.query(`UPDATE email_webhook_registration_control SET endpoint_id=$2,endpoint_state='rotation_disabled',secret_parameter_version=$3,control_revision=$4,updated_at=now(),updated_by_actor=$5,reason='replacement endpoint disabled and overlapping secrets stored' WHERE environment=$1 RETURNING *`,[environment,endpoint.id,version,next,actor])).rows[0];await audit(c,before,after,'replacement endpoint disabled and overlapping secrets stored');await c.query('COMMIT');console.log(`rotation replacement reconciled for ${environment}; replacement remains disabled`);}catch(e){await c.query('ROLLBACK').catch(()=>{});throw e;}finally{c.release();} +} +async function retirePrevious(){const c=await pool.connect();try{await c.query('BEGIN');const before=(await c.query('SELECT * FROM email_webhook_registration_control WHERE environment=$1 FOR UPDATE',[environment])).rows[0];if(!before?.previous_endpoint_id||options.confirm!==before.previous_endpoint_id)throw new Error('retire-previous requires --confirm with the retained previous endpoint id');if(before.endpoint_state!=='enabled')throw new Error('replacement endpoint must be enabled before retiring previous');await provider('PATCH',`/webhooks/${encodeURIComponent(before.previous_endpoint_id)}`,{status:'disabled'});deleteSecret(previousSecretParameter);const next=Number(before.control_revision)+1;const after=(await c.query(`UPDATE email_webhook_registration_control SET previous_endpoint_id=NULL,previous_endpoint_state=NULL,previous_secret_parameter_version=NULL,control_revision=$2,updated_at=now(),updated_by_actor=$3,reason='previous endpoint disabled and previous secret cleared' WHERE environment=$1 RETURNING *`,[environment,next,actor])).rows[0];await audit(c,before,after,'previous endpoint disabled and previous secret cleared after overlap');await c.query('COMMIT');console.log(`previous endpoint retired for ${environment}`);}catch(e){await c.query('ROLLBACK').catch(()=>{});throw e;}finally{c.release();}} +async function enable(){ + if(!options['expected-release'])throw new Error('enable requires --expected-release'); + const artifactRelease=fs.readFileSync(path.join(process.cwd(),'..','REVISION'),'utf8').trim();if(artifactRelease!==options['expected-release'])throw new Error('--expected-release does not match the current artifact'); + let endpointId; + const first=await pool.connect(); + try{await first.query('BEGIN');const before=(await first.query('SELECT * FROM email_webhook_registration_control WHERE environment=$1 FOR UPDATE',[environment])).rows[0];if(!before?.endpoint_id||options.confirm!==before.endpoint_id)throw new Error('enable requires --confirm with the recorded endpoint id');endpointId=before.endpoint_id;const next=Number(before.control_revision)+1;const after=(await first.query(`UPDATE email_webhook_registration_control SET endpoint_state='enable_pending',ingestion_required=true,minimum_ingest_release=$2,control_revision=$3,updated_at=now(),updated_by_actor=$4,reason='provider enable pending' WHERE environment=$1 RETURNING *`,[environment,options['expected-release'],next,actor])).rows[0];await audit(first,before,after,'rollback capability floor committed before provider enable');await first.query('COMMIT');}catch(e){await first.query('ROLLBACK').catch(()=>{});throw e;}finally{first.release();} + await provider('PATCH',`/webhooks/${encodeURIComponent(endpointId)}`,{status:'enabled'}); + const second=await pool.connect(); + try{await second.query('BEGIN');const before=(await second.query('SELECT * FROM email_webhook_registration_control WHERE environment=$1 FOR UPDATE',[environment])).rows[0];if(before.endpoint_id!==endpointId||!before.ingestion_required)throw new Error('registration changed while provider enable was pending');const next=Number(before.control_revision)+1;const after=(await second.query(`UPDATE email_webhook_registration_control SET endpoint_state='enabled',control_revision=$2,updated_at=now(),updated_by_actor=$3,reason='provider enable confirmed' WHERE environment=$1 RETURNING *`,[environment,next,actor])).rows[0];await audit(second,before,after,'provider enable confirmed after capability floor');await second.query('COMMIT');console.log(`registration enabled for ${environment}`);}catch(e){await second.query('ROLLBACK').catch(()=>{});throw e;}finally{second.release();} +} +async function mutate(action){const client=await pool.connect();try{await client.query('BEGIN');const before=(await client.query('SELECT * FROM email_webhook_registration_control WHERE environment=$1 FOR UPDATE',[environment])).rows[0];if(!before?.endpoint_id||options.confirm!==before.endpoint_id)throw new Error(`${action} requires --confirm with the recorded endpoint id`);if(action==='delete'){const pending=await client.query(`SELECT count(*)::int AS count FROM email_webhook_events WHERE receiving_environment=$1 AND status IN ('pending','leased','retry_wait','unmatched')`,[environment]);if(pending.rows[0].count>0&&!options['break-glass-reason'])throw new Error('refusing deletion while unprocessed inbox events remain');await provider('DELETE',`/webhooks/${encodeURIComponent(before.endpoint_id)}`);}else {if(action==='enable'&&!options['expected-release'])throw new Error('enable requires --expected-release');await provider('PATCH',`/webhooks/${encodeURIComponent(before.endpoint_id)}`,{status:action==='enable'?'enabled':'disabled'});}const next=Number(before.control_revision)+1,state=action==='delete'?'deleted':action==='enable'?'enabled':'disabled';const reason=options['break-glass-reason']||`${action} confirmed by operator`;const after=(await client.query(`UPDATE email_webhook_registration_control SET endpoint_state=$2,ingestion_required=$6,minimum_ingest_release=$7,control_revision=$3,updated_at=now(),updated_by_actor=$4,reason=$5 WHERE environment=$1 RETURNING *`,[environment,state,next,actor,String(reason).slice(0,500),action==='enable',action==='enable'?options['expected-release']:''])).rows[0];await audit(client,before,after,reason);await client.query('COMMIT');console.log(`registration ${state} for ${environment}`);}catch(e){await client.query('ROLLBACK').catch(()=>{});throw e;}finally{client.release();}} +(async()=>{let lockClient=null;try{if(command!=='status'){lockClient=await pool.connect();await lockClient.query('SELECT pg_advisory_lock(hashtextextended($1,0))',[`resend-webhook-registration:${environment}`]);}if(command==='bootstrap')await bootstrap();else if(command==='rotate')await rotate();else if(command==='retire-previous')await retirePrevious();else if(command==='status'){const control=(await pool.query('SELECT environment,provider_account_scope,endpoint_id,endpoint_state,previous_endpoint_id,previous_endpoint_state,event_set_hash,secret_parameter_version,previous_secret_parameter_version,ingestion_required,minimum_ingest_release,control_revision,updated_at FROM email_webhook_registration_control WHERE environment=$1',[environment])).rows[0];console.log(JSON.stringify(control||{environment,endpoint_state:'unregistered'},null,2));}else if(command==='enable')await enable();else await mutate(command);}finally{if(lockClient){await lockClient.query('SELECT pg_advisory_unlock(hashtextextended($1,0))',[`resend-webhook-registration:${environment}`]).catch(()=>{});lockClient.release();}}})().catch(e=>{console.error(e.message);process.exitCode=1;}).finally(()=>pool.end()); diff --git a/scripts/deploy/override-email-suppression.js b/scripts/deploy/override-email-suppression.js new file mode 100644 index 0000000..e968333 --- /dev/null +++ b/scripts/deploy/override-email-suppression.js @@ -0,0 +1,5 @@ +'use strict'; +const path=require('path'),fs=require('fs');require(path.join(process.cwd(),'node_modules/dotenv')).config({path:path.join(process.cwd(),'.env.prod')});const{Pool}=require(path.join(process.cwd(),'node_modules/pg')); +const [addressArg,reason,causeVersionArg,actorArg,...detailParts]=process.argv.slice(2);const address=String(addressArg||'').trim().toLowerCase();if(!address.includes('@')||!['permanent_bounce','complaint','provider_suppression'].includes(reason)||!/^\d+$/.test(causeVersionArg||'')||!actorArg||!detailParts.length)throw new Error('usage: override-email-suppression.js
');const environment=process.env.EMAIL_WORKER_ENV,scope=process.env.RESEND_PROVIDER_ACCOUNT_SCOPE;if(!['staging','prod'].includes(environment)||!scope)throw new Error('EMAIL_WORKER_ENV and RESEND_PROVIDER_ACCOUNT_SCOPE are required');const actor=actorArg.slice(0,255),detail=detailParts.join(' ').slice(0,500),version=Number(causeVersionArg); +const pool=new Pool({user:process.env.DB_USER,password:process.env.DB_PASSWORD,host:process.env.DB_HOST,port:process.env.DB_PORT,database:process.env.DB_NAME||'ONA',ssl:process.env.DB_SSL==='true'?{ca:process.env.DB_SSL_CA?fs.readFileSync(process.env.DB_SSL_CA,'utf8'):undefined,rejectUnauthorized:Boolean(process.env.DB_SSL_CA)}:undefined}); +(async()=>{const c=await pool.connect();try{await c.query('BEGIN');await c.query('SELECT pg_advisory_xact_lock(hashtextextended($1,0))',[`email-suppression-boundary:${scope}:${address}`]);const row=(await c.query(`SELECT * FROM email_suppressions WHERE provider_account_scope=$1 AND normalized_address=$2 AND reason=$3 FOR UPDATE`,[scope,address,reason])).rows[0];if(!row||Number(row.cause_version)!==version)throw new Error('suppression cause not found or cause version changed');if(row.provider_active)throw new Error('provider-active suppression cannot be locally overridden');if(row.override_cause_version)throw new Error('exact suppression cause version is already overridden');await c.query(`UPDATE email_suppressions SET locally_overridden_at=now(),override_cause_version=cause_version,locally_overridden_by_actor=$4,updated_at=now() WHERE provider_account_scope=$1 AND normalized_address=$2 AND reason=$3`,[scope,address,reason,actor]);await c.query(`INSERT INTO email_suppression_audit(provider_account_scope,normalized_address,reason,receiving_environment,action,cause_version,provider_active,occurrence_at,source_webhook_event_id,source_event_svix_id,provider_suppression_id,actor,detail) VALUES($1,$2,$3,$4,'local_override',$5,false,now(),$6,$7,$8,$9,jsonb_build_object('reason',$10::text))`,[scope,address,reason,environment,version,row.source_webhook_event_id,row.state_event_svix_id,row.provider_suppression_id,actor,detail]);await c.query('COMMIT');console.log(`suppression cause version ${version} overridden by audited platform operator`);}catch(e){await c.query('ROLLBACK').catch(()=>{});throw e;}finally{c.release();await pool.end();}})().catch(e=>{console.error(e.message);process.exit(1);}); diff --git a/scripts/deploy/remote-deploy.sh b/scripts/deploy/remote-deploy.sh index 3e7d779..563bdb8 100755 --- a/scripts/deploy/remote-deploy.sh +++ b/scripts/deploy/remote-deploy.sh @@ -16,11 +16,14 @@ SOURCE_DIR=${1:?usage: remote-deploy.sh } SERVICE_DIR=/opt/service PM2_APP=ona-api PM2_WORKER=ona-email-worker +PM2_WEBHOOK_WORKER=ona-email-webhook-worker source "$SERVICE_DIR/deploy.env" export AWS_DEFAULT_REGION REVISION=$(cat "$SOURCE_DIR/REVISION") +node "$SOURCE_DIR/deploy/validate-release-capabilities.js" "$SOURCE_DIR" +test -f "$SOURCE_DIR/api/webhook-worker.js" || { echo "Release lacks dedicated webhook worker" >&2; exit 1; } DEPLOYMENT_ID="${REVISION}-$(date +%s)-$$" PREVIOUS_RELEASE=$(readlink -f "$SERVICE_DIR/current" 2>/dev/null || true) RELEASE_DIR="$SERVICE_DIR/releases/$REVISION" @@ -83,6 +86,34 @@ echo "==> Resolving runtime secrets from SSM Parameter Store" append_secret_from_ssm DB_PASSWORD DB_PASSWORD_PARAMETER append_secret_from_ssm SESSION_SECRET SESSION_SECRET_PARAMETER append_secret_from_ssm RESEND_API_KEY RESEND_API_KEY_PARAMETER +if [ "$(get_env_value RESEND_WEBHOOK_INGEST_ENABLED)" = "true" ]; then + append_secret_from_ssm RESEND_WEBHOOK_SECRET RESEND_WEBHOOK_SECRET_PARAMETER +fi +PREVIOUS_WEBHOOK_PARAMETER=$(get_env_value RESEND_WEBHOOK_PREVIOUS_SECRET_PARAMETER) +if [ -n "$PREVIOUS_WEBHOOK_PARAMETER" ]; then + OPTIONAL_SSM_ERROR=$(mktemp) + set +e + PREVIOUS_WEBHOOK_SECRET=$(aws ssm get-parameter --name "$PREVIOUS_WEBHOOK_PARAMETER" --with-decryption --query 'Parameter.Value' --output text 2>"$OPTIONAL_SSM_ERROR") + OPTIONAL_SSM_STATUS=$? + set -e + if [ "$OPTIONAL_SSM_STATUS" -ne 0 ]; then + if grep -q 'ParameterNotFound' "$OPTIONAL_SSM_ERROR"; then + PREVIOUS_WEBHOOK_SECRET= + else + rm -f "$OPTIONAL_SSM_ERROR" + echo "Unable to read optional previous webhook secret parameter" >&2 + exit 1 + fi + fi + rm -f "$OPTIONAL_SSM_ERROR" + if [ -n "$PREVIOUS_WEBHOOK_SECRET" ] && [ "$PREVIOUS_WEBHOOK_SECRET" != "None" ]; then + ENV_KEY=RESEND_WEBHOOK_PREVIOUS_SECRET SECRET_VALUE="$PREVIOUS_WEBHOOK_SECRET" node - <<'NODE' >> "$RELEASE_DIR/api/.env.prod" +const value = process.env.SECRET_VALUE || ''; +process.stdout.write(`RESEND_WEBHOOK_PREVIOUS_SECRET=${JSON.stringify(value)}\n`); +NODE + fi + unset PREVIOUS_WEBHOOK_SECRET +fi WORKER_ENV=$(get_env_value EMAIL_WORKER_ENV) case "$WORKER_ENV" in staging|prod) ;; @@ -121,6 +152,11 @@ liquibase \ --searchPath="$RELEASE_DIR/db" \ update +# Registration/suppression activation can raise the rollback floor. Validate +# the artifact marker against durable controls before any process or symlink is +# changed. +node "$RELEASE_DIR/deploy/validate-release-capabilities.js" "$RELEASE_DIR" --database + if [ -n "$(get_env_value BOOTSTRAP_ADMIN_PASSWORD_PARAMETER)" ]; then echo "==> Ensuring bootstrap dashboard administrator" BOOTSTRAP_ADMIN_PASSWORD=$(get_secret_from_ssm BOOTSTRAP_ADMIN_PASSWORD_PARAMETER) @@ -137,6 +173,49 @@ if [ -n "$(get_env_value BOOTSTRAP_ADMIN_PASSWORD_PARAMETER)" ]; then unset BOOTSTRAP_ADMIN_PASSWORD fi +CLAIMING_WAS_ENABLED=false +WEBHOOK_PROCESSING_WAS_ENABLED=false +SENDING_WAS_ENABLED=false +SENDING_CONTROL_REVISION=0 +WEBHOOK_CONTROL_REVISION=0 +PREVIOUS_WEBHOOK_DEPLOYMENT_ID="" +restore_pre_activation_handoff() { + local status=$? + if [ "$status" -ne 0 ] && { [ "$CLAIMING_WAS_ENABLED" = true ] || [ "$SENDING_WAS_ENABLED" = true ] || [ "$WEBHOOK_PROCESSING_WAS_ENABLED" = true ]; } && [ -n "$PREVIOUS_RELEASE" ]; then + local previous_revision + previous_revision=$(cat "$PREVIOUS_RELEASE/REVISION" 2>/dev/null || true) + if [ -n "$previous_revision" ]; then + set +e + if [ "$CLAIMING_WAS_ENABLED" = true ]; then + (cd "$RELEASE_DIR/api" && EMAIL_WORKER_ENV="$WORKER_ENV" EXPECTED_RELEASE_REVISION="$previous_revision" NODE_ENV=prod node ../deploy/set-email-claiming.js true failed-pre-activation-handoff) + fi + if [ "$SENDING_WAS_ENABLED" = true ]; then + (cd "$RELEASE_DIR/api" && EMAIL_WORKER_ENV="$WORKER_ENV" EXPECTED_RELEASE_REVISION="$previous_revision" NODE_ENV=prod node ../deploy/set-email-sending.js true "$SENDING_CONTROL_REVISION" deploy failed-pre-activation-handoff) + fi + if [ "$WEBHOOK_PROCESSING_WAS_ENABLED" = true ] && [ -n "$PREVIOUS_WEBHOOK_DEPLOYMENT_ID" ]; then + (cd "$RELEASE_DIR/api" && EMAIL_WORKER_ENV="$WORKER_ENV" EXPECTED_RELEASE_REVISION="$previous_revision" EXPECTED_DEPLOYMENT_ID="$PREVIOUS_WEBHOOK_DEPLOYMENT_ID" NODE_ENV=prod node ../deploy/set-webhook-processing.js true "$WEBHOOK_CONTROL_REVISION" deploy failed-pre-activation-handoff) + fi + set -e + fi + fi +} +trap restore_pre_activation_handoff EXIT + +SENDING_CONTROL=$(cd "$RELEASE_DIR/api" && node - <<'NODE' +require('dotenv').config({path:'.env.prod'}); +const {Pool}=require('pg'); +const pool=new Pool({user:process.env.DB_USER,password:process.env.DB_PASSWORD,host:process.env.DB_HOST,port:process.env.DB_PORT,database:process.env.DB_NAME||'ONA',ssl:process.env.DB_SSL==='true'?{ca:process.env.DB_SSL_CA?require('fs').readFileSync(process.env.DB_SSL_CA,'utf8'):undefined,rejectUnauthorized:Boolean(process.env.DB_SSL_CA)}:undefined}); +pool.query('SELECT sending_enabled,control_revision FROM email_sending_control WHERE environment=$1',[process.env.EMAIL_WORKER_ENV]).then(r=>process.stdout.write(`${r.rows[0]?.sending_enabled?'true':'false'}:${r.rows[0]?.control_revision??0}`)).finally(()=>pool.end()); +NODE +) +SENDING_WAS_ENABLED=${SENDING_CONTROL%%:*} +SENDING_CONTROL_REVISION=${SENDING_CONTROL#*:} +if [ "$SENDING_WAS_ENABLED" = true ]; then + echo "==> Pausing all application email sends for release handoff" + (cd "$RELEASE_DIR/api" && EMAIL_WORKER_ENV="$WORKER_ENV" NODE_ENV=prod node ../deploy/set-email-sending.js false "$SENDING_CONTROL_REVISION" deploy release-handoff) + SENDING_CONTROL_REVISION=$((SENDING_CONTROL_REVISION + 1)) +fi + CLAIMING_WAS_ENABLED=$(cd "$RELEASE_DIR/api" && node - <<'NODE' require('dotenv').config({path:'.env.prod'}); const {Pool}=require('pg'); @@ -149,17 +228,52 @@ if [ "$CLAIMING_WAS_ENABLED" = true ]; then (cd "$RELEASE_DIR/api" && EMAIL_WORKER_ENV="$WORKER_ENV" NODE_ENV=prod node ../deploy/set-email-claiming.js false release-handoff) fi +WEBHOOK_CONTROL=$(cd "$RELEASE_DIR/api" && node - <<'NODE' +require('dotenv').config({path:'.env.prod'}); +const {Pool}=require('pg'); +const pool=new Pool({user:process.env.DB_USER,password:process.env.DB_PASSWORD,host:process.env.DB_HOST,port:process.env.DB_PORT,database:process.env.DB_NAME||'ONA',ssl:process.env.DB_SSL==='true'?{ca:process.env.DB_SSL_CA?require('fs').readFileSync(process.env.DB_SSL_CA,'utf8'):undefined,rejectUnauthorized:Boolean(process.env.DB_SSL_CA)}:undefined}); +pool.query(`SELECT c.processing_enabled,c.control_revision,(SELECT split_part(h.worker_instance,'/',1) FROM email_webhook_worker_heartbeats h WHERE h.environment=c.environment AND h.heartbeat_at>now()-interval '45 seconds' ORDER BY h.heartbeat_at DESC LIMIT 1) AS deployment_id FROM email_webhook_worker_control c WHERE c.environment=$1`,[process.env.EMAIL_WORKER_ENV]).then(r=>process.stdout.write(`${r.rows[0]?.processing_enabled?'true':'false'}:${r.rows[0]?.control_revision??0}:${r.rows[0]?.deployment_id??''}`)).finally(()=>pool.end()); +NODE +) +WEBHOOK_PROCESSING_WAS_ENABLED=${WEBHOOK_CONTROL%%:*} +WEBHOOK_CONTROL_REST=${WEBHOOK_CONTROL#*:} +WEBHOOK_CONTROL_REVISION=${WEBHOOK_CONTROL_REST%%:*} +PREVIOUS_WEBHOOK_DEPLOYMENT_ID=${WEBHOOK_CONTROL_REST#*:} +if [ "$WEBHOOK_PROCESSING_WAS_ENABLED" = true ]; then + echo "==> Pausing webhook projection for release handoff" + (cd "$RELEASE_DIR/api" && EMAIL_WORKER_ENV="$WORKER_ENV" NODE_ENV=prod node ../deploy/set-webhook-processing.js false "$WEBHOOK_CONTROL_REVISION" deploy release-handoff) + WEBHOOK_CONTROL_REVISION=$((WEBHOOK_CONTROL_REVISION + 1)) +fi + echo "==> Activating release" chown -R ubuntu:ubuntu "$RELEASE_DIR" ACTIVATED=false HANDOFF_REENABLED=false +WEBHOOK_HANDOFF_REENABLED=false +SENDING_HANDOFF_REENABLED=false restore_previous_release() { local status=$? if [ "$status" -ne 0 ] && [ -n "$PREVIOUS_RELEASE" ] && [ -d "$PREVIOUS_RELEASE" ]; then + echo "!! Validating previous release before automatic restore" >&2 + if ! node "$RELEASE_DIR/deploy/validate-release-capabilities.js" "$PREVIOUS_RELEASE" --database; then + echo "!! Previous release is below the active capability floor; refusing unsafe automatic restore" >&2 + set +e + run_pm2 stop "$PM2_APP" "$PM2_WORKER" "$PM2_WEBHOOK_WORKER" >/dev/null 2>&1 + set -e + return + fi echo "!! Restoring previous release $PREVIOUS_RELEASE" >&2 set +e if [ "$ACTIVATED" = true ]; then ln -sfn "$PREVIOUS_RELEASE" "$SERVICE_DIR/current"; fi REVISION=$(cat "$PREVIOUS_RELEASE/REVISION" 2>/dev/null || basename "$PREVIOUS_RELEASE") + if [ "$WEBHOOK_HANDOFF_REENABLED" = true ]; then + (cd "$RELEASE_DIR/api" && EMAIL_WORKER_ENV="$WORKER_ENV" NODE_ENV=prod node ../deploy/set-webhook-processing.js false "$WEBHOOK_CONTROL_REVISION" deploy failed-release-handoff) + WEBHOOK_CONTROL_REVISION=$((WEBHOOK_CONTROL_REVISION + 1)) + fi + if [ "$SENDING_HANDOFF_REENABLED" = true ]; then + (cd "$RELEASE_DIR/api" && EMAIL_WORKER_ENV="$WORKER_ENV" NODE_ENV=prod node ../deploy/set-email-sending.js false "$SENDING_CONTROL_REVISION" deploy failed-release-handoff) + SENDING_CONTROL_REVISION=$((SENDING_CONTROL_REVISION + 1)) + fi if [ -f "$PREVIOUS_RELEASE/deploy/set-email-claiming.js" ]; then (cd "$PREVIOUS_RELEASE/api" && EMAIL_WORKER_ENV="$WORKER_ENV" NODE_ENV=prod node ../deploy/set-email-claiming.js false failed-release-handoff) else @@ -173,11 +287,35 @@ restore_previous_release() { run_pm2 start "$PREVIOUS_RELEASE/api/server.js" --name "$PM2_APP" --cwd "$PREVIOUS_RELEASE/api" fi run_pm2 save + echo "!! Verifying restored API and fresh paused worker heartbeats before restoring traffic" >&2 + RESTORE_HEALTHY=false + for _ in $(seq 1 15); do + if curl -fsS http://localhost:3000/health >/dev/null 2>&1 && (cd "$PREVIOUS_RELEASE/api" && EXPECTED_REVISION="$REVISION" EXPECTED_WORKER_ENV="$WORKER_ENV" EXPECTED_DEPLOYMENT_ID="$DEPLOYMENT_ID" node - <<'NODE' +require('dotenv').config({path:'.env.prod'});const{Pool}=require('pg');const fs=require('fs');const p=new Pool({user:process.env.DB_USER,password:process.env.DB_PASSWORD,host:process.env.DB_HOST,port:process.env.DB_PORT,database:process.env.DB_NAME||'ONA',ssl:process.env.DB_SSL==='true'?{ca:process.env.DB_SSL_CA?fs.readFileSync(process.env.DB_SSL_CA,'utf8'):undefined,rejectUnauthorized:Boolean(process.env.DB_SSL_CA)}:undefined});Promise.all([p.query(`SELECT 1 FROM email_worker_heartbeats WHERE environment=$1 AND release_revision=$2 AND worker_instance LIKE $3||'/%' AND heartbeat_at>now()-interval '20 seconds'`,[process.env.EXPECTED_WORKER_ENV,process.env.EXPECTED_REVISION,process.env.EXPECTED_DEPLOYMENT_ID]),p.query(`SELECT 1 FROM email_webhook_worker_heartbeats WHERE environment=$1 AND release_revision=$2 AND worker_instance LIKE $3||'/%' AND heartbeat_at>now()-interval '20 seconds'`,[process.env.EXPECTED_WORKER_ENV,process.env.EXPECTED_REVISION,process.env.EXPECTED_DEPLOYMENT_ID])]).then(r=>process.exitCode=r.every(x=>x.rowCount)?0:1).catch(()=>process.exitCode=1).finally(()=>p.end()); +NODE + ); then RESTORE_HEALTHY=true; break; fi + sleep 2 + done + if [ "$RESTORE_HEALTHY" != true ]; then + echo "!! Restored release did not become healthy; leaving all outbound controls paused" >&2 + run_pm2 stop "$PM2_APP" "$PM2_WORKER" "$PM2_WEBHOOK_WORKER" >/dev/null 2>&1 || true + set -e + return + fi + if [ "$SENDING_WAS_ENABLED" = true ] && [ -f "$PREVIOUS_RELEASE/deploy/set-email-sending.js" ]; then + (cd "$PREVIOUS_RELEASE/api" && EMAIL_WORKER_ENV="$WORKER_ENV" EXPECTED_RELEASE_REVISION="$REVISION" NODE_ENV=prod node ../deploy/set-email-sending.js true "$SENDING_CONTROL_REVISION" deploy failed-release-restore) + SENDING_CONTROL_REVISION=$((SENDING_CONTROL_REVISION + 1)) + fi if [ "$CLAIMING_WAS_ENABLED" = true ]; then if [ -f "$PREVIOUS_RELEASE/deploy/set-email-claiming.js" ]; then (cd "$PREVIOUS_RELEASE/api" && EMAIL_WORKER_ENV="$WORKER_ENV" EXPECTED_RELEASE_REVISION="$REVISION" NODE_ENV=prod node ../deploy/set-email-claiming.js true failed-release-restore) fi fi + if [ "$WEBHOOK_PROCESSING_WAS_ENABLED" = true ] && [ -f "$PREVIOUS_RELEASE/deploy/set-webhook-processing.js" ]; then + sleep 2 + (cd "$PREVIOUS_RELEASE/api" && EMAIL_WORKER_ENV="$WORKER_ENV" EXPECTED_RELEASE_REVISION="$REVISION" EXPECTED_DEPLOYMENT_ID="$DEPLOYMENT_ID" NODE_ENV=prod node ../deploy/set-webhook-processing.js true "$WEBHOOK_CONTROL_REVISION" deploy failed-release-restore) + WEBHOOK_CONTROL_REVISION=$((WEBHOOK_CONTROL_REVISION + 1)) + fi set -e fi } @@ -195,14 +333,28 @@ run_pm2 save echo "==> Waiting for health check" for i in $(seq 1 15); do - if curl -fsS http://localhost:3000/health >/dev/null 2>&1; then + if curl -fsS http://localhost:3000/health >/dev/null 2>&1 && \ + run_pm2 jlist | EXPECTED_REVISION="$REVISION" EXPECTED_DEPLOYMENT_ID="$DEPLOYMENT_ID" node -e ' + let input=""; process.stdin.on("data",d=>input+=d); process.stdin.on("end",()=>{const app=JSON.parse(input).find(p=>p.name==="ona-api"); process.exit(app?.pm2_env?.RELEASE_REVISION===process.env.EXPECTED_REVISION&&app?.pm2_env?.DEPLOYMENT_ID===process.env.EXPECTED_DEPLOYMENT_ID?0:1);}); + '; then if (cd "$RELEASE_DIR/api" && EXPECTED_REVISION="$REVISION" EXPECTED_WORKER_ENV="$WORKER_ENV" EXPECTED_DEPLOYMENT_ID="$DEPLOYMENT_ID" node - <<'NODE' -require('dotenv-flow').config(); +require('dotenv').config({path:'.env.prod'}); const { Pool } = require('pg'); const pool = new Pool({ user:process.env.DB_USER,password:process.env.DB_PASSWORD,host:process.env.DB_HOST,port:process.env.DB_PORT,database:process.env.DB_NAME||'ONA',ssl:process.env.DB_SSL==='true'?{ca:process.env.DB_SSL_CA?require('fs').readFileSync(process.env.DB_SSL_CA,'utf8'):undefined,rejectUnauthorized:Boolean(process.env.DB_SSL_CA)}:undefined }); -pool.query(`SELECT 1 FROM email_worker_heartbeats h JOIN email_worker_control c USING(environment) WHERE h.environment=$2 AND h.release_revision=$1 AND h.worker_instance LIKE $3||'/%' AND h.enabled=true AND h.claiming=c.claiming_enabled AND h.heartbeat_at>now()-interval '45 seconds' LIMIT 1`,[process.env.EXPECTED_REVISION,process.env.EXPECTED_WORKER_ENV,process.env.EXPECTED_DEPLOYMENT_ID]).then((r)=>{process.exitCode=r.rowCount?0:1;}).catch(()=>{process.exitCode=1;}).finally(()=>pool.end()); +Promise.all([ + pool.query(`SELECT 1 FROM email_worker_heartbeats h JOIN email_worker_control c USING(environment) WHERE h.environment=$2 AND h.release_revision=$1 AND h.worker_instance LIKE $3||'/%' AND h.enabled=true AND h.claiming=c.claiming_enabled AND h.heartbeat_at>now()-interval '45 seconds' LIMIT 1`,[process.env.EXPECTED_REVISION,process.env.EXPECTED_WORKER_ENV,process.env.EXPECTED_DEPLOYMENT_ID]), + pool.query(`SELECT 1 FROM email_webhook_worker_heartbeats h JOIN email_webhook_worker_control c USING(environment) WHERE h.environment=$2 AND h.release_revision=$1 AND h.worker_instance LIKE $3||'/%' AND h.enabled=true AND h.processing=c.processing_enabled AND h.heartbeat_at>now()-interval '45 seconds' LIMIT 1`,[process.env.EXPECTED_REVISION,process.env.EXPECTED_WORKER_ENV,process.env.EXPECTED_DEPLOYMENT_ID]), +]).then((results)=>{process.exitCode=results.every((r)=>r.rowCount)?0:1;}).catch(()=>{process.exitCode=1;}).finally(()=>pool.end()); NODE ); then + if [ "$SENDING_WAS_ENABLED" = true ] && [ "$SENDING_HANDOFF_REENABLED" = false ]; then + echo "==> Fencing all application email sends to release $REVISION" + (cd "$RELEASE_DIR/api" && EMAIL_WORKER_ENV="$WORKER_ENV" EXPECTED_RELEASE_REVISION="$REVISION" NODE_ENV=prod node ../deploy/set-email-sending.js true "$SENDING_CONTROL_REVISION" deploy release-handoff-complete) + SENDING_HANDOFF_REENABLED=true + SENDING_CONTROL_REVISION=$((SENDING_CONTROL_REVISION + 1)) + sleep 2 + continue + fi if [ "$CLAIMING_WAS_ENABLED" = true ] && [ "$HANDOFF_REENABLED" = false ]; then echo "==> Fencing email claims to release $REVISION" (cd "$RELEASE_DIR/api" && EMAIL_WORKER_ENV="$WORKER_ENV" EXPECTED_RELEASE_REVISION="$REVISION" NODE_ENV=prod node ../deploy/set-email-claiming.js true release-handoff-complete) @@ -210,7 +362,15 @@ NODE sleep 2 continue fi - echo "==> Deploy of $REVISION succeeded (API and worker healthy)" + if [ "$WEBHOOK_PROCESSING_WAS_ENABLED" = true ] && [ "$WEBHOOK_HANDOFF_REENABLED" = false ]; then + echo "==> Fencing webhook projection to release $REVISION" + (cd "$RELEASE_DIR/api" && EMAIL_WORKER_ENV="$WORKER_ENV" EXPECTED_RELEASE_REVISION="$REVISION" EXPECTED_DEPLOYMENT_ID="$DEPLOYMENT_ID" NODE_ENV=prod node ../deploy/set-webhook-processing.js true "$WEBHOOK_CONTROL_REVISION" deploy release-handoff-complete) + WEBHOOK_HANDOFF_REENABLED=true + WEBHOOK_CONTROL_REVISION=$((WEBHOOK_CONTROL_REVISION + 1)) + sleep 2 + continue + fi + echo "==> Deploy of $REVISION succeeded (API, delivery worker, and webhook worker healthy)" # Keep the five most recent releases ls -1dt "$SERVICE_DIR"/releases/* | tail -n +6 | xargs -r rm -rf trap - EXIT @@ -223,4 +383,5 @@ done echo "!! Health check failed after deploy of $REVISION" >&2 run_pm2 logs "$PM2_APP" --nostream --lines 50 || true run_pm2 logs "$PM2_WORKER" --nostream --lines 50 || true +run_pm2 logs "$PM2_WEBHOOK_WORKER" --nostream --lines 50 || true exit 1 diff --git a/scripts/deploy/replay-webhook-event.js b/scripts/deploy/replay-webhook-event.js new file mode 100644 index 0000000..be96be9 --- /dev/null +++ b/scripts/deploy/replay-webhook-event.js @@ -0,0 +1,11 @@ +'use strict'; +const path=require('path'),fs=require('fs'); +require(path.join(process.cwd(),'node_modules/dotenv')).config({path:path.join(process.cwd(),'.env.prod')}); +const{Pool}=require(path.join(process.cwd(),'node_modules/pg')); +const[idArg,actorArg,...reasonParts]=process.argv.slice(2); +if(!/^\d+$/.test(idArg||'')||!actorArg||!reasonParts.length)throw new Error('usage: replay-webhook-event.js '); +const environment=process.env.EMAIL_WORKER_ENV; +if(!['staging','prod'].includes(environment))throw new Error('EMAIL_WORKER_ENV must be staging or prod'); +const actor=actorArg.slice(0,255),reason=reasonParts.join(' ').slice(0,500); +const pool=new Pool({user:process.env.DB_USER,password:process.env.DB_PASSWORD,host:process.env.DB_HOST,port:process.env.DB_PORT,database:process.env.DB_NAME||'ONA',ssl:process.env.DB_SSL==='true'?{ca:process.env.DB_SSL_CA?fs.readFileSync(process.env.DB_SSL_CA,'utf8'):undefined,rejectUnauthorized:Boolean(process.env.DB_SSL_CA)}:undefined}); +(async()=>{const result=await pool.query(`UPDATE email_webhook_events SET status='pending',next_attempt_at=now(),unmatched_since_at=NULL,lease_owner=NULL,lease_token=NULL,lease_expires_at=NULL,last_error_code=NULL,last_error_message=NULL,processed_at=NULL,dead_lettered_at=NULL,replay_count=replay_count+1,last_replayed_at=now(),last_replayed_by_actor=$3,last_replay_reason=$4,updated_at=now() WHERE id=$1 AND receiving_environment=$2 AND status IN ('processed','ignored','dead_letter','unmatched') AND raw_payload IS NOT NULL RETURNING id,replay_count`,[idArg,environment,actor,reason]);if(!result.rowCount)throw new Error('event not found, not terminal/unmatched, foreign environment, or raw payload already expired');console.log(`webhook event ${result.rows[0].id} queued for audited local replay ${result.rows[0].replay_count}`);})().catch(e=>{console.error(e.message);process.exitCode=1;}).finally(()=>pool.end()); diff --git a/scripts/deploy/set-email-sending.js b/scripts/deploy/set-email-sending.js new file mode 100644 index 0000000..ef8f779 --- /dev/null +++ b/scripts/deploy/set-email-sending.js @@ -0,0 +1,12 @@ +'use strict'; + +const path=require('path'),fs=require('fs'),crypto=require('crypto'); +require(path.join(process.cwd(),'node_modules/dotenv')).config({path:path.join(process.cwd(),'.env.prod')}); +const {Pool}=require(path.join(process.cwd(),'node_modules/pg')); +const [value,revisionArg,actorArg,...reasonParts]=process.argv.slice(2); +if(!['true','false'].includes(value)||!/^\d+$/.test(revisionArg||'')||!actorArg||!reasonParts.length)throw new Error('usage: set-email-sending.js '); +const enabled=value==='true',environment=process.env.EMAIL_WORKER_ENV,release=process.env.EXPECTED_RELEASE_REVISION||'',expected=Number(revisionArg),actor=actorArg.slice(0,255),reason=reasonParts.join(' ').slice(0,500); +if(!['staging','prod'].includes(environment))throw new Error('EMAIL_WORKER_ENV must be staging or prod'); +if(enabled&&!release)throw new Error('enabling requires EXPECTED_RELEASE_REVISION'); +const pool=new Pool({user:process.env.DB_USER,password:process.env.DB_PASSWORD,host:process.env.DB_HOST,port:process.env.DB_PORT,database:process.env.DB_NAME||'ONA',ssl:process.env.DB_SSL==='true'?{ca:process.env.DB_SSL_CA?fs.readFileSync(process.env.DB_SSL_CA,'utf8'):undefined,rejectUnauthorized:Boolean(process.env.DB_SSL_CA)}:undefined}); +(async()=>{const c=await pool.connect();try{await c.query('BEGIN');await c.query(`SET LOCAL lock_timeout='30s'`);await c.query('SELECT pg_advisory_xact_lock(hashtextextended($1,0))',[`email-provider-boundary:${environment}`]);const before=(await c.query('SELECT * FROM email_sending_control WHERE environment=$1 FOR UPDATE',[environment])).rows[0];if(!before)throw new Error('email sending control row not found');if(Number(before.control_revision)!==expected)throw new Error(`control revision changed; current revision is ${before.control_revision}`);const next=expected+1;const after=(await c.query(`UPDATE email_sending_control SET sending_enabled=$3,minimum_release=CASE WHEN $3 THEN $4 ELSE minimum_release END,control_revision=$5,updated_at=now(),updated_by_actor=$6,reason=$7 WHERE environment=$1 AND control_revision=$2 RETURNING *`,[environment,expected,enabled,release,next,actor,reason])).rows[0];if(!after)throw new Error('stale email sending control revision');await c.query(`INSERT INTO email_control_audit(environment,control_name,control_revision,operation_id,previous_value,new_value,actor,reason) VALUES($1,'email_sending',$2,$3,$4,$5,$6,$7)`,[environment,next,crypto.randomUUID(),before,after,actor,reason]);await c.query('COMMIT');console.log(`application email sending ${enabled?'enabled':'disabled'} for ${environment}; control revision ${next}`);}catch(e){await c.query('ROLLBACK').catch(()=>{});throw e;}finally{c.release();await pool.end();}})().catch(e=>{console.error(e.message);process.exit(1);}); diff --git a/scripts/deploy/set-webhook-processing.js b/scripts/deploy/set-webhook-processing.js new file mode 100644 index 0000000..34c2213 --- /dev/null +++ b/scripts/deploy/set-webhook-processing.js @@ -0,0 +1,70 @@ +'use strict'; + +const path = require('path'); +const fs = require('fs'); +const crypto = require('crypto'); +const dotenv = require(path.join(process.cwd(), 'node_modules/dotenv')); +dotenv.config({ path: path.join(process.cwd(), '.env.prod') }); +const { Pool } = require(path.join(process.cwd(), 'node_modules/pg')); + +const [enabledArg, revisionArg, actorArg, ...reasonParts] = process.argv.slice(2); +if (!['true', 'false'].includes(enabledArg) || !/^\d+$/.test(revisionArg || '') || !actorArg || reasonParts.length === 0) { + throw new Error('usage: set-webhook-processing.js '); +} +const enabled = enabledArg === 'true'; +const expectedControlRevision = Number(revisionArg); +const environment = process.env.EMAIL_WORKER_ENV; +const release = process.env.EXPECTED_RELEASE_REVISION || ''; +const deploymentId = process.env.EXPECTED_DEPLOYMENT_ID || ''; +const actor = actorArg.slice(0, 255); +const reason = reasonParts.join(' ').slice(0, 500); +if (!['staging', 'prod'].includes(environment)) throw new Error('EMAIL_WORKER_ENV must be staging or prod'); +if (enabled && (!release || !deploymentId)) throw new Error('enabling requires EXPECTED_RELEASE_REVISION and EXPECTED_DEPLOYMENT_ID'); + +const pool = new Pool({ + user: process.env.DB_USER, password: process.env.DB_PASSWORD, host: process.env.DB_HOST, + port: process.env.DB_PORT, database: process.env.DB_NAME || 'ONA', + ssl: process.env.DB_SSL === 'true' ? { ca: process.env.DB_SSL_CA ? fs.readFileSync(process.env.DB_SSL_CA, 'utf8') : undefined, rejectUnauthorized: Boolean(process.env.DB_SSL_CA) } : undefined, +}); + +(async () => { + const client = await pool.connect(); + try { + await client.query('BEGIN'); + await client.query('SELECT pg_advisory_xact_lock(hashtextextended($1,0))', [`webhook-processing-control:${environment}`]); + const currentResult = await client.query('SELECT * FROM email_webhook_worker_control WHERE environment=$1 FOR UPDATE', [environment]); + const current = currentResult.rows[0]; + if (!current) throw new Error(`webhook worker control row not found for ${environment}`); + if (Number(current.control_revision) !== expectedControlRevision) throw new Error(`control revision changed; current revision is ${current.control_revision}`); + if (enabled) { + const heartbeat = await client.query( + `SELECT 1 FROM email_webhook_worker_heartbeats WHERE environment=$1 AND release_revision=$2 + AND worker_instance LIKE $3||'/%' AND enabled=true AND heartbeat_at>now()-interval '45 seconds' LIMIT 1`, + [environment, release, deploymentId] + ); + if (!heartbeat.rowCount) throw new Error('no fresh exact-deployment webhook worker heartbeat'); + } + const nextRevision = expectedControlRevision + 1; + const updated = await client.query( + `UPDATE email_webhook_worker_control SET processing_enabled=$3, claiming_enabled=$3, + minimum_release=CASE WHEN $3 THEN $4 ELSE minimum_release END, release_revision=$4, + control_revision=$5, updated_at=now(), updated_by_actor=$6, reason=$7 + WHERE environment=$1 AND control_revision=$2 RETURNING *`, + [environment, expectedControlRevision, enabled, release, nextRevision, actor, reason] + ); + if (!updated.rowCount) throw new Error('stale webhook processing control revision'); + await client.query( + `INSERT INTO email_control_audit(environment,control_name,control_revision,operation_id,previous_value,new_value,actor,reason) + VALUES($1,'webhook_processing',$2,$3,$4,$5,$6,$7)`, + [environment, nextRevision, crypto.randomUUID(), current, updated.rows[0], actor, reason] + ); + await client.query('COMMIT'); + console.log(`webhook processing ${enabled ? 'enabled' : 'disabled'} for ${environment}; control revision ${nextRevision}`); + } catch (error) { + await client.query('ROLLBACK').catch(() => {}); + throw error; + } finally { + client.release(); + await pool.end(); + } +})().catch((error) => { console.error(error.message); process.exit(1); }); diff --git a/scripts/deploy/validate-release-capabilities.js b/scripts/deploy/validate-release-capabilities.js new file mode 100644 index 0000000..120b10d --- /dev/null +++ b/scripts/deploy/validate-release-capabilities.js @@ -0,0 +1,56 @@ +'use strict'; + +const fs = require('fs'); +const path = require('path'); + +const releaseDir = path.resolve(process.argv[2] || '.'); +const checkDatabase = process.argv.includes('--database'); +const markerPath = path.join(releaseDir, 'deploy', 'CAPABILITIES.json'); +const required = ['webhook_ingest', 'webhook_projection', 'suppression_enforcement']; + +let marker; +try { + marker = JSON.parse(fs.readFileSync(markerPath, 'utf8')); +} catch (_) { + throw new Error('release is missing a valid deploy/CAPABILITIES.json marker'); +} +if (marker.format_version !== 1) throw new Error('unsupported capability marker format'); +for (const capability of required) { + if (!Number.isSafeInteger(marker[capability]) || marker[capability] < 1) { + throw new Error(`release lacks required ${capability} capability`); + } +} +if (!Number.isSafeInteger(marker.schema?.webhook_delivery_truth) || marker.schema.webhook_delivery_truth < 1) { + throw new Error('release lacks webhook delivery-truth schema capability'); +} +async function validateDatabaseFloor() { + if (!checkDatabase) return; + const apiDir = path.join(releaseDir, 'api'); + require(path.join(apiDir, 'node_modules', 'dotenv')).config({ path: path.join(apiDir, '.env.prod') }); + const { Pool } = require(path.join(apiDir, 'node_modules', 'pg')); + const env = process.env.EMAIL_WORKER_ENV; + const pool = new Pool({ + user:process.env.DB_USER,password:process.env.DB_PASSWORD,host:process.env.DB_HOST, + port:process.env.DB_PORT,database:process.env.DB_NAME||'ONA', + ssl:process.env.DB_SSL==='true'?{ca:process.env.DB_SSL_CA?fs.readFileSync(process.env.DB_SSL_CA,'utf8'):undefined,rejectUnauthorized:Boolean(process.env.DB_SSL_CA)}:undefined, + }); + try { + const result = await pool.query( + `SELECT + COALESCE((SELECT ingestion_required FROM email_webhook_registration_control WHERE environment=$1),false) AS ingestion_required, + COALESCE((SELECT processing_enabled FROM email_webhook_worker_control WHERE environment=$1),false) AS projection_required, + COALESCE((SELECT enforcement_enabled FROM email_suppression_control WHERE environment=$1),false) AS suppression_required`, + [env] + ); + const floor = result.rows[0]; + if (floor.ingestion_required && marker.webhook_ingest < 1) throw new Error('registration requires webhook ingestion capability'); + if (floor.projection_required && marker.webhook_projection < 1) throw new Error('processing control requires webhook projection capability'); + if (floor.suppression_required && marker.suppression_enforcement < 1) throw new Error('suppression latch requires suppression enforcement capability'); + } finally { + await pool.end(); + } +} + +validateDatabaseFloor() + .then(() => process.stdout.write(`release capability marker valid${checkDatabase ? ' for database floor' : ''}\n`)) + .catch((error) => { console.error(error.message); process.exit(1); }); diff --git a/scripts/local-dev.js b/scripts/local-dev.js index dad709b..2327a1f 100644 --- a/scripts/local-dev.js +++ b/scripts/local-dev.js @@ -423,6 +423,14 @@ async function main() { env: { EMAIL_WORKER_ENV: 'local', RELEASE_REVISION: 'local' }, }); + console.log('Starting webhook projection worker...'); + startService({ + name: 'email-webhook-worker', + cwd: path.join(repoRoot, 'api'), + command: `${npmCommand} run webhook-worker:dev`, + env: { EMAIL_WORKER_ENV: 'local', RELEASE_REVISION: 'local', RESEND_PROVIDER_ACCOUNT_SCOPE: 'network-survey-resend-team' }, + }); + console.log('Starting dashboard and survey app...'); startService({ name: 'dashboard', diff --git a/terraform/README.md b/terraform/README.md index a8ab2c3..c718a15 100644 --- a/terraform/README.md +++ b/terraform/README.md @@ -32,14 +32,22 @@ API runtime secrets are stored in SSM Parameter Store SecureString values, e.g.: /network-survey/staging/db/password /network-survey/staging/api/session-secret /network-survey/staging/api/resend-api-key +/network-survey/staging/api/resend-webhook-secret +/network-survey/staging/api/resend-webhook-secret-previous # optional rotation overlap /network-survey/staging/api/bootstrap-admin-password /network-survey/prod/db/password /network-survey/prod/api/session-secret /network-survey/prod/api/resend-api-key +/network-survey/prod/api/resend-webhook-secret +/network-survey/prod/api/resend-webhook-secret-previous # optional rotation overlap /network-survey/prod/api/bootstrap-admin-password ``` -Never commit secret values or local `*.local.tfvars` files. +Never commit secret values or local `*.local.tfvars` files. Terraform stores only the webhook parameter names and grants each EC2 runtime role read-only access to its own environment paths; webhook registration writes values only under separately authorized operator credentials. + +The shared non-secret `resend_provider_account_scope` must be configured identically in staging and production because they use one Resend team. `resend_webhook_ingest_enabled` defaults false and is the only Phase 2 database-independent release gate. Processing, suppression enforcement, and all-email sending are audited database controls. + +Each backend has separate API, delivery-worker, and webhook-worker CloudWatch log groups, a Terraform-managed SNS alarm topic/subscription, and environment-only webhook metric alarms. Terraform also applies an SSM Association so the CloudWatch Agent is configured in place on existing hosts rather than relying only on cloud-init. ### Staging dashboard bootstrap administrator diff --git a/terraform/cloud-init-template.sh b/terraform/cloud-init-template.sh index f5c3644..87012ab 100644 --- a/terraform/cloud-init-template.sh +++ b/terraform/cloud-init-template.sh @@ -12,6 +12,14 @@ export AWS_DEFAULT_REGION=${aws_region} apt-get update -y apt-get install -y curl unzip awscli openjdk-17-jre-headless postgresql-client build-essential python3 +# CloudWatch Agent forwards bounded API/worker JSON and Embedded Metric Format +# records. Application code must never write payloads, addresses, signatures, or +# secrets to these files. +curl -fsSL https://amazoncloudwatch-agent.s3.amazonaws.com/ubuntu/amd64/latest/amazon-cloudwatch-agent.deb \ + -o /tmp/amazon-cloudwatch-agent.deb +dpkg -i /tmp/amazon-cloudwatch-agent.deb +rm -f /tmp/amazon-cloudwatch-agent.deb + # Node.js 20 LTS curl -fsSL https://deb.nodesource.com/setup_20.x | bash - apt-get install -y nodejs @@ -47,6 +55,29 @@ ENVIRONMENT=${environment} EOF chown ubuntu:ubuntu $SERVICE_DIR/deploy.env +cat > /opt/aws/amazon-cloudwatch-agent/etc/amazon-cloudwatch-agent.json <<'EOF' +{ + "agent": { "metrics_collection_interval": 60, "run_as_user": "root" }, + "logs": { + "logs_collected": { + "files": { + "collect_list": [ + { "file_path": "/home/ubuntu/.pm2/logs/ona-api-out.log", "log_group_name": "${api_log_group}", "log_stream_name": "{instance_id}/stdout", "timezone": "UTC" }, + { "file_path": "/home/ubuntu/.pm2/logs/ona-api-error.log", "log_group_name": "${api_log_group}", "log_stream_name": "{instance_id}/stderr", "timezone": "UTC" }, + { "file_path": "/home/ubuntu/.pm2/logs/ona-email-worker-out.log", "log_group_name": "${email_worker_log_group}", "log_stream_name": "{instance_id}/stdout", "timezone": "UTC" }, + { "file_path": "/home/ubuntu/.pm2/logs/ona-email-worker-error.log", "log_group_name": "${email_worker_log_group}", "log_stream_name": "{instance_id}/stderr", "timezone": "UTC" }, + { "file_path": "/home/ubuntu/.pm2/logs/ona-email-webhook-worker-out.log", "log_group_name": "${webhook_worker_log_group}", "log_stream_name": "{instance_id}/stdout", "timezone": "UTC" }, + { "file_path": "/home/ubuntu/.pm2/logs/ona-email-webhook-worker-error.log", "log_group_name": "${webhook_worker_log_group}", "log_stream_name": "{instance_id}/stderr", "timezone": "UTC" } + ] + } + } + } +} +EOF +/opt/aws/amazon-cloudwatch-agent/bin/amazon-cloudwatch-agent-ctl \ + -a fetch-config -m ec2 \ + -c file:/opt/aws/amazon-cloudwatch-agent/etc/amazon-cloudwatch-agent.json -s + # Host firewall (the security group is the real boundary; this is defense in depth). # SSH is intentionally not opened; use SSM Session Manager for access. ufw allow 3000 @@ -55,13 +86,13 @@ ufw --force enable # Bootstrap the latest release if CI has published one; otherwise the first # run of the deploy workflow will bring the app up. BOOTSTRAP_DIR=/tmp/ona-bootstrap -if aws s3 cp "s3://${artifacts_bucket}/api/latest.tar.gz" /tmp/ona-latest.tar.gz; then +if aws s3 cp "s3://${artifacts_bucket}/api/latest-compatible.tar.gz" /tmp/ona-latest.tar.gz; then mkdir -p $BOOTSTRAP_DIR tar -xzf /tmp/ona-latest.tar.gz -C $BOOTSTRAP_DIR bash $BOOTSTRAP_DIR/deploy/remote-deploy.sh $BOOTSTRAP_DIR rm -rf $BOOTSTRAP_DIR /tmp/ona-latest.tar.gz else - echo "No release artifact found in s3://${artifacts_bucket}/api/latest.tar.gz — run the deploy workflow to install the app." + echo "No capability-verified release artifact found in s3://${artifacts_bucket}/api/latest-compatible.tar.gz — run the deploy workflow to install the app." fi echo "Setup complete." diff --git a/terraform/envs/prod/app_stack.tf b/terraform/envs/prod/app_stack.tf index 36b4a65..f7f43d2 100644 --- a/terraform/envs/prod/app_stack.tf +++ b/terraform/envs/prod/app_stack.tf @@ -23,11 +23,13 @@ locals { app_name_prefix = "${var.project_name}-${var.environment}-v2" # Runtime secrets intentionally keep the existing production Parameter Store paths. - ssm_parameter_prefix = "/network-survey/${var.environment}" - db_password_parameter_name = "${local.ssm_parameter_prefix}/db/password" - session_secret_parameter_name = "${local.ssm_parameter_prefix}/api/session-secret" - resend_api_key_parameter_name = "${local.ssm_parameter_prefix}/api/resend-api-key" - bootstrap_admin_password_parameter_name = "${local.ssm_parameter_prefix}/api/bootstrap-admin-password" + ssm_parameter_prefix = "/network-survey/${var.environment}" + db_password_parameter_name = "${local.ssm_parameter_prefix}/db/password" + session_secret_parameter_name = "${local.ssm_parameter_prefix}/api/session-secret" + resend_api_key_parameter_name = "${local.ssm_parameter_prefix}/api/resend-api-key" + resend_webhook_secret_parameter_name = "${local.ssm_parameter_prefix}/api/resend-webhook-secret" + resend_webhook_previous_secret_parameter_name = "${local.ssm_parameter_prefix}/api/resend-webhook-secret-previous" + bootstrap_admin_password_parameter_name = "${local.ssm_parameter_prefix}/api/bootstrap-admin-password" frontend_url = "https://${var.dashboard_domain}" survey_url = "https://${var.survey_domain}" @@ -100,29 +102,35 @@ module "api_backend" { cloud_init_template_path = "${path.module}/../../cloud-init-template.sh" env_template_path = "${path.module}/../../templates/env.tmpl" - db_host = local.api_config_db_host - db_port = aws_db_instance.prod_replacement.port - db_name = aws_db_instance.prod_replacement.db_name - db_user = var.db_user - db_password_parameter_name = local.db_password_parameter_name - session_secret_parameter_name = local.session_secret_parameter_name - resend_api_key_parameter_name = local.resend_api_key_parameter_name - bootstrap_admin_username = var.enable_cla_owner_bootstrap ? "sgarcia@cladvisors.com" : null - bootstrap_admin_email = var.enable_cla_owner_bootstrap ? "sgarcia@cladvisors.com" : null - bootstrap_admin_password_parameter_name = var.enable_cla_owner_bootstrap ? local.bootstrap_admin_password_parameter_name : null - bootstrap_organization_name = "CLA" - bootstrap_organization_slug = "cla" - bootstrap_platform_admin = false - bootstrap_account_mode = "create-or-verify" - cla_production_cutover = var.enable_cla_production_cutover - frontend_url = local.frontend_url - survey_url = local.survey_url - session_cookie_name = local.session_cookie_name - email_worker_environment = "prod" - survey_delivery_v2_enabled = var.survey_delivery_v2_enabled - legacy_start_enabled = false - email_rate_per_second = var.email_rate_per_second - email_rate_budget_environment = "prod" + db_host = local.api_config_db_host + db_port = aws_db_instance.prod_replacement.port + db_name = aws_db_instance.prod_replacement.db_name + db_user = var.db_user + db_password_parameter_name = local.db_password_parameter_name + session_secret_parameter_name = local.session_secret_parameter_name + resend_api_key_parameter_name = local.resend_api_key_parameter_name + resend_webhook_secret_parameter_name = local.resend_webhook_secret_parameter_name + resend_webhook_previous_secret_parameter_name = local.resend_webhook_previous_secret_parameter_name + resend_provider_account_scope = var.resend_provider_account_scope + resend_webhook_ingest_enabled = var.resend_webhook_ingest_enabled + webhook_payload_retention_days = var.webhook_payload_retention_days + operations_alert_email = var.operations_alert_email + bootstrap_admin_username = var.enable_cla_owner_bootstrap ? "sgarcia@cladvisors.com" : null + bootstrap_admin_email = var.enable_cla_owner_bootstrap ? "sgarcia@cladvisors.com" : null + bootstrap_admin_password_parameter_name = var.enable_cla_owner_bootstrap ? local.bootstrap_admin_password_parameter_name : null + bootstrap_organization_name = "CLA" + bootstrap_organization_slug = "cla" + bootstrap_platform_admin = false + bootstrap_account_mode = "create-or-verify" + cla_production_cutover = var.enable_cla_production_cutover + frontend_url = local.frontend_url + survey_url = local.survey_url + session_cookie_name = local.session_cookie_name + email_worker_environment = "prod" + survey_delivery_v2_enabled = var.survey_delivery_v2_enabled + legacy_start_enabled = false + email_rate_per_second = var.email_rate_per_second + email_rate_budget_environment = "prod" common_tags = local.app_common_tags config_bucket_tags = merge(local.app_common_tags, { Name = "${local.app_name_prefix}-config", App = "ona-config" }) diff --git a/terraform/envs/prod/outputs.tf b/terraform/envs/prod/outputs.tf index ee5e702..64bbd12 100644 --- a/terraform/envs/prod/outputs.tf +++ b/terraform/envs/prod/outputs.tf @@ -97,14 +97,25 @@ output "survey_distribution_id" { output "runtime_secret_parameter_names" { value = { - db_password = local.db_password_parameter_name - session_secret = local.session_secret_parameter_name - resend_api_key = local.resend_api_key_parameter_name - bootstrap_admin_password = local.bootstrap_admin_password_parameter_name + db_password = local.db_password_parameter_name + session_secret = local.session_secret_parameter_name + resend_api_key = local.resend_api_key_parameter_name + resend_webhook_secret = local.resend_webhook_secret_parameter_name + resend_webhook_previous_secret = local.resend_webhook_previous_secret_parameter_name + bootstrap_admin_password = local.bootstrap_admin_password_parameter_name } description = "Existing production SSM Parameter Store paths reused by the replacement app runtime." } +output "operations_alert_topic_arn" { + value = module.api_backend.operations_alert_topic_arn + description = "SNS topic for production webhook and runtime alarms." +} + +output "runtime_log_group_names" { + value = module.api_backend.runtime_log_group_names +} + output "api_alb_dns_name" { value = module.api_backend.alb_dns_name description = "External DNS target: point demo.ona.api.bennetts.work CNAME here at cutover." diff --git a/terraform/envs/prod/variables.tf b/terraform/envs/prod/variables.tf index 33507de..cc32368 100644 --- a/terraform/envs/prod/variables.tf +++ b/terraform/envs/prod/variables.tf @@ -154,6 +154,31 @@ variable "survey_delivery_v2_enabled" { default = false } +variable "resend_provider_account_scope" { + description = "Stable scope shared with staging because both use the same Resend team." + type = string + default = "network-survey-resend-team" +} + +variable "resend_webhook_ingest_enabled" { + description = "Release gate for signed Resend webhook ingestion. Keep false until production endpoint capacity and secret bootstrap are approved." + type = bool + default = false +} + +variable "webhook_payload_retention_days" { + description = "Days to retain bounded raw webhook payloads." + type = number + default = 30 +} + +variable "operations_alert_email" { + description = "Optional initial SNS email subscriber. Production routing must be explicitly approved before webhook enablement." + type = string + default = "bgarcia2324@gmail.com" + nullable = true +} + variable "email_rate_per_second" { description = "Approved aggregate Resend account request budget shared with staging." type = number diff --git a/terraform/envs/staging/locals.tf b/terraform/envs/staging/locals.tf index d2fb626..7507f1b 100644 --- a/terraform/envs/staging/locals.tf +++ b/terraform/envs/staging/locals.tf @@ -14,11 +14,13 @@ locals { # as a host-only cookie, but distinct names keep environments unambiguous. session_cookie_name = "ona-session-${local.environment}-v2" - ssm_parameter_prefix = "/network-survey/${local.environment}" - db_password_parameter_name = "${local.ssm_parameter_prefix}/db/password" - session_secret_parameter_name = "${local.ssm_parameter_prefix}/api/session-secret" - resend_api_key_parameter_name = "${local.ssm_parameter_prefix}/api/resend-api-key" - bootstrap_admin_password_parameter_name = "${local.ssm_parameter_prefix}/api/bootstrap-admin-password" + ssm_parameter_prefix = "/network-survey/${local.environment}" + db_password_parameter_name = "${local.ssm_parameter_prefix}/db/password" + session_secret_parameter_name = "${local.ssm_parameter_prefix}/api/session-secret" + resend_api_key_parameter_name = "${local.ssm_parameter_prefix}/api/resend-api-key" + resend_webhook_secret_parameter_name = "${local.ssm_parameter_prefix}/api/resend-webhook-secret" + resend_webhook_previous_secret_parameter_name = "${local.ssm_parameter_prefix}/api/resend-webhook-secret-previous" + bootstrap_admin_password_parameter_name = "${local.ssm_parameter_prefix}/api/bootstrap-admin-password" common_tags = { Project = "network-survey" diff --git a/terraform/envs/staging/main.tf b/terraform/envs/staging/main.tf index 7509a38..1aea087 100644 --- a/terraform/envs/staging/main.tf +++ b/terraform/envs/staging/main.tf @@ -127,39 +127,45 @@ resource "random_string" "suffix" { module "api_backend" { source = "../../modules/api_backend" - aws_region = var.aws_region - environment = local.environment - name_prefix = local.name_prefix - vpc_id = aws_vpc.main.id - backend_subnet_id = aws_subnet.db_subnet_1.id - alb_subnet_ids = [aws_subnet.db_subnet_1.id, aws_subnet.db_subnet_2.id] - instance_type = var.instance_type - ssh_allowed_cidrs = var.ssh_allowed_cidrs - ssh_key_name = var.ssh_key_name - certificate_arn = aws_acm_certificate.ssl_cert.arn - config_bucket_name = local.config_bucket_name - artifacts_bucket_name = local.artifacts_bucket_name - artifact_retention_days = var.artifact_retention_days - alb_deletion_protection = var.alb_deletion_protection - cloud_init_template_path = "${path.module}/../../cloud-init-template.sh" - env_template_path = "${path.module}/../../templates/env.tmpl" - db_host = coalesce(var.api_config_db_host_override, aws_db_instance.postgres.address) - db_port = aws_db_instance.postgres.port - db_name = aws_db_instance.postgres.db_name - db_user = var.db_user - db_password_parameter_name = local.db_password_parameter_name - session_secret_parameter_name = local.session_secret_parameter_name - resend_api_key_parameter_name = local.resend_api_key_parameter_name - bootstrap_admin_username = "admin" - bootstrap_admin_password_parameter_name = local.bootstrap_admin_password_parameter_name - frontend_url = local.frontend_url - survey_url = local.survey_url - session_cookie_name = local.session_cookie_name - email_worker_environment = "staging" - survey_delivery_v2_enabled = var.survey_delivery_v2_enabled - legacy_start_enabled = false - email_rate_per_second = var.email_rate_per_second - email_rate_budget_environment = "staging" + aws_region = var.aws_region + environment = local.environment + name_prefix = local.name_prefix + vpc_id = aws_vpc.main.id + backend_subnet_id = aws_subnet.db_subnet_1.id + alb_subnet_ids = [aws_subnet.db_subnet_1.id, aws_subnet.db_subnet_2.id] + instance_type = var.instance_type + ssh_allowed_cidrs = var.ssh_allowed_cidrs + ssh_key_name = var.ssh_key_name + certificate_arn = aws_acm_certificate.ssl_cert.arn + config_bucket_name = local.config_bucket_name + artifacts_bucket_name = local.artifacts_bucket_name + artifact_retention_days = var.artifact_retention_days + alb_deletion_protection = var.alb_deletion_protection + cloud_init_template_path = "${path.module}/../../cloud-init-template.sh" + env_template_path = "${path.module}/../../templates/env.tmpl" + db_host = coalesce(var.api_config_db_host_override, aws_db_instance.postgres.address) + db_port = aws_db_instance.postgres.port + db_name = aws_db_instance.postgres.db_name + db_user = var.db_user + db_password_parameter_name = local.db_password_parameter_name + session_secret_parameter_name = local.session_secret_parameter_name + resend_api_key_parameter_name = local.resend_api_key_parameter_name + resend_webhook_secret_parameter_name = local.resend_webhook_secret_parameter_name + resend_webhook_previous_secret_parameter_name = local.resend_webhook_previous_secret_parameter_name + resend_provider_account_scope = var.resend_provider_account_scope + resend_webhook_ingest_enabled = var.resend_webhook_ingest_enabled + webhook_payload_retention_days = var.webhook_payload_retention_days + operations_alert_email = var.operations_alert_email + bootstrap_admin_username = "admin" + bootstrap_admin_password_parameter_name = local.bootstrap_admin_password_parameter_name + frontend_url = local.frontend_url + survey_url = local.survey_url + session_cookie_name = local.session_cookie_name + email_worker_environment = "staging" + survey_delivery_v2_enabled = var.survey_delivery_v2_enabled + legacy_start_enabled = false + email_rate_per_second = var.email_rate_per_second + email_rate_budget_environment = "staging" common_tags = local.common_tags config_bucket_tags = merge(local.common_tags, { Name = "Config Bucket" }) diff --git a/terraform/envs/staging/outputs.tf b/terraform/envs/staging/outputs.tf index e17b2f1..e2f9a63 100644 --- a/terraform/envs/staging/outputs.tf +++ b/terraform/envs/staging/outputs.tf @@ -30,10 +30,12 @@ output "artifacts_bucket_name" { output "runtime_secret_parameter_names" { value = { - db_password = local.db_password_parameter_name - session_secret = local.session_secret_parameter_name - resend_api_key = local.resend_api_key_parameter_name - bootstrap_admin_password = local.bootstrap_admin_password_parameter_name + db_password = local.db_password_parameter_name + session_secret = local.session_secret_parameter_name + resend_api_key = local.resend_api_key_parameter_name + resend_webhook_secret = local.resend_webhook_secret_parameter_name + resend_webhook_previous_secret = local.resend_webhook_previous_secret_parameter_name + bootstrap_admin_password = local.bootstrap_admin_password_parameter_name } description = "SSM Parameter Store names that must exist as SecureString values before deploys run" } @@ -92,6 +94,15 @@ output "ssl_cert_survey_validation_records" { ] } +output "operations_alert_topic_arn" { + value = module.api_backend.operations_alert_topic_arn + description = "SNS topic for staging webhook and runtime alarms." +} + +output "runtime_log_group_names" { + value = module.api_backend.runtime_log_group_names +} + output "alb_dns_name" { value = module.api_backend.alb_dns_name description = "The DNS name of the ALB. Point the API domain's CNAME record here." diff --git a/terraform/envs/staging/variables.tf b/terraform/envs/staging/variables.tf index 91d98c6..406c6a0 100644 --- a/terraform/envs/staging/variables.tf +++ b/terraform/envs/staging/variables.tf @@ -93,6 +93,31 @@ variable "survey_delivery_v2_enabled" { default = false } +variable "resend_provider_account_scope" { + description = "Stable scope shared with production because both use the same Resend team." + type = string + default = "network-survey-resend-team" +} + +variable "resend_webhook_ingest_enabled" { + description = "Release gate for signed Resend webhook ingestion. Enable only after secret bootstrap and disabled-endpoint reconciliation." + type = bool + default = false +} + +variable "webhook_payload_retention_days" { + description = "Days to retain bounded raw webhook payloads." + type = number + default = 30 +} + +variable "operations_alert_email" { + description = "Optional initial SNS email subscriber for staging infrastructure alarms." + type = string + default = "bgarcia2324@gmail.com" + nullable = true +} + variable "email_rate_per_second" { description = "Approved aggregate Resend account request budget shared with production." type = number diff --git a/terraform/locals.tf b/terraform/locals.tf index 5fac591..ff2b2e7 100644 --- a/terraform/locals.tf +++ b/terraform/locals.tf @@ -21,10 +21,12 @@ locals { # runtime config consistent if it is used for a plan/import operation. session_cookie_name = local.is_prod ? "ona-session-prod-v2" : "ona-session-${local.environment}-v2" - ssm_parameter_prefix = "/network-survey/${local.environment}" - db_password_parameter_name = "${local.ssm_parameter_prefix}/db/password" - session_secret_parameter_name = "${local.ssm_parameter_prefix}/api/session-secret" - resend_api_key_parameter_name = "${local.ssm_parameter_prefix}/api/resend-api-key" + ssm_parameter_prefix = "/network-survey/${local.environment}" + db_password_parameter_name = "${local.ssm_parameter_prefix}/db/password" + session_secret_parameter_name = "${local.ssm_parameter_prefix}/api/session-secret" + resend_api_key_parameter_name = "${local.ssm_parameter_prefix}/api/resend-api-key" + resend_webhook_secret_parameter_name = "${local.ssm_parameter_prefix}/api/resend-webhook-secret" + resend_webhook_previous_secret_parameter_name = "${local.ssm_parameter_prefix}/api/resend-webhook-secret-previous" common_tags = { Project = "network-survey" diff --git a/terraform/main.tf b/terraform/main.tf index 77b2c0b..0092978 100644 --- a/terraform/main.tf +++ b/terraform/main.tf @@ -115,6 +115,13 @@ data "aws_ami" "ubuntu" { } } +resource "aws_cloudwatch_log_group" "runtime" { + for_each = toset(["api", "email-worker", "webhook-worker"]) + name = "/network-survey/${local.environment}/${each.value}" + retention_in_days = 30 + tags = local.common_tags +} + resource "aws_instance" "backend" { ami = data.aws_ami.ubuntu.id instance_type = var.instance_type @@ -125,10 +132,13 @@ resource "aws_instance" "backend" { key_name = length(var.ssh_allowed_cidrs) > 0 ? var.ssh_key_name : null user_data = templatefile("cloud-init-template.sh", { - config_bucket = aws_s3_bucket.config_bucket.bucket - artifacts_bucket = aws_s3_bucket.artifacts.bucket - aws_region = var.aws_region - environment = local.environment + config_bucket = aws_s3_bucket.config_bucket.bucket + artifacts_bucket = aws_s3_bucket.artifacts.bucket + aws_region = var.aws_region + environment = local.environment + api_log_group = aws_cloudwatch_log_group.runtime["api"].name + email_worker_log_group = aws_cloudwatch_log_group.runtime["email-worker"].name + webhook_worker_log_group = aws_cloudwatch_log_group.runtime["webhook-worker"].name }) lifecycle { @@ -255,29 +265,35 @@ resource "aws_s3_object" "api_config" { bucket = aws_s3_bucket.config_bucket.id key = "configs/.env.prod" content = templatefile("./templates/env.tmpl", { - db_host = coalesce(var.api_config_db_host_override, aws_db_instance.postgres.address) - db_port = aws_db_instance.postgres.port - db_name = aws_db_instance.postgres.db_name - db_user = var.db_user - db_password_parameter_name = local.db_password_parameter_name - frontend_url = local.frontend_url - survey_url = local.survey_url - session_secret_parameter_name = local.session_secret_parameter_name - session_cookie_name = local.session_cookie_name - email_worker_environment = local.is_prod ? "prod" : "staging" - survey_delivery_v2_enabled = false - legacy_start_enabled = false - email_rate_per_second = local.is_prod ? 4 : 1 - email_rate_budget_environment = local.is_prod ? "prod" : "staging" - resend_api_key_parameter_name = local.resend_api_key_parameter_name - cla_production_cutover = false - bootstrap_admin_username = null - bootstrap_admin_password_parameter_name = null - bootstrap_admin_email = null - bootstrap_organization_name = "" - bootstrap_organization_slug = "" - bootstrap_platform_admin = false - bootstrap_account_mode = "local" + db_host = coalesce(var.api_config_db_host_override, aws_db_instance.postgres.address) + db_port = aws_db_instance.postgres.port + db_name = aws_db_instance.postgres.db_name + db_user = var.db_user + db_password_parameter_name = local.db_password_parameter_name + frontend_url = local.frontend_url + survey_url = local.survey_url + session_secret_parameter_name = local.session_secret_parameter_name + session_cookie_name = local.session_cookie_name + email_worker_environment = local.is_prod ? "prod" : "staging" + survey_delivery_v2_enabled = false + legacy_start_enabled = false + email_rate_per_second = local.is_prod ? 4 : 1 + email_rate_budget_environment = local.is_prod ? "prod" : "staging" + resend_api_key_parameter_name = local.resend_api_key_parameter_name + resend_webhook_secret_parameter_name = local.resend_webhook_secret_parameter_name + resend_webhook_previous_secret_parameter_name = local.resend_webhook_previous_secret_parameter_name + resend_provider_account_scope = var.resend_provider_account_scope + resend_webhook_ingest_enabled = false + webhook_payload_retention_days = 30 + webhook_metric_namespace = "NetworkSurvey/Webhooks" + cla_production_cutover = false + bootstrap_admin_username = null + bootstrap_admin_password_parameter_name = null + bootstrap_admin_email = null + bootstrap_organization_name = "" + bootstrap_organization_slug = "" + bootstrap_platform_admin = false + bootstrap_account_mode = "local" }) } @@ -376,7 +392,19 @@ data "aws_iam_policy_document" "s3_access_policy" { "arn:aws:ssm:${var.aws_region}:${data.aws_caller_identity.current.account_id}:parameter${local.db_password_parameter_name}", "arn:aws:ssm:${var.aws_region}:${data.aws_caller_identity.current.account_id}:parameter${local.session_secret_parameter_name}", "arn:aws:ssm:${var.aws_region}:${data.aws_caller_identity.current.account_id}:parameter${local.resend_api_key_parameter_name}", + "arn:aws:ssm:${var.aws_region}:${data.aws_caller_identity.current.account_id}:parameter${local.resend_webhook_secret_parameter_name}", + "arn:aws:ssm:${var.aws_region}:${data.aws_caller_identity.current.account_id}:parameter${local.resend_webhook_previous_secret_parameter_name}", + ] + } + + statement { + effect = "Allow" + actions = [ + "logs:CreateLogStream", + "logs:DescribeLogStreams", + "logs:PutLogEvents", ] + resources = [for group in aws_cloudwatch_log_group.runtime : "${group.arn}:*"] } statement { diff --git a/terraform/modules/api_backend/main.tf b/terraform/modules/api_backend/main.tf index 894ca2a..fbc34e0 100644 --- a/terraform/modules/api_backend/main.tf +++ b/terraform/modules/api_backend/main.tf @@ -110,29 +110,35 @@ resource "aws_s3_object" "api_config" { } } content = templatefile(var.env_template_path, { - db_host = var.db_host - db_port = var.db_port - db_name = var.db_name - db_user = var.db_user - db_password_parameter_name = var.db_password_parameter_name - frontend_url = var.frontend_url - survey_url = var.survey_url - session_secret_parameter_name = var.session_secret_parameter_name - session_cookie_name = var.session_cookie_name - email_worker_environment = var.email_worker_environment - survey_delivery_v2_enabled = var.survey_delivery_v2_enabled - legacy_start_enabled = var.legacy_start_enabled - email_rate_per_second = var.email_rate_per_second - email_rate_budget_environment = var.email_rate_budget_environment - resend_api_key_parameter_name = var.resend_api_key_parameter_name - bootstrap_admin_username = var.bootstrap_admin_username - bootstrap_admin_password_parameter_name = var.bootstrap_admin_password_parameter_name - bootstrap_admin_email = var.bootstrap_admin_email - bootstrap_organization_name = var.bootstrap_organization_name - bootstrap_organization_slug = var.bootstrap_organization_slug - bootstrap_platform_admin = var.bootstrap_platform_admin - bootstrap_account_mode = var.bootstrap_account_mode - cla_production_cutover = var.cla_production_cutover + db_host = var.db_host + db_port = var.db_port + db_name = var.db_name + db_user = var.db_user + db_password_parameter_name = var.db_password_parameter_name + frontend_url = var.frontend_url + survey_url = var.survey_url + session_secret_parameter_name = var.session_secret_parameter_name + session_cookie_name = var.session_cookie_name + email_worker_environment = var.email_worker_environment + survey_delivery_v2_enabled = var.survey_delivery_v2_enabled + legacy_start_enabled = var.legacy_start_enabled + email_rate_per_second = var.email_rate_per_second + email_rate_budget_environment = var.email_rate_budget_environment + resend_api_key_parameter_name = var.resend_api_key_parameter_name + resend_webhook_secret_parameter_name = var.resend_webhook_secret_parameter_name + resend_webhook_previous_secret_parameter_name = var.resend_webhook_previous_secret_parameter_name + resend_provider_account_scope = var.resend_provider_account_scope + resend_webhook_ingest_enabled = var.resend_webhook_ingest_enabled + webhook_payload_retention_days = var.webhook_payload_retention_days + webhook_metric_namespace = var.webhook_metric_namespace + bootstrap_admin_username = var.bootstrap_admin_username + bootstrap_admin_password_parameter_name = var.bootstrap_admin_password_parameter_name + bootstrap_admin_email = var.bootstrap_admin_email + bootstrap_organization_name = var.bootstrap_organization_name + bootstrap_organization_slug = var.bootstrap_organization_slug + bootstrap_platform_admin = var.bootstrap_platform_admin + bootstrap_account_mode = var.bootstrap_account_mode + cla_production_cutover = var.cla_production_cutover }) } @@ -242,11 +248,25 @@ data "aws_iam_policy_document" "s3_access_policy" { "arn:aws:ssm:${var.aws_region}:${data.aws_caller_identity.current.account_id}:parameter${var.db_password_parameter_name}", "arn:aws:ssm:${var.aws_region}:${data.aws_caller_identity.current.account_id}:parameter${var.session_secret_parameter_name}", "arn:aws:ssm:${var.aws_region}:${data.aws_caller_identity.current.account_id}:parameter${var.resend_api_key_parameter_name}", - ], var.bootstrap_admin_password_parameter_name == null ? [] : [ - "arn:aws:ssm:${var.aws_region}:${data.aws_caller_identity.current.account_id}:parameter${var.bootstrap_admin_password_parameter_name}", - ]) + "arn:aws:ssm:${var.aws_region}:${data.aws_caller_identity.current.account_id}:parameter${var.resend_webhook_secret_parameter_name}", + ], concat(var.resend_webhook_previous_secret_parameter_name == null ? [] : [ + "arn:aws:ssm:${var.aws_region}:${data.aws_caller_identity.current.account_id}:parameter${var.resend_webhook_previous_secret_parameter_name}", + ], var.bootstrap_admin_password_parameter_name == null ? [] : [ + "arn:aws:ssm:${var.aws_region}:${data.aws_caller_identity.current.account_id}:parameter${var.bootstrap_admin_password_parameter_name}", + ])) } + statement { + effect = "Allow" + actions = [ + "logs:CreateLogStream", + "logs:DescribeLogStreams", + "logs:PutLogEvents", + ] + resources = [for group in aws_cloudwatch_log_group.runtime : "${group.arn}:*"] + } + + statement { effect = "Allow" actions = ["kms:Decrypt"] @@ -306,6 +326,231 @@ data "aws_iam_policy_document" "ec2_assume_role_policy" { } } +resource "aws_cloudwatch_log_group" "runtime" { + for_each = toset(["api", "email-worker", "webhook-worker"]) + + name = "/network-survey/${var.environment}/${each.key}" + retention_in_days = var.cloudwatch_log_retention_days + tags = var.common_tags +} + +resource "aws_sns_topic" "operations_alerts" { + name = "${trimsuffix(var.name_prefix, "-")}-operations-alerts" + tags = var.common_tags +} + +data "aws_iam_policy_document" "operations_alert_topic" { + statement { + sid = "TopicOwnerAdministration" + effect = "Allow" + actions = ["SNS:GetTopicAttributes", "SNS:SetTopicAttributes", "SNS:AddPermission", "SNS:RemovePermission", "SNS:DeleteTopic", "SNS:Subscribe", "SNS:ListSubscriptionsByTopic", "SNS:Publish"] + resources = [aws_sns_topic.operations_alerts.arn] + principals { + type = "AWS" + identifiers = [data.aws_caller_identity.current.account_id] + } + } + + statement { + sid = "CloudWatchAlarmPublish" + effect = "Allow" + actions = ["SNS:Publish"] + resources = [aws_sns_topic.operations_alerts.arn] + principals { + type = "Service" + identifiers = ["cloudwatch.amazonaws.com"] + } + condition { + test = "StringEquals" + variable = "AWS:SourceAccount" + values = [data.aws_caller_identity.current.account_id] + } + condition { + test = "ArnLike" + variable = "AWS:SourceArn" + values = ["arn:aws:cloudwatch:${var.aws_region}:${data.aws_caller_identity.current.account_id}:alarm:*"] + } + } +} + +resource "aws_sns_topic_policy" "operations_alerts" { + arn = aws_sns_topic.operations_alerts.arn + policy = data.aws_iam_policy_document.operations_alert_topic.json +} + +resource "aws_sns_topic_subscription" "operations_email" { + count = var.operations_alert_email == null ? 0 : 1 + + topic_arn = aws_sns_topic.operations_alerts.arn + protocol = "email" + endpoint = var.operations_alert_email +} + +locals { + cloudwatch_agent_config_b64 = base64encode(jsonencode({ + agent = { metrics_collection_interval = 60, run_as_user = "root" } + logs = { logs_collected = { files = { collect_list = [ + { file_path = "/home/ubuntu/.pm2/logs/ona-api-out.log", log_group_name = aws_cloudwatch_log_group.runtime["api"].name, log_stream_name = "{instance_id}/stdout", timezone = "UTC" }, + { file_path = "/home/ubuntu/.pm2/logs/ona-api-error.log", log_group_name = aws_cloudwatch_log_group.runtime["api"].name, log_stream_name = "{instance_id}/stderr", timezone = "UTC" }, + { file_path = "/home/ubuntu/.pm2/logs/ona-email-worker-out.log", log_group_name = aws_cloudwatch_log_group.runtime["email-worker"].name, log_stream_name = "{instance_id}/stdout", timezone = "UTC" }, + { file_path = "/home/ubuntu/.pm2/logs/ona-email-worker-error.log", log_group_name = aws_cloudwatch_log_group.runtime["email-worker"].name, log_stream_name = "{instance_id}/stderr", timezone = "UTC" }, + { file_path = "/home/ubuntu/.pm2/logs/ona-email-webhook-worker-out.log", log_group_name = aws_cloudwatch_log_group.runtime["webhook-worker"].name, log_stream_name = "{instance_id}/stdout", timezone = "UTC" }, + { file_path = "/home/ubuntu/.pm2/logs/ona-email-webhook-worker-error.log", log_group_name = aws_cloudwatch_log_group.runtime["webhook-worker"].name, log_stream_name = "{instance_id}/stderr", timezone = "UTC" }, + ] } } } + })) + + webhook_metric_alarms = { + heartbeat = { + metric = "WebhookWorkerHeartbeat" + comparison = "LessThanThreshold" + threshold = 1 + periods = 2 + period = 60 + statistic = "Minimum" + description = "Webhook worker heartbeat absent or unhealthy for two minutes." + } + oldest-pending = { + metric = "OldestPendingEventAgeSeconds" + comparison = "GreaterThanThreshold" + threshold = 300 + periods = 2 + period = 60 + statistic = "Maximum" + description = "Oldest pending or retry webhook event exceeds five minutes." + } + unmatched-warning = { + metric = "OldestUnmatchedEventAgeSeconds" + comparison = "GreaterThanThreshold" + threshold = 3600 + periods = 2 + period = 60 + statistic = "Maximum" + description = "Oldest unmatched webhook event exceeds one hour." + } + unmatched-alarm = { + metric = "OldestUnmatchedEventAgeSeconds" + comparison = "GreaterThanThreshold" + threshold = 86400 + periods = 1 + period = 60 + statistic = "Maximum" + description = "Oldest unmatched webhook event exceeds 24 hours." + } + dead-letter = { + metric = "DeadLetterCount" + comparison = "GreaterThanOrEqualToThreshold" + threshold = 1 + periods = 1 + period = 60 + statistic = "Maximum" + description = "At least one webhook event is dead-lettered." + } + invalid-signature = { + metric = "InvalidSignatureCount" + comparison = "GreaterThanThreshold" + threshold = 10 + periods = 1 + period = 300 + statistic = "Sum" + description = "More than ten webhook signature failures occurred in five minutes." + missing = "notBreaching" + } + suppression-reconciliation = { + metric = "SuppressionReconciliationFailureCount" + comparison = "GreaterThanOrEqualToThreshold" + threshold = 1 + periods = 1 + period = 60 + statistic = "Sum" + description = "Suppression reconciliation failed." + missing = "notBreaching" + } + payload-purge = { + metric = "PayloadPurgeFailureCount" + comparison = "GreaterThanOrEqualToThreshold" + threshold = 1 + periods = 1 + period = 60 + statistic = "Sum" + description = "Expired raw webhook payload purge failed." + missing = "notBreaching" + } + canary = { + metric = "WebhookCanaryAgeSeconds" + comparison = "GreaterThanThreshold" + threshold = 64800 + periods = 1 + period = 3600 + statistic = "Maximum" + description = "No successful controlled provider webhook canary for 18 hours." + } + } +} + +resource "aws_cloudwatch_metric_alarm" "webhook" { + for_each = var.resend_webhook_ingest_enabled ? local.webhook_metric_alarms : {} + + alarm_name = "${var.name_prefix}-webhook-${each.key}" + alarm_description = each.value.description + namespace = var.webhook_metric_namespace + metric_name = each.value.metric + dimensions = { Environment = var.environment } + comparison_operator = each.value.comparison + threshold = each.value.threshold + evaluation_periods = each.value.periods + datapoints_to_alarm = each.value.periods + period = each.value.period + statistic = each.value.statistic + treat_missing_data = lookup(each.value, "missing", "breaching") + alarm_actions = [aws_sns_topic.operations_alerts.arn] + ok_actions = [aws_sns_topic.operations_alerts.arn] + tags = var.common_tags +} + +resource "aws_cloudwatch_metric_alarm" "uncertain_quota_disabled" { + count = var.resend_webhook_ingest_enabled ? 1 : 0 + + alarm_name = "${var.name_prefix}-email-uncertain-and-quota-disabled" + alarm_description = "At least one uncertain delivery exists while quota protection has disabled claiming." + comparison_operator = "GreaterThanOrEqualToThreshold" + threshold = 1 + evaluation_periods = 1 + datapoints_to_alarm = 1 + treat_missing_data = "breaching" + alarm_actions = [aws_sns_topic.operations_alerts.arn] + ok_actions = [aws_sns_topic.operations_alerts.arn] + tags = var.common_tags + + metric_query { + id = "both" + expression = "IF(uncertain >= 1, IF(quota >= 1, 1, 0), 0)" + label = "Uncertain delivery and quota disable" + return_data = true + } + + metric_query { + id = "uncertain" + metric { + namespace = var.webhook_metric_namespace + metric_name = "UncertainDeliveryCount" + period = 60 + stat = "Maximum" + dimensions = { Environment = var.environment } + } + } + + metric_query { + id = "quota" + metric { + namespace = var.webhook_metric_namespace + metric_name = "QuotaClaimingDisabled" + period = 60 + stat = "Maximum" + dimensions = { Environment = var.environment } + } + } +} + resource "aws_instance" "backend" { ami = data.aws_ami.ubuntu.id instance_type = var.instance_type @@ -316,10 +561,13 @@ resource "aws_instance" "backend" { key_name = length(var.ssh_allowed_cidrs) > 0 ? var.ssh_key_name : null user_data = templatefile(var.cloud_init_template_path, { - config_bucket = aws_s3_bucket.config_bucket.bucket - artifacts_bucket = aws_s3_bucket.artifacts.bucket - aws_region = var.aws_region - environment = var.environment + config_bucket = aws_s3_bucket.config_bucket.bucket + artifacts_bucket = aws_s3_bucket.artifacts.bucket + aws_region = var.aws_region + environment = var.environment + api_log_group = aws_cloudwatch_log_group.runtime["api"].name + email_worker_log_group = aws_cloudwatch_log_group.runtime["email-worker"].name + webhook_worker_log_group = aws_cloudwatch_log_group.runtime["webhook-worker"].name }) lifecycle { @@ -331,6 +579,32 @@ resource "aws_instance" "backend" { }) } +# Cloud-init does not rerun on existing instances. This association installs or +# refreshes observability in place during Terraform apply as well as on future +# replacement hosts. +resource "aws_ssm_association" "cloudwatch_agent" { + name = "AWS-RunShellScript" + association_name = "${trimsuffix(var.name_prefix, "-")}-cloudwatch-agent" + + targets { + key = "InstanceIds" + values = [aws_instance.backend.id] + } + + parameters = { + commands = join("\n", [ + "set -eu", + "if [ ! -x /opt/aws/amazon-cloudwatch-agent/bin/amazon-cloudwatch-agent-ctl ]; then curl -fsSL https://amazoncloudwatch-agent.s3.amazonaws.com/ubuntu/amd64/latest/amazon-cloudwatch-agent.deb -o /tmp/amazon-cloudwatch-agent.deb; dpkg -i /tmp/amazon-cloudwatch-agent.deb; rm -f /tmp/amazon-cloudwatch-agent.deb; fi", + "mkdir -p /opt/aws/amazon-cloudwatch-agent/etc", + "printf '%s' '${local.cloudwatch_agent_config_b64}' | base64 -d > /opt/aws/amazon-cloudwatch-agent/etc/amazon-cloudwatch-agent.json", + "/opt/aws/amazon-cloudwatch-agent/bin/amazon-cloudwatch-agent-ctl -a fetch-config -m ec2 -c file:/opt/aws/amazon-cloudwatch-agent/etc/amazon-cloudwatch-agent.json -s", + "/opt/aws/amazon-cloudwatch-agent/bin/amazon-cloudwatch-agent-ctl -a status", + ]) + } + + depends_on = [aws_iam_role_policy_attachment.ec2_s3_policy, aws_iam_role_policy_attachment.ec2_ssm_policy] +} + resource "aws_lb_target_group" "backend_targets" { name = var.target_group_name protocol = "HTTP" diff --git a/terraform/modules/api_backend/outputs.tf b/terraform/modules/api_backend/outputs.tf index bf99eb7..71dcf92 100644 --- a/terraform/modules/api_backend/outputs.tf +++ b/terraform/modules/api_backend/outputs.tf @@ -33,3 +33,13 @@ output "alb_arn" { output "target_group_arn" { value = aws_lb_target_group.backend_targets.arn } + +output "operations_alert_topic_arn" { + value = aws_sns_topic.operations_alerts.arn + description = "Environment-specific SNS topic used by runtime and webhook alarms." +} + +output "runtime_log_group_names" { + value = { for name, group in aws_cloudwatch_log_group.runtime : name => group.name } + description = "CloudWatch log groups for the API and dedicated workers." +} diff --git a/terraform/modules/api_backend/variables.tf b/terraform/modules/api_backend/variables.tf index c00149f..cd78c96 100644 --- a/terraform/modules/api_backend/variables.tf +++ b/terraform/modules/api_backend/variables.tf @@ -106,6 +106,63 @@ variable "resend_api_key_parameter_name" { type = string } +variable "resend_webhook_secret_parameter_name" { + description = "Environment-specific SSM parameter name for the primary Resend webhook signing secret. Terraform never manages the value." + type = string +} + +variable "resend_webhook_previous_secret_parameter_name" { + description = "Optional environment-specific SSM parameter name used only during signing-secret rotation overlap. Terraform never manages the value." + type = string + default = null +} + +variable "resend_provider_account_scope" { + description = "Stable non-secret scope shared by every environment using the same Resend account." + type = string + + validation { + condition = can(regex("^[A-Za-z0-9_-]{1,128}$", var.resend_provider_account_scope)) + error_message = "resend_provider_account_scope must be a stable 1-128 character identifier containing only letters, digits, underscore, or dash." + } +} + +variable "resend_webhook_ingest_enabled" { + description = "Exact-true release gate for webhook ingestion. Keep false until the disabled provider endpoint and SSM secret are reconciled." + type = bool + default = false +} + +variable "webhook_payload_retention_days" { + description = "Retention period for raw webhook payloads; metadata and projections are retained separately." + type = number + default = 30 + + validation { + condition = var.webhook_payload_retention_days >= 1 && var.webhook_payload_retention_days <= 365 + error_message = "webhook_payload_retention_days must be between 1 and 365." + } +} + +variable "cloudwatch_log_retention_days" { + description = "Retention for API and worker CloudWatch log groups." + type = number + default = 30 +} + +variable "operations_alert_email" { + description = "Optional operations email subscribed to the environment SNS alert topic. Confirm the AWS subscription before relying on alerts." + type = string + default = "bgarcia2324@gmail.com" + nullable = true +} + +variable "webhook_metric_namespace" { + description = "CloudWatch namespace used by webhook worker EMF metrics." + type = string + default = "NetworkSurvey/Webhooks" +} + variable "bootstrap_admin_username" { description = "Username for the deploy-time initial dashboard administrator. Null disables bootstrapping." type = string diff --git a/terraform/templates/env.tmpl b/terraform/templates/env.tmpl index 25efb5e..96c9dcb 100644 --- a/terraform/templates/env.tmpl +++ b/terraform/templates/env.tmpl @@ -15,6 +15,12 @@ LEGACY_START_ENABLED=${legacy_start_enabled} EMAIL_RATE_PER_SECOND=${email_rate_per_second} EMAIL_RATE_BUDGET_ENV=${email_rate_budget_environment} RESEND_API_KEY_PARAMETER=${resend_api_key_parameter_name} +RESEND_WEBHOOK_SECRET_PARAMETER=${resend_webhook_secret_parameter_name} +RESEND_WEBHOOK_PREVIOUS_SECRET_PARAMETER=${resend_webhook_previous_secret_parameter_name != null ? resend_webhook_previous_secret_parameter_name : ""} +RESEND_PROVIDER_ACCOUNT_SCOPE=${resend_provider_account_scope} +RESEND_WEBHOOK_INGEST_ENABLED=${resend_webhook_ingest_enabled} +WEBHOOK_PAYLOAD_RETENTION_DAYS=${webhook_payload_retention_days} +WEBHOOK_METRIC_NAMESPACE=${webhook_metric_namespace} CLA_PRODUCTION_CUTOVER=${cla_production_cutover} %{ if bootstrap_admin_password_parameter_name != null ~} BOOTSTRAP_ADMIN_USERNAME=${bootstrap_admin_username} diff --git a/terraform/variables.tf b/terraform/variables.tf index 8d09c1c..261273f 100644 --- a/terraform/variables.tf +++ b/terraform/variables.tf @@ -87,6 +87,19 @@ variable "artifact_retention_days" { default = 30 } +variable "resend_provider_account_scope" { + description = "Stable non-secret scope shared by environments using the same Resend team." + type = string + default = "network-survey-resend-team" +} + +variable "operations_alert_email" { + description = "Optional initial SNS email subscriber for infrastructure alarms." + type = string + default = "bgarcia2324@gmail.com" + nullable = true +} + variable "api_config_db_host_override" { description = "Optional DB host written to the API runtime config instead of this stack's RDS address. Temporary safety valve while prod DB ownership is split during the infra refactor. Leave null for normal environments." type = string