Add guarded CLA production data migration

.github/workflows/deploy.yml

@@ -135,7 +135,7 @@ jobs:
           mkdir -p "$STAGE_DIR/deploy" "$STAGE_DIR/db"
           rsync -a --exclude node_modules api "$STAGE_DIR/"
           cp -a db/changelogs "$STAGE_DIR/db/"
-          cp scripts/deploy/remote-deploy.sh scripts/deploy/bootstrap-admin.js "$STAGE_DIR/deploy/"
+          cp scripts/deploy/remote-deploy.sh scripts/deploy/bootstrap-admin.js scripts/deploy/finalize-legacy-accounts.js "$STAGE_DIR/deploy/"
           echo "$GITHUB_SHA" > "$STAGE_DIR/REVISION"
           tar -czf api-release.tar.gz -C "$STAGE_DIR" .
 

.github/workflows/terraform-apply.yml

@@ -15,7 +15,9 @@ permissions:
   contents: read
 
 concurrency:
-  group: terraform-apply-${{ inputs.environment }}
+  # Production infrastructure/config updates serialize with production API
+  # releases so deploy cannot read partially updated IAM or runtime config.
+  group: ${{ inputs.environment == 'prod' && 'api-release-production' || 'terraform-apply-staging' }}
   cancel-in-progress: false
 
 jobs:

api/test/security.test.js

@@ -1689,6 +1689,43 @@ test('remaining IAM migration adds audit, invite/reset, and non-destructive surv
   assert.doesNotMatch(remaining, /\bDROP\b|\bTRUNCATE\b|\bDELETE\s+FROM\b|ALTER\s+TABLE[\s\S]+DROP\s+COLUMN/i);
 });
 
+test('CLA organization migration preserves survey data and enforces stable child relationships', () => {
+  const changelog = fs.readFileSync(path.join(__dirname, '../../db/changelogs/master-changelog.xml'), 'utf8');
+  const cutoverChangelog = fs.readFileSync(path.join(__dirname, '../../db/changelogs/cla-production-cutover.xml'), 'utf8');
+  const migration = fs.readFileSync(path.join(__dirname, '../../db/changelogs/v1_7_cla_organization_backfill.sql'), 'utf8');
+  const bootstrap = fs.readFileSync(path.join(__dirname, '../../scripts/deploy/bootstrap-admin.js'), 'utf8');
+  const cleanup = fs.readFileSync(path.join(__dirname, '../../scripts/deploy/finalize-legacy-accounts.js'), 'utf8');
+
+  assert.doesNotMatch(changelog, /v1_7_cla_organization_backfill\.sql/);
+  assert.match(cutoverChangelog, /master-changelog\.xml/);
+  assert.match(cutoverChangelog, /v1_7_cla_organization_backfill\.sql/);
+  assert.match(migration, /VALUES \('CLA', 'cla'\)/);
+  assert.match(migration, /WHERE r\.survey_id IS NULL/);
+  assert.match(migration, /WHERE e\.survey_id IS NULL/);
+  assert.match(migration, /organization_id IS DISTINCT FROM/);
+  assert.match(migration, /Respondent contains null, orphaned, or disagreeing survey relationships/);
+  assert.match(migration, /EMAIL contains null, orphaned, or disagreeing survey relationships/);
+  assert.match(migration, /FOREIGN KEY \(survey_id\) REFERENCES Survey\(id\) NOT VALID/i);
+  assert.match(migration, /ALTER TABLE Respondent VALIDATE CONSTRAINT respondent_survey_id_fkey/i);
+  assert.match(migration, /ALTER TABLE EMAIL VALIDATE CONSTRAINT email_survey_id_fkey/i);
+  assert.doesNotMatch(migration, /UPDATE\s+Respondent[\s\S]+\b(response|uuid|respondent_id|email_sent)\s*=/i);
+  assert.doesNotMatch(migration, /UPDATE\s+EMAIL[\s\S]+\b(text|invitation_subject)\s*=/i);
+
+  assert.match(bootstrap, /BOOTSTRAP_ORGANIZATION_SLUG/);
+  assert.match(bootstrap, /BOOTSTRAP_PLATFORM_ADMIN/);
+  assert.match(bootstrap, /create-or-verify/);
+  assert.match(bootstrap, /bcrypt\.compare/);
+  assert.match(bootstrap, /created_by_user_id/);
+  assert.match(cleanup, /CLA owner-only access is not active and validated/);
+  assert.match(cleanup, /CLEANUP_MODE/);
+  assert.match(cleanup, /CONFIRM_FINAL_SNAPSHOT_ID/);
+  assert.match(cleanup, /EXPECTED_LEGACY_USER_IDS/);
+  assert.match(cleanup, /last_login_at/);
+  assert.match(cleanup, /SET status = 'disabled', is_platform_admin = false/);
+  assert.match(cleanup, /DELETE FROM sessions/);
+  assert.doesNotMatch(cleanup, /DELETE FROM users/);
+});
+
 test('password reset request stores only token hash and returns raw token only with explicit manual-delivery flag', async (t) => {
   const originalQuery = pool.query;
   const originalReturnDevTokens = process.env.RETURN_DEV_TOKENS;

db/changelogs/cla-production-cutover.xml

@@ -0,0 +1,13 @@
+<?xml version="1.0" encoding="UTF-8" standalone="no"?>
+<databaseChangeLog
+    xmlns="http://www.liquibase.org/xml/ns/dbchangelog"
+    xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+    xsi:schemaLocation="http://www.liquibase.org/xml/ns/dbchangelog
+    http://www.liquibase.org/xml/ns/dbchangelog/dbchangelog-4.10.xsd">
+
+    <!-- Normal additive product schema first, then the explicitly selected
+         one-time production survey-data consolidation. Local, CI, and staging
+         continue to use master-changelog.xml and cannot run the cutover. -->
+    <include file="master-changelog.xml" relativeToChangelogFile="true" />
+    <include file="v1_7_cla_organization_backfill.sql" relativeToChangelogFile="true" />
+</databaseChangeLog>

db/changelogs/v1_7_cla_organization_backfill.sql

@@ -0,0 +1,173 @@
+--liquibase formatted sql
+
+--changeset cladvisors:cla-organization-backfill-1 splitStatements:false
+--comment Reconcile legacy survey relationships and place the complete survey data space under the CLA organization without changing survey, respondent, token, response, or template identities.
+
+SET LOCAL lock_timeout = '10s';
+SET LOCAL statement_timeout = '10min';
+LOCK TABLE Survey, Respondent, EMAIL, audit_events, organization_memberships IN SHARE ROW EXCLUSIVE MODE;
+
+-- Create the explicitly approved production organization. Reusing the slug on a
+-- partially applied environment preserves its UUID and all references.
+INSERT INTO organizations (name, slug)
+VALUES ('CLA', 'cla')
+ON CONFLICT (slug) DO UPDATE
+SET name = EXCLUDED.name,
+    updated_at = CURRENT_TIMESTAMP,
+    archived_at = NULL;
+
+-- v1_2 normally supplies IDs. This defensive pass only assigns identities where
+-- none exist; existing IDs are never rewritten.
+UPDATE Survey
+SET id = gen_random_uuid()
+WHERE id IS NULL;
+
+DO $$
+BEGIN
+  IF NOT EXISTS (SELECT 1 FROM Survey) THEN
+    RAISE EXCEPTION 'CLA backfill refused: no surveys found in the target database';
+  END IF;
+
+  IF EXISTS (
+    SELECT 1
+    FROM Respondent r
+    LEFT JOIN Survey by_name ON by_name.name = r.survey_name
+    LEFT JOIN Survey by_id ON by_id.id = r.survey_id
+    WHERE r.survey_name IS NULL
+       OR by_name.id IS NULL
+       OR (r.survey_id IS NOT NULL AND by_id.id IS NULL)
+       OR (r.survey_id IS NOT NULL AND by_id.id <> by_name.id)
+  ) THEN
+    RAISE EXCEPTION 'CLA backfill refused: Respondent contains null, orphaned, or disagreeing survey relationships';
+  END IF;
+
+  IF EXISTS (
+    SELECT 1
+    FROM EMAIL e
+    LEFT JOIN Survey by_name ON by_name.name = e.survey_name
+    LEFT JOIN Survey by_id ON by_id.id = e.survey_id
+    WHERE e.survey_name IS NULL
+       OR by_name.id IS NULL
+       OR (e.survey_id IS NOT NULL AND by_id.id IS NULL)
+       OR (e.survey_id IS NOT NULL AND by_id.id <> by_name.id)
+  ) THEN
+    RAISE EXCEPTION 'CLA backfill refused: EMAIL contains null, orphaned, or disagreeing survey relationships';
+  END IF;
+
+  IF EXISTS (
+    SELECT 1
+    FROM Survey
+    WHERE archived_at IS NULL AND slug IS NOT NULL
+    GROUP BY slug
+    HAVING COUNT(*) > 1
+  ) THEN
+    RAISE EXCEPTION 'CLA backfill refused: active survey slugs would collide in the CLA organization';
+  END IF;
+
+  IF EXISTS (
+    SELECT 1
+    FROM audit_events ae
+    LEFT JOIN Survey s ON s.id = ae.survey_id
+    WHERE ae.survey_id IS NOT NULL AND s.id IS NULL
+  ) THEN
+    RAISE EXCEPTION 'CLA backfill refused: audit event contains an orphaned survey_id';
+  END IF;
+END $$;
+
+-- Fill only missing stable child references from the still-globally-unique
+-- legacy name relationship. Responses, respondent IDs/tokens, email state, and
+-- invitation contents are untouched.
+UPDATE Respondent r
+SET survey_id = s.id
+FROM Survey s
+WHERE r.survey_id IS NULL
+  AND r.survey_name = s.name;
+
+UPDATE EMAIL e
+SET survey_id = s.id
+FROM Survey s
+WHERE e.survey_id IS NULL
+  AND e.survey_name = s.name;
+
+-- Include active, archived, and demo surveys. Legacy creator fields deliberately
+-- remain unchanged/null because the old schema did not record ownership.
+UPDATE Survey
+SET organization_id = (SELECT id FROM organizations WHERE slug = 'cla')
+WHERE organization_id IS DISTINCT FROM (SELECT id FROM organizations WHERE slug = 'cla');
+
+-- Maintain access between the committed data move and deploy-time creation of the
+-- explicitly approved CLA owner. These transitional memberships are removed only
+-- by the separately gated post-login cleanup.
+INSERT INTO organization_memberships (organization_id, user_id, role)
+SELECT o.id, u.id, 'owner'
+FROM organizations o
+CROSS JOIN users u
+WHERE o.slug = 'cla'
+ON CONFLICT (organization_id, user_id) DO NOTHING;
+
+-- Keep survey-scoped audit records internally consistent without changing their
+-- IDs, actors, event types, metadata, or timestamps.
+UPDATE audit_events ae
+SET organization_id = s.organization_id
+FROM Survey s
+WHERE ae.survey_id = s.id
+  AND ae.organization_id IS DISTINCT FROM s.organization_id;
+
+DO $$
+BEGIN
+  IF EXISTS (
+    SELECT 1 FROM Survey s
+    CROSS JOIN organizations o
+    WHERE o.slug = 'cla'
+      AND (s.id IS NULL OR s.organization_id IS DISTINCT FROM o.id)
+  ) THEN
+    RAISE EXCEPTION 'CLA backfill refused: Survey stable identity or CLA organization assignment is incomplete';
+  END IF;
+  IF EXISTS (
+    SELECT 1 FROM Respondent r
+    LEFT JOIN Survey s ON s.id = r.survey_id
+    WHERE r.survey_id IS NULL OR s.id IS NULL OR r.survey_name IS DISTINCT FROM s.name
+  ) THEN
+    RAISE EXCEPTION 'CLA backfill refused: Respondent stable and legacy survey relationships do not reconcile';
+  END IF;
+  IF EXISTS (
+    SELECT 1 FROM EMAIL e
+    LEFT JOIN Survey s ON s.id = e.survey_id
+    WHERE e.survey_id IS NULL OR s.id IS NULL OR e.survey_name IS DISTINCT FROM s.name
+  ) THEN
+    RAISE EXCEPTION 'CLA backfill refused: EMAIL stable and legacy survey relationships do not reconcile';
+  END IF;
+
+  IF NOT EXISTS (
+    SELECT 1 FROM pg_constraint
+    WHERE conname = 'survey_id_key' AND conrelid = 'survey'::regclass
+  ) THEN
+    ALTER TABLE Survey ADD CONSTRAINT survey_id_key UNIQUE (id);
+  END IF;
+
+  IF NOT EXISTS (
+    SELECT 1 FROM pg_constraint
+    WHERE conname = 'respondent_survey_id_fkey' AND conrelid = 'respondent'::regclass
+  ) THEN
+    ALTER TABLE Respondent
+      ADD CONSTRAINT respondent_survey_id_fkey
+      FOREIGN KEY (survey_id) REFERENCES Survey(id) NOT VALID;
+  END IF;
+
+  IF NOT EXISTS (
+    SELECT 1 FROM pg_constraint
+    WHERE conname = 'email_survey_id_fkey' AND conrelid = 'email'::regclass
+  ) THEN
+    ALTER TABLE EMAIL
+      ADD CONSTRAINT email_survey_id_fkey
+      FOREIGN KEY (survey_id) REFERENCES Survey(id) NOT VALID;
+  END IF;
+END $$;
+
+ALTER TABLE Survey ALTER COLUMN id SET NOT NULL;
+ALTER TABLE Survey ALTER COLUMN organization_id SET NOT NULL;
+ALTER TABLE Respondent ALTER COLUMN survey_id SET NOT NULL;
+ALTER TABLE EMAIL ALTER COLUMN survey_id SET NOT NULL;
+
+ALTER TABLE Respondent VALIDATE CONSTRAINT respondent_survey_id_fkey;
+ALTER TABLE EMAIL VALIDATE CONSTRAINT email_survey_id_fkey;

docs/runbooks/cla-production-migration.md

@@ -0,0 +1,119 @@
+# CLA production survey-data migration runbook
+
+Target application baseline: `a379f101aa4a722d6fc6fab7cdbb092547b0c717`
+
+Recorded pre-cutover API release: `c22e3031787c2403b5e9174bdd9a71385c481dde`
+
+This cutover moves the complete legacy survey data space into organization `CLA`
+(slug `cla`) while preserving survey/respondent identities, bearer tokens,
+responses, invitation bodies/subjects, archive state, and email-send state.
+Legacy dashboard users are retained as disabled rows after cutover but lose
+memberships and active sessions.
+
+## Recorded production preflight
+
+- RDS: `network-survey-prod-postgres-v2`, PostgreSQL 15.18, encrypted and deletion-protected
+- Final snapshot: `network-survey-prod-postgres-v2-pre-cla-20260804191658`
+- Snapshot status at creation: `available`
+- Database: `ONA`
+- Surveys: 11
+- Respondents: 113
+- Stored responses: 21
+- Respondents marked emailed: 80
+- Email templates: 111
+- Legacy users: IDs `1,2` (`Admin`, `Admin1`)
+- Sessions: one for each legacy user at preflight
+- Null survey names/tokens: 0
+- Duplicate non-null respondent tokens: 0
+- Legacy respondent/email orphans: 0
+- Pre-migration digests:
+  - Survey: `ad95ee6ecd6e4d201dcb69f2d7ede646`
+  - Respondent/response: `4f68353e26d6cc383b3680cb03a31ad4`
+  - Email: `40ca155d5df7c78dc1ef2b005318ad99`
+
+The survey digest is expected to change because v1.4–v1.6 add/backfill survey
+slugs/display names, materialize SurveyJS `isRequired:false`, and add invitation
+subjects. Respondent/response payloads and legacy email bodies must reconcile.
+
+## Execution order
+
+1. Confirm application downtime and no active database writers.
+2. Confirm the final snapshot above is still `available`.
+3. Confirm `/network-survey/prod/api/bootstrap-admin-password` exists as a
+   SecureString and the production API instance can decrypt it without printing it.
+4. Apply the reviewed production Terraform plan with
+   `enable_cla_production_cutover=true` and `enable_cla_owner_bootstrap=true`.
+   It must contain only in-place runtime IAM policy and config-object changes; no
+   destroys or replacements. This one-time apply was completed before cutover.
+5. Deploy the reviewed release with `CLA_PRODUCTION_CUTOVER=true`. This selects
+   `cla-production-cutover.xml`; the universal local/CI/staging master changelog
+   cannot execute `v1_7`. Liquibase aborts on null/orphaned/disagreeing child
+   relationships, active slug collisions, orphaned audit survey IDs, or an empty
+   survey set. Bootstrap `create-or-verify` mode is retry-safe: it creates the
+   approved owner once and subsequently requires exact identity and credential.
+6. Confirm external API health, then authenticate as the new CLA owner and verify
+   survey listing/results. Login updates `users.last_login_at` and is required by
+   cleanup.
+7. Run `finalize-legacy-accounts.js` in `dry-run` mode with the exact snapshot ID,
+   counts, and legacy user IDs above. Review output.
+8. Repeat in `apply` mode with `CONFIRM_FINAL_SNAPSHOT_ID` exactly matching the
+   recorded snapshot.
+9. Run post-migration reconciliation and respondent-link smoke tests.
+10. Remove the one-time production bootstrap config/IAM access and rotate/delete
+    the bootstrap SecureString after the owner password is rotated.
+
+## Cleanup invocation environment
+
+Run on the production API instance from the active release with DB variables loaded
+from runtime config. Required non-secret controls:
+
+```text
+CLA_OWNER_USERNAME=sgarcia@cladvisors.com
+CLA_ORGANIZATION_SLUG=cla
+EXPECTED_DB_NAME=ONA
+EXPECTED_SURVEY_COUNT=11
+EXPECTED_RESPONDENT_COUNT=113
+EXPECTED_EMAIL_COUNT=111
+EXPECTED_LEGACY_USER_IDS=1,2
+FINAL_SNAPSHOT_ID=network-survey-prod-postgres-v2-pre-cla-20260804191658
+CLA_CUTOVER_STARTED_AT=<ISO timestamp before owner login>
+CLEANUP_MODE=dry-run|apply
+CONFIRM_FINAL_SNAPSHOT_ID=<required in apply mode>
+```
+
+Never place the owner password or decrypted database password in this runbook,
+command logs, Terraform variables, or repository files.
+
+## Post-migration acceptance
+
+- Exactly 11 surveys belong to CLA, including archived/demo surveys.
+- Exactly 113 respondents and 21 non-null responses remain.
+- Exactly 80 respondents remain marked emailed.
+- Exactly 111 templates have stable survey IDs and non-null invitation subjects.
+- No null/orphaned/disagreeing respondent or email survey relationships exist.
+- Respondent IDs, tokens, response JSON, contact data, and legacy email bodies are unchanged.
+- `sgarcia@cladvisors.com` is active CLA owner and not platform admin.
+- Legacy users 1 and 2 are disabled, have no memberships, and have no sessions.
+
+## Rollback
+
+Application artifact rollback does not reverse this data migration. If validation
+fails after commit:
+
+1. Restore snapshot `network-survey-prod-postgres-v2-pre-cla-20260804191658` to a
+   unique RDS identifier in subnet group `db-subnet-group`, attaching security
+   group `sg-00d61e181de4cfb48`, with public access disabled.
+2. Wait for `available`, record the restored endpoint, and verify TLS/connectivity
+   from production instance `i-065f1e1f497ab1481`.
+3. Set Terraform variable `api_config_db_host_override` to that endpoint; leave
+   `enable_cla_production_cutover=false` and `enable_cla_owner_bootstrap=false`.
+   Apply only the reviewed runtime config/IAM changes.
+4. Redeploy recorded pre-cutover artifact
+   `c22e3031787c2403b5e9174bdd9a71385c481dde`; its historical changelog must not
+   execute the CLA cutover.
+5. Verify health plus the recorded 11/113/111 counts and pre-migration digests.
+6. Preserve the migrated database for forensic comparison. To return, clear
+   `api_config_db_host_override`, review the plan, apply, and redeploy the intended
+   migrated artifact.
+
+Do not overwrite either database.

scripts/ci/api-smoke.sh

@@ -88,7 +88,7 @@ curl -fsS -b "$COOKIES" "$BASE/api/check-auth" | grep -q '"isAuthenticated":true
 echo "==> Authenticated survey CRUD"
 curl -fsS -b "$COOKIES" -X POST "$BASE/api/survey" \
   -H 'Content-Type: application/json' \
-  -d '{"surveyName":"ci-smoke-survey"}' >/dev/null
-curl -fsS -b "$COOKIES" "$BASE/api/surveys" | grep -q 'ci-smoke-survey'
+  -d '{"surveyName":"CISmokeSurvey"}' >/dev/null
+curl -fsS -b "$COOKIES" "$BASE/api/surveys" | grep -q 'CISmokeSurvey'
 
 echo "==> Smoke test passed"