diff --git a/scripts/import-db-dump.sh b/scripts/import-db-dump.sh new file mode 100755 index 000000000..82b9336e5 --- /dev/null +++ b/scripts/import-db-dump.sh @@ -0,0 +1,470 @@ +#!/usr/bin/env bash +set -euo pipefail + +SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) +PROJECT_ROOT=$( dirname "${SCRIPT_DIR}" ) + +usage() { + local USAGE + USAGE=" +Usage: $(basename "${0}") [options] + +Loads a per-table Postgres COPY-TEXT dump (see scrub-db-dump.sh) into a Postgres instance - e.g. +the one 'docker compose --profile db up postgres' starts, matching +server/src/dev/resources/application.yml's connection details, which are this script's defaults. +The target can be either completely empty or already fully migrated - see below. + +Deletes all existing rows from the 8 tables the dump covers first (children before parents, so no +foreign key elsewhere in that same set gets violated), then imports every dump file and fixes up +each table's id sequence so the running app doesn't collide with the imported ids on its next +insert. The whole delete+import runs as ONE transaction: if anything fails partway through - +including a table outside these 8 that has its own foreign key into one of them, which a plain +DELETE (deliberately not CASCADE) will refuse to run past - nothing changes on the target at all. + +This does NOT figure out which Flyway version the dump predates (scrub-db-dump.sh already needed +to for its own reasons - if you still have that terminal output, reuse it here with -t). It uses +whatever's given via -t, or migrates a throwaway scratch container through the full history to +detect it, purely to learn each table's dump-era column *names*. + +What happens next depends on the target's own state, checked by looking for these 8 tables (not +by table count, so unrelated tables - e.g. spring_session - don't confuse it): + - None of the 8 exist: the target is treated as a fresh database and migrated directly, via a + real 'flyway migrate -target=' run against it, to the dump's own Flyway + version - not to the latest one. Afterwards its schema is an exact match for the dump, so + there's nothing to adapt. If you want the target on the current schema afterwards (e.g. to run + the app against it), run 'flyway migrate' (no -target) yourself once this script is done. + This migrate also applies server/src/dev/resources/db/migration (the 'dev' Gradle source + set's own migrations, normally only applied by running the app locally that way) alongside + the main ones, so e.g. V1_62_1's rate-limit tier/customer seed data ends up present too, the + same as a real local dev run would have it. The one exception is the 'super_user' / + 'super_token' seed (V1_0_1 + V1_1_1): it touches user_data/personal_access_token, so the + DELETE+import below wipes it regardless of having run - deliberately not reseeded afterwards + (its own migrations' hardcoded id isn't guaranteed free in real dump data, and it's not worth + reconstructing outside of a real flyway migrate just for this). + - All 8 already exist: the target is assumed to already be on the current schema (the app or + 'flyway migrate' got it there), same as before. The dump's old column layout is adapted to fit + via an explicit column list per \\copy, so newer columns the target has gained since simply + keep their column default (with known exceptions backfilled - see KNOWN_BACKFILLS below). + - Some but not all 8 exist: refused - this is neither state above, and guessing which columns + are missing on a half-migrated schema is more likely to corrupt data than help. +Either way, the target is only ever touched starting at the DELETE+import step (or, for a fresh +target, the flyway migrate right before it). + +Options: + -d Dump directory (default: db/dump under the repo root) + -t Skip auto-detection; the dump is from this Flyway version (e.g. 1.71) + -H Target Postgres host (default: localhost) + -p Target Postgres port (default: 5432) + -U Target Postgres user (default: openvsx) + -W Target Postgres password (default: openvsx) + -n Target Postgres database name (default: postgres) + -y Skip the confirmation prompt + -h show this help +" + echo "$USAGE" + exit 1 +} + +DUMP_DIR="${PROJECT_ROOT}/db/dump" +FLYWAY_TARGET="" +TARGET_HOST="localhost" +TARGET_PORT="5432" +TARGET_USER="openvsx" +TARGET_PASSWORD="openvsx" +TARGET_DB="postgres" +ASSUME_YES=false + +while getopts "d:t:H:p:U:W:n:yh" o; do + case "${o}" in + d) DUMP_DIR="${OPTARG}" ;; + t) FLYWAY_TARGET="${OPTARG}" ;; + H) TARGET_HOST="${OPTARG}" ;; + p) TARGET_PORT="${OPTARG}" ;; + U) TARGET_USER="${OPTARG}" ;; + W) TARGET_PASSWORD="${OPTARG}" ;; + n) TARGET_DB="${OPTARG}" ;; + y) ASSUME_YES=true ;; + *) usage ;; + esac +done + +for tool in docker python3; do + command -v "${tool}" >/dev/null || { echo "This script needs '${tool}' on PATH." >&2; exit 1; } +done + +MIGRATIONS_DIR="${PROJECT_ROOT}/server/src/main/resources/db/migration" +# The 'dev' Gradle source set's own migrations (server/build.gradle) - not part of the real app's +# jar, only ever applied when running the app via that source set locally. Interleaved by version +# among the main ones above (e.g. V1_0_1, V1_1_1 sort between main's V1 and V1_1). Used below only +# for a freshly-migrated target, to match what running the app locally would have applied. +DEV_MIGRATIONS_DIR="${PROJECT_ROOT}/server/src/dev/resources/db/migration" +FLYWAY_IMAGE="flyway/flyway:10" +POSTGRES_IMAGE="postgres:16.2" +RUN_ID="import-$$" +SCRATCH_CONTAINER="openvsx-${RUN_ID}" +SCRATCH_NETWORK="openvsx-${RUN_ID}-net" +COPY_FORMAT="format text, delimiter ','" + +# All 8 tables scrub-db-dump.sh's dump files cover, dependency order computed dynamically below +# (do not assume it here - the target's actual constraints are authoritative). +TABLES=(extension extension_version file_resource namespace namespace_membership personal_access_token signature_key_pair user_data) + +for t in "${TABLES[@]}"; do + if [ ! -f "${DUMP_DIR}/${t}.csv" ]; then + echo "Missing ${DUMP_DIR}/${t}.csv" >&2 + exit 1 + fi +done + +WORKDIR=$(mktemp -d) +cleanup() { + docker rm -f "${SCRATCH_CONTAINER}" >/dev/null 2>&1 || true + docker network rm "${SCRATCH_NETWORK}" >/dev/null 2>&1 || true + rm -rf "${WORKDIR}" +} +trap cleanup EXIT + +scratch_psql() { + docker exec -i "${SCRATCH_CONTAINER}" psql -U openvsx -d postgres -v ON_ERROR_STOP=1 "$@" +} + +# --- "localhost" (the default, and what naturally refers to the caller's own machine) means +# something different from inside a container, so a throwaway container needs it rewritten to +# Docker's host.docker.internal - resolved via --add-host=host.docker.internal:host-gateway, +# which Docker Desktop supports natively and Linux's Docker Engine (20.10+) supports via that +# flag. Deliberately NOT --network host: that fails outright ("cannot share the host's network +# namespace") wherever user namespaces are enabled, which is common enough not to assume against. +# Any other host (a LAN address, a docker-compose service name on a network already joined, ...) +# is used as given - it means the same thing inside a container as outside one. +target_docker_host() { + case "${TARGET_HOST}" in + localhost|127.0.0.1|::1) echo "host.docker.internal" ;; + *) echo "${TARGET_HOST}" ;; + esac +} + +# --- Run the psql client against the target, either the host's own psql or, if that isn't +# installed, a throwaway containerized one (see target_docker_host above for how it reaches the +# target). +target_psql() { + if command -v psql >/dev/null 2>&1; then + PGPASSWORD="${TARGET_PASSWORD}" psql -h "${TARGET_HOST}" -p "${TARGET_PORT}" -U "${TARGET_USER}" -d "${TARGET_DB}" -v ON_ERROR_STOP=1 "$@" + else + # -f references a file under WORKDIR; -c/\copy reference dump files under DUMP_DIR - both + # need to be visible inside the container at the same absolute path the SQL text uses. + docker run --rm -i --add-host=host.docker.internal:host-gateway \ + -v "${DUMP_DIR}:${DUMP_DIR}:ro" -v "${WORKDIR}:${WORKDIR}" \ + -e PGPASSWORD="${TARGET_PASSWORD}" "${POSTGRES_IMAGE}" \ + psql -h "$(target_docker_host)" -p "${TARGET_PORT}" -U "${TARGET_USER}" -d "${TARGET_DB}" -v ON_ERROR_STOP=1 "$@" + fi +} + +# --- Migrate the TARGET itself (not the scratch container) directly to a given Flyway version, +# for the "target is empty" case below. Includes the dev migrations (see DEV_MIGRATIONS_DIR above) +# alongside the main ones, if that directory exists, so a fresh target ends up the same way +# running the app locally would have left it - including e.g. V1_62_1's rate-limit tier/customer +# seed data, which isn't part of any of these 8 dump tables so survives the import step untouched. +# The one exception is V1_0_1/V1_1_1's 'super_user' seed, which DOES touch two of these 8 tables +# (user_data, personal_access_token) and so gets wiped by the DELETE+import below regardless of +# having been applied here - deliberately left that way, not reseeded afterwards. +target_flyway_migrate() { + local target="$1" + local mounts=(-v "${MIGRATIONS_DIR}:/flyway/sql/main:ro") + local locations="filesystem:/flyway/sql/main" + if [ -d "${DEV_MIGRATIONS_DIR}" ]; then + mounts+=(-v "${DEV_MIGRATIONS_DIR}:/flyway/sql/dev:ro") + locations="${locations},filesystem:/flyway/sql/dev" + fi + docker run --rm --add-host=host.docker.internal:host-gateway "${mounts[@]}" "${FLYWAY_IMAGE}" \ + -url="jdbc:postgresql://$(target_docker_host):${TARGET_PORT}/${TARGET_DB}" \ + -user="${TARGET_USER}" -password="${TARGET_PASSWORD}" -locations="${locations}" \ + -connectRetries=10 ${target:+-target="${target}"} migrate >/dev/null +} + +echo "Checking the target is reachable (${TARGET_HOST}:${TARGET_PORT}/${TARGET_DB})..." +target_psql -c "select 1;" >/dev/null + +# --- Learn the dump's column names per table, without ever touching the target for this part. +echo "Starting a scratch Postgres to read the dump's schema..." +docker network create "${SCRATCH_NETWORK}" >/dev/null +docker run -d --name "${SCRATCH_CONTAINER}" --network "${SCRATCH_NETWORK}" \ + -e POSTGRES_USER=openvsx -e POSTGRES_PASSWORD=openvsx "${POSTGRES_IMAGE}" >/dev/null +until docker exec "${SCRATCH_CONTAINER}" pg_isready -U openvsx >/dev/null 2>&1; do sleep 1; done + +scratch_migrate() { + local target="$1" + docker run --rm --network "${SCRATCH_NETWORK}" -v "${MIGRATIONS_DIR}:/flyway/sql" "${FLYWAY_IMAGE}" \ + -url="jdbc:postgresql://${SCRATCH_CONTAINER}:5432/postgres" -user=openvsx -password=openvsx \ + -connectRetries=10 ${target:+-target="${target}"} migrate >/dev/null +} + +scratch_reset() { + docker exec "${SCRATCH_CONTAINER}" psql -U openvsx -d postgres -c \ + "DROP SCHEMA public CASCADE; CREATE SCHEMA public;" >/dev/null +} + +table_column_count() { + scratch_psql -t -c "select count(*) from information_schema.columns where table_schema='public' and table_name='${1}';" | tr -d ' ' +} + +table_column_list() { + scratch_psql -t -c "select string_agg(column_name, ',' order by ordinal_position) from information_schema.columns where table_schema='public' and table_name='${1}';" | tr -d ' ' +} + +if [ -z "${FLYWAY_TARGET}" ]; then + echo "No -t given, auto-detecting the Flyway version this dump matches..." + + detect_actual_cols() { + python3 - "${DUMP_DIR}/${1}.csv" <<'PY' +import csv, sys, collections +counts = collections.Counter() +with open(sys.argv[1], newline='', encoding='utf-8', errors='replace') as f: + # Postgres TEXT format has no quoting at all (unlike CSV) - only backslash-escaping of the + # delimiter, backslash itself, and embedded newlines/carriage returns. QUOTE_NONE keeps a + # literal '"' in a field (e.g. inside README/HTML content) from being misread as a quote. + reader = csv.reader(f, delimiter=',', escapechar='\\', quoting=csv.QUOTE_NONE) + for i, row in enumerate(reader): + if i >= 2000: + break + counts[len(row)] += 1 +print(counts.most_common(1)[0][0]) +PY + } + + declare -A actual_cols + for t in "${TABLES[@]}"; do + actual_cols["${t}"]=$(detect_actual_cols "${t}") + done + echo " dump column counts: $(for t in "${TABLES[@]}"; do printf '%s=%s ' "${t}" "${actual_cols[${t}]}"; done)" + + mapfile -t VERSIONS < <( + find "${MIGRATIONS_DIR}" -maxdepth 1 -name 'V*__*' -printf '%f\n' \ + | sed -E 's/^V([0-9_]+)__.*/\1/; s/_/./' \ + | sort -t. -k1,1n -k2,2n + ) + + scratch_reset + for v in "${VERSIONS[@]}"; do + scratch_migrate "${v}" + match=true + for t in "${TABLES[@]}"; do + if [ "$(table_column_count "${t}")" != "${actual_cols[${t}]}" ]; then + match=false + break + fi + done + if [ "${match}" = true ]; then + FLYWAY_TARGET="${v}" + break + fi + done + + if [ -z "${FLYWAY_TARGET}" ]; then + echo "Could not find a single Flyway version matching all 8 tables' column counts." >&2 + echo "Pass -t to pin a version yourself." >&2 + exit 1 + fi + echo " detected Flyway target: ${FLYWAY_TARGET}" +else + scratch_reset + scratch_migrate "${FLYWAY_TARGET}" +fi + +declare -A COLUMN_LISTS +for t in "${TABLES[@]}"; do + COLUMN_LISTS["${t}"]=$(table_column_list "${t}") + echo " ${t}: (${COLUMN_LISTS[${t}]})" +done + +# --- Decide whether the target is a fresh database (none of the 8 tables exist yet) or one +# that's presumably already fully migrated (all 8 exist). Checked by table presence, not a bare +# table count, so other tables the target happens to have (spring_session, flyway_schema_history, +# ...) don't affect the decision either way. +echo "Checking whether the target already has these 8 tables..." +present=0 +declare -a MISSING_TABLES=() +for t in "${TABLES[@]}"; do + exists=$(target_psql -t -c "select (to_regclass('public.${t}') is not null);" | tr -d ' ') + if [ "${exists}" = "t" ]; then + present=$((present + 1)) + else + MISSING_TABLES+=("${t}") + fi +done + +if [ "${present}" -eq 0 ]; then + echo " none present - treating the target as a fresh database." + echo "Migrating the target directly to Flyway version ${FLYWAY_TARGET} (the dump's own schema -" + echo "not necessarily the latest one; run 'flyway migrate' yourself afterwards for that)..." + target_flyway_migrate "${FLYWAY_TARGET}" +elif [ "${#MISSING_TABLES[@]}" -eq 0 ]; then + echo " all present - assuming the target is already on the current schema." +else + echo "" >&2 + echo "The target has some but not all of these 8 tables - missing: ${MISSING_TABLES[*]}." >&2 + echo "That's neither a fresh database nor a fully migrated one, so this script won't guess." >&2 + echo "Point it at an empty database, or one already fully migrated via 'flyway migrate'." >&2 + exit 1 +fi + +# --- A column the target has gained since the dump's era, if it's NOT NULL with no default, +# can't just be left out of the \copy column list - Postgres would reject every imported row. +# Known cases get a backfill: the constraint is dropped, the import runs, the same backfill the +# introducing migration itself used runs, then the constraint is restored - all still inside the +# one transaction. Add an entry here (keyed "table.column") for anything new, matching whatever +# UPDATE the migration that introduced the column used for pre-existing rows; a gap with no entry +# here fails the preflight check below instead of guessing. +declare -A KNOWN_BACKFILLS +KNOWN_BACKFILLS["personal_access_token.version"]="UPDATE personal_access_token SET version = 0;" +KNOWN_BACKFILLS["personal_access_token.type"]=" + UPDATE personal_access_token SET type = 'OTT' WHERE description = 'One time use publish token'; + UPDATE personal_access_token SET type = 'LLT' WHERE type IS NULL; +" + +echo "Checking for target columns the dump doesn't have that require a value..." +declare -A GAP_COLUMNS # "table.column" -> 1, for every gap found (known or not) +declare -a UNKNOWN_GAPS=() +for t in "${TABLES[@]}"; do + IFS=',' read -r -a dump_cols <<< "${COLUMN_LISTS[${t}]}" + gap_cols=$(target_psql -t -c " + select column_name from information_schema.columns + where table_schema='public' and table_name='${t}' + and is_nullable='NO' and column_default is null + and column_name not in ($(printf "'%s'," "${dump_cols[@]}" | sed 's/,$//')); + " | tr -d ' ' | grep -v '^$' || true) + while IFS= read -r col; do + [ -z "${col}" ] && continue + key="${t}.${col}" + GAP_COLUMNS["${key}"]=1 + if [ -z "${KNOWN_BACKFILLS[${key}]:-}" ]; then + UNKNOWN_GAPS+=("${key}") + else + echo " ${key}: no value in the dump, not nullable, no default - will backfill (known)" + fi + done <<< "${gap_cols}" +done + +if [ "${#UNKNOWN_GAPS[@]}" -gt 0 ]; then + echo "" >&2 + echo "These target columns are NOT NULL with no default, have no value in the dump, and have" >&2 + echo "no known backfill in this script's KNOWN_BACKFILLS table:" >&2 + printf ' %s\n' "${UNKNOWN_GAPS[@]}" >&2 + echo "" >&2 + echo "Add an entry for each, matching whatever UPDATE the migration that introduced the column" >&2 + echo "used to backfill pre-existing rows (check server/src/main/resources/db/migration). Nothing" >&2 + echo "has been touched on the target." >&2 + exit 1 +fi + +# --- Compute a safe delete order from the TARGET's actual foreign keys among these 8 tables - +# not assumed, and not necessarily the dump-era graph, since the target may be on a newer schema +# than the dump (the "already fully migrated" case above; a freshly-migrated target's graph is the +# dump-era one exactly, which this reads just as well). A table can be deleted once nothing else +# still-undeleted references it; +# any foreign key from OUTSIDE this set of 8 into one of them is deliberately left for Postgres +# to enforce (see the module docstring above on why this isn't CASCADE). +echo "Computing a safe delete order from the target's current foreign keys..." +edges=$(target_psql -t -A -F',' -c " + select tc.table_name, ccu.table_name + from information_schema.table_constraints tc + join information_schema.key_column_usage kcu on tc.constraint_name = kcu.constraint_name + join information_schema.constraint_column_usage ccu on tc.constraint_name = ccu.constraint_name + where tc.constraint_type = 'FOREIGN KEY' + and tc.table_name in ($(printf "'%s'," "${TABLES[@]}" | sed 's/,$//')) + and ccu.table_name in ($(printf "'%s'," "${TABLES[@]}" | sed 's/,$//')) + and tc.table_name != ccu.table_name; +") + +DELETE_ORDER=$(python3 - "${TABLES[*]}" <= 0; i--)); do + IMPORT_ORDER_ARR+=("${DELETE_ORDER_ARR[i]}") +done +echo " delete order: ${DELETE_ORDER_ARR[*]}" +echo " import order: ${IMPORT_ORDER_ARR[*]}" + +echo "" +echo "About to DELETE ALL ROWS from these tables in ${TARGET_USER}@${TARGET_HOST}:${TARGET_PORT}/${TARGET_DB}:" +printf ' %s\n' "${TABLES[@]}" +echo "and reload them from ${DUMP_DIR}." +if [ "${ASSUME_YES}" = false ]; then + read -r -p "Continue? [y/N] " reply + case "${reply}" in + [yY]|[yY][eE][sS]) ;; + *) echo "Aborted, nothing changed."; exit 1 ;; + esac +fi + +SQL_FILE="${WORKDIR}/reload.sql" +{ + echo "BEGIN;" + for t in "${DELETE_ORDER_ARR[@]}"; do + [ -n "${t}" ] && echo "DELETE FROM ${t};" + done + for t in "${IMPORT_ORDER_ARR[@]}"; do + gap_cols_for_table=() + for key in "${!GAP_COLUMNS[@]}"; do + [ "${key%%.*}" = "${t}" ] && gap_cols_for_table+=("${key#*.}") + done + for col in "${gap_cols_for_table[@]}"; do + echo "ALTER TABLE ${t} ALTER COLUMN ${col} DROP NOT NULL;" + done + echo "\\copy ${t} (${COLUMN_LISTS[${t}]}) from '${DUMP_DIR}/${t}.csv' with (${COPY_FORMAT})" + for col in "${gap_cols_for_table[@]}"; do + echo "${KNOWN_BACKFILLS[${t}.${col}]}" + echo "ALTER TABLE ${t} ALTER COLUMN ${col} SET NOT NULL;" + done + done + for t in "${TABLES[@]}"; do + # Not every table's id is sequence-backed (or has an "id" column at all, though all 8 here + # do) - pg_get_serial_sequence returns NULL rather than erroring for those, so the setval + # call is skipped for them via the WHERE-less guard below. + cat < "${SQL_FILE}" + +echo "Reloading (single transaction - any failure rolls everything back)..." +target_psql -f "${SQL_FILE}" + +echo "" +echo "Done. Row counts now on the target:" +target_psql -t -c "$(for t in "${TABLES[@]}"; do echo "select '${t}: ' || count(*) from ${t};"; done)" diff --git a/scripts/scrub-db-dump.sh b/scripts/scrub-db-dump.sh new file mode 100755 index 000000000..b9a15b4e5 --- /dev/null +++ b/scripts/scrub-db-dump.sh @@ -0,0 +1,260 @@ +#!/usr/bin/env bash +set -euo pipefail + +SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) +PROJECT_ROOT=$( dirname "${SCRIPT_DIR}" ) + +usage() { + local USAGE + USAGE=" +Usage: $(basename "${0}") [options] + +Redacts sensitive columns from a per-table Postgres COPY-TEXT dump (as produced by +'\\copy to ... with (format text, delimiter '\',\'')') before it's shared or used for +local dev/testing. Only touches personal_access_token.csv, signature_key_pair.csv and +user_data.csv - the other dump files have no columns in the redaction list below and are left +untouched. + +What gets redacted (see the REDACTION step further down to change this): + personal_access_token.value -> deterministic fake hash, unique per row + signature_key_pair.private_key -> 32 zero bytes (this column is NOT NULL) + user_data.email -> synthetic, unique per row + user_data.full_name -> synthetic, unique per row + user_data.avatar_url -> cleared + user_data.auth_id -> synthetic, unique per row (only where already non-null) + user_data.eclipse_token -> cleared (has held raw OAuth access/refresh token JSON) + user_data.eclipse_person_id -> cleared + user_data.login_name is left untouched - it's the public registry username already. + +How it works: this dump's exact column layout depends on which Flyway migration was HEAD when +it was taken (columns get added/dropped over time, and Postgres never reuses a dropped column's +position). The script spins up a scratch Postgres container and walks forward through every +migration from server/src/main/resources/db/migration one version at a time, checking the three +tables' real column counts after each one, stopping at the first version whose counts match the +dump files - no guessing, and no assumption that a future dump is from the same schema version +this was built against. That walk is O(n) migrations total either way (each step only applies +whatever's newly pending), just possibly with extra column-count checks along the way. Pass -t to +skip detection and pin a version directly (faster, and necessary if detection is ambiguous - e.g. +two versions in a row happen to match on count alone). + +The scratch database only ever holds these three tables' data (plus whatever they cascade to via +foreign keys) and is destroyed when the script exits, successfully or not. + +Options: + -d Dump directory (default: db/dump under the repo root) + -t Skip auto-detection; migrate to this Flyway version directly (e.g. 1.71) + -f Overwrite existing *.bak backups from a previous run instead of refusing to run + -h show this help + +If a file fails to import (Postgres reports a COPY parse error), the script stops and prints that +error verbatim - it names the exact offending line and content. This is NOT auto-repaired: past +corruption in this dump has been genuine data bugs (a missing closing quote, a stray unescaped +quote) with no reliable general fix, and guessing wrong on a redaction pass is worse than +stopping. Open the reported line, fix it by hand, and re-run. +" + echo "$USAGE" + exit 1 +} + +DUMP_DIR="${PROJECT_ROOT}/db/dump" +FLYWAY_TARGET="" +FORCE=false + +while getopts "d:t:fh" o; do + case "${o}" in + d) DUMP_DIR="${OPTARG}" ;; + t) FLYWAY_TARGET="${OPTARG}" ;; + f) FORCE=true ;; + *) usage ;; + esac +done + +for tool in docker python3; do + command -v "${tool}" >/dev/null || { echo "This script needs '${tool}' on PATH." >&2; exit 1; } +done + +MIGRATIONS_DIR="${PROJECT_ROOT}/server/src/main/resources/db/migration" +FLYWAY_IMAGE="flyway/flyway:10" +POSTGRES_IMAGE="postgres:16.2" +RUN_ID="scrub-$$" +CONTAINER_NAME="openvsx-${RUN_ID}" +NETWORK_NAME="openvsx-${RUN_ID}-net" +COPY_FORMAT="format text, delimiter ','" + +# Tables touched by this script, in dependency order for import (user_data must be present +# before personal_access_token, which references it by foreign key). +TABLES=(user_data signature_key_pair personal_access_token) + +for f in "${TABLES[@]}"; do + if [ ! -f "${DUMP_DIR}/${f}.csv" ]; then + echo "Missing ${DUMP_DIR}/${f}.csv" >&2 + exit 1 + fi + if [ "$FORCE" = false ] && [ -f "${DUMP_DIR}/${f}.csv.bak" ]; then + echo "${DUMP_DIR}/${f}.csv.bak already exists from a previous run - pass -f to overwrite it, or remove/move it first." >&2 + exit 1 + fi +done + +WORKDIR=$(mktemp -d) +cleanup() { + docker rm -f "${CONTAINER_NAME}" >/dev/null 2>&1 || true + docker network rm "${NETWORK_NAME}" >/dev/null 2>&1 || true + rm -rf "${WORKDIR}" +} +trap cleanup EXIT + +psql_exec() { + docker exec -i "${CONTAINER_NAME}" psql -U openvsx -d postgres -v ON_ERROR_STOP=1 "$@" +} + +echo "Starting scratch Postgres..." +docker network create "${NETWORK_NAME}" >/dev/null +docker run -d --name "${CONTAINER_NAME}" --network "${NETWORK_NAME}" \ + -e POSTGRES_USER=openvsx -e POSTGRES_PASSWORD=openvsx "${POSTGRES_IMAGE}" >/dev/null +until docker exec "${CONTAINER_NAME}" pg_isready -U openvsx >/dev/null 2>&1; do sleep 1; done + +flyway_migrate() { + local target="$1" + docker run --rm --network "${NETWORK_NAME}" -v "${MIGRATIONS_DIR}:/flyway/sql" "${FLYWAY_IMAGE}" \ + -url="jdbc:postgresql://${CONTAINER_NAME}:5432/postgres" -user=openvsx -password=openvsx \ + -connectRetries=10 ${target:+-target="${target}"} migrate >/dev/null +} + +reset_schema() { + docker exec "${CONTAINER_NAME}" psql -U openvsx -d postgres -c \ + "DROP SCHEMA public CASCADE; CREATE SCHEMA public;" >/dev/null +} + +table_column_count() { + psql_exec -t -c "select count(*) from information_schema.columns where table_schema='public' and table_name='${1}';" | tr -d ' ' +} + +if [ -z "${FLYWAY_TARGET}" ]; then + echo "No -t given, auto-detecting the Flyway version this dump matches..." + + # Rough column count per dump file: the mode across many rows, tolerating a handful of + # malformed outliers (like the corrupted rows this dump has had before) without needing to + # correctly parse them - this is only used to pick a migration target, not for the redaction + # itself, which always goes through Postgres's own COPY parser. + detect_actual_cols() { + python3 - "${DUMP_DIR}/${1}.csv" <<'PY' +import csv, sys, collections +counts = collections.Counter() +with open(sys.argv[1], newline='', encoding='utf-8', errors='replace') as f: + # Postgres TEXT format has no quoting at all (unlike CSV) - only backslash-escaping of the + # delimiter, backslash itself, and embedded newlines/carriage returns. QUOTE_NONE keeps a + # literal '"' in a field (e.g. inside README/HTML content) from being misread as a quote. + reader = csv.reader(f, delimiter=',', escapechar='\\', quoting=csv.QUOTE_NONE) + for i, row in enumerate(reader): + if i >= 2000: + break + counts[len(row)] += 1 +print(counts.most_common(1)[0][0]) +PY + } + + actual_pat=$(detect_actual_cols personal_access_token) + actual_skp=$(detect_actual_cols signature_key_pair) + actual_ud=$(detect_actual_cols user_data) + echo " dump column counts: personal_access_token=${actual_pat} signature_key_pair=${actual_skp} user_data=${actual_ud}" + + # Every migration version, oldest first, exactly as Flyway orders them (matches the V__ / + # V___ filenames under db/migration). Walked forward one -target=X call at a time - + # each call only applies whatever is newly pending, so this is one full migration replay + # overall, not O(n^2) work, and every state actually comes from a real `flyway migrate` + # rather than from custom bookkeeping. + mapfile -t VERSIONS < <( + find "${MIGRATIONS_DIR}" -maxdepth 1 -name 'V*__*' -printf '%f\n' \ + | sed -E 's/^V([0-9_]+)__.*/\1/; s/_/./' \ + | sort -t. -k1,1n -k2,2n + ) + + reset_schema + for v in "${VERSIONS[@]}"; do + flyway_migrate "${v}" + pat=$(table_column_count personal_access_token) + skp=$(table_column_count signature_key_pair) + ud=$(table_column_count user_data) + if [ "${pat}" = "${actual_pat}" ] && [ "${skp}" = "${actual_skp}" ] && [ "${ud}" = "${actual_ud}" ]; then + FLYWAY_TARGET="${v}" + break + fi + done + + if [ -z "${FLYWAY_TARGET}" ]; then + echo "Could not find a single Flyway version matching all three tables' column counts." >&2 + echo "This can happen if the dump predates the oldest migration, or if the three files come" >&2 + echo "from different points in time. Pass -t to pin a version yourself." >&2 + exit 1 + fi + echo " detected Flyway target: ${FLYWAY_TARGET}" + reset_schema +fi + +echo "Migrating scratch schema to Flyway version ${FLYWAY_TARGET}..." +reset_schema +flyway_migrate "${FLYWAY_TARGET}" + +echo "Importing dump files..." +for t in "${TABLES[@]}"; do + docker cp "${DUMP_DIR}/${t}.csv" "${CONTAINER_NAME}:/tmp/${t}.csv" + set +e + import_output=$(psql_exec -c "\\copy ${t} from '/tmp/${t}.csv' with (${COPY_FORMAT})" 2>&1) + import_status=$? + set -e + if [ ${import_status} -ne 0 ]; then + echo "" >&2 + echo "Import of ${t}.csv failed:" >&2 + echo "${import_output}" >&2 + echo "" >&2 + echo "Fix the offending line in ${DUMP_DIR}/${t}.csv by hand and re-run. Nothing has been" >&2 + echo "modified in ${DUMP_DIR} yet." >&2 + exit 1 + fi + echo " ${t}: ${import_output}" +done + +echo "Redacting..." +psql_exec <<'SQL' +UPDATE personal_access_token +SET value = encode(sha256(('redacted-pat-' || id)::bytea), 'hex'); + +UPDATE signature_key_pair +SET private_key = decode(repeat('00', 32), 'hex'); + +UPDATE user_data +SET email = 'user' || id || '@example.invalid', + full_name = 'Redacted User ' || id, + avatar_url = NULL, + eclipse_token = NULL, + eclipse_person_id = NULL; +SQL + +echo "Exporting redacted tables..." +for t in "${TABLES[@]}"; do + psql_exec -c "\\copy ${t} to '/tmp/${t}.redacted.csv' with (${COPY_FORMAT})" >/dev/null + docker cp "${CONTAINER_NAME}:/tmp/${t}.redacted.csv" "${WORKDIR}/${t}.redacted.csv" +done + +echo "Verifying the redacted export round-trips cleanly..." +reset_schema +flyway_migrate "${FLYWAY_TARGET}" +for t in "${TABLES[@]}"; do + docker cp "${WORKDIR}/${t}.redacted.csv" "${CONTAINER_NAME}:/tmp/${t}.redacted.csv" + round_trip=$(psql_exec -c "\\copy ${t} from '/tmp/${t}.redacted.csv' with (${COPY_FORMAT})" 2>&1) + echo " ${t}: ${round_trip}" + if ! echo "${round_trip}" | grep -q "^COPY "; then + echo "Round-trip verification failed for ${t} - refusing to touch ${DUMP_DIR}. See output above." >&2 + exit 1 + fi +done + +echo "Replacing dump files (originals backed up as *.csv.bak)..." +for t in "${TABLES[@]}"; do + cp "${DUMP_DIR}/${t}.csv" "${DUMP_DIR}/${t}.csv.bak" + cp "${WORKDIR}/${t}.redacted.csv" "${DUMP_DIR}/${t}.csv" +done + +echo "Done. Redacted: ${TABLES[*]}" +echo "Originals kept as *.csv.bak - delete them once you've confirmed the result looks right."