From 98ca39ee871469562fdcd54ef8c9c7cf5bbc29f6 Mon Sep 17 00:00:00 2001 From: Otfrugger <296287929+Otfrugger@users.noreply.github.com> Date: Sun, 30 Aug 2026 13:29:35 +0100 Subject: [PATCH] feat(ci): nightly pg_dump backup workflow + restore runbook Adds a scheduled GitHub Actions workflow that pg_dumps each network's Postgres database independently (DATABASE_URL_TESTNET/_MAINNET), gzips the result, and uploads it as a 14-day artifact. Skips a network gracefully if its backup secret isn't configured yet. Also documents the restore procedure and marks #165 done in the dual-network roadmap. closes #165 --- .github/workflows/db-backup.yml | 109 ++++++++++++++++++++++++++++++++ docs/DUAL_NETWORK.md | 7 +- docs/backup-restore.md | 84 ++++++++++++++++++++++++ ops/backup/dump.sh | 24 +++++++ 4 files changed, 222 insertions(+), 2 deletions(-) create mode 100644 .github/workflows/db-backup.yml create mode 100644 docs/backup-restore.md create mode 100644 ops/backup/dump.sh diff --git a/.github/workflows/db-backup.yml b/.github/workflows/db-backup.yml new file mode 100644 index 00000000..d9d8431c --- /dev/null +++ b/.github/workflows/db-backup.yml @@ -0,0 +1,109 @@ +name: Nightly DB backup + +# Nightly logical backup of the mainnet and testnet Postgres databases, so a +# lost or capped managed instance has a minutes-not-days restore path. See +# docs/backup-restore.md for the restore procedure. This is the durability +# fallback noted in docs/DUAL_NETWORK.md (#165) — Wraith's data is also +# re-derivable by re-indexing from chain, so this backup exists to make that +# unnecessary, not because it's the only copy. + +on: + schedule: + # Nightly at 03:00 UTC. + - cron: "0 3 * * *" + workflow_dispatch: + inputs: + network: + description: "Network to back up" + required: false + default: "both" + type: choice + options: + - both + - testnet + - mainnet + +jobs: + backup-testnet: + name: pg_dump (testnet) + if: ${{ github.event.inputs.network == '' || github.event.inputs.network == 'both' || github.event.inputs.network == 'testnet' }} + runs-on: ubuntu-latest + timeout-minutes: 20 + steps: + - uses: actions/checkout@v4 + + - name: Check secret is configured + id: check + env: + DATABASE_URL_TESTNET: ${{ secrets.DATABASE_URL_TESTNET }} + run: | + if [[ -z "$DATABASE_URL_TESTNET" ]]; then + echo "::warning::DATABASE_URL_TESTNET is not set — skipping testnet backup." + echo "configured=false" >> "$GITHUB_OUTPUT" + else + echo "configured=true" >> "$GITHUB_OUTPUT" + fi + + - name: Install postgresql-client + if: steps.check.outputs.configured == 'true' + run: sudo apt-get update && sudo apt-get install -y --no-install-recommends postgresql-client + + - name: Dump and compress + if: steps.check.outputs.configured == 'true' + env: + NETWORK: testnet + DATABASE_URL: ${{ secrets.DATABASE_URL_TESTNET }} + run: | + chmod +x ops/backup/dump.sh + ops/backup/dump.sh + + - name: Upload backup artifact + if: steps.check.outputs.configured == 'true' + uses: actions/upload-artifact@v4 + with: + name: wraith-db-backup-testnet-${{ github.run_id }} + path: backups/*.dump.gz + retention-days: 14 + if-no-files-found: error + + backup-mainnet: + name: pg_dump (mainnet) + if: ${{ github.event.inputs.network == '' || github.event.inputs.network == 'both' || github.event.inputs.network == 'mainnet' }} + runs-on: ubuntu-latest + timeout-minutes: 20 + steps: + - uses: actions/checkout@v4 + + - name: Check secret is configured + id: check + env: + DATABASE_URL_MAINNET: ${{ secrets.DATABASE_URL_MAINNET }} + run: | + if [[ -z "$DATABASE_URL_MAINNET" ]]; then + echo "::warning::DATABASE_URL_MAINNET is not set — skipping mainnet backup." + echo "configured=false" >> "$GITHUB_OUTPUT" + else + echo "configured=true" >> "$GITHUB_OUTPUT" + fi + + - name: Install postgresql-client + if: steps.check.outputs.configured == 'true' + run: sudo apt-get update && sudo apt-get install -y --no-install-recommends postgresql-client + + - name: Dump and compress + if: steps.check.outputs.configured == 'true' + env: + NETWORK: mainnet + DATABASE_URL: ${{ secrets.DATABASE_URL_MAINNET }} + run: | + chmod +x ops/backup/dump.sh + ops/backup/dump.sh + + - name: Upload backup artifact + if: steps.check.outputs.configured == 'true' + uses: actions/upload-artifact@v4 + with: + name: wraith-db-backup-mainnet-${{ github.run_id }} + path: backups/*.dump.gz + retention-days: 14 + if-no-files-found: error diff --git a/docs/DUAL_NETWORK.md b/docs/DUAL_NETWORK.md index e13abe7e..2bf30281 100644 --- a/docs/DUAL_NETWORK.md +++ b/docs/DUAL_NETWORK.md @@ -63,7 +63,7 @@ Dependencies: **#159 → #161** and #160 before #161. | [#162](../../issues/162) | Per-network SAC/NFT watch-lists | — | | [#163](../../issues/163) | `network` selector on REST/GraphQL/WS | #159–#161 | | [#164](../../issues/164) | Serve stale cached data instead of 503 | — | -| [#165](../../issues/165) | Nightly `pg_dump` backup + restore runbook | — | +| ~~[#165](../../issues/165)~~ | ~~Nightly `pg_dump` backup + restore runbook~~ (done) | — | | [#166](../../issues/166) | Mainnet deploy guide | the rest | ## Ops (Render + UptimeRobot + external Postgres) @@ -74,5 +74,8 @@ Dependencies: **#159 → #161** and #160 before #161. always-on and testnet on-demand / a second account / a paid instance. - **Database:** use Neon or another managed Postgres. **Do not** use Render's free Postgres — it is **deleted after 90 days**. One DB per network. -- **Durability fallback:** nightly `pg_dump` (#165) for fast restore; because +- **Durability fallback:** nightly `pg_dump` (#165, see + [`docs/backup-restore.md`](backup-restore.md)) for fast restore; because Wraith is an indexer, the DB is also re-derivable by re-indexing from chain. + Requires `DATABASE_URL_TESTNET` / `DATABASE_URL_MAINNET` backup secrets, one + per network's database. diff --git a/docs/backup-restore.md b/docs/backup-restore.md new file mode 100644 index 00000000..c123e247 --- /dev/null +++ b/docs/backup-restore.md @@ -0,0 +1,84 @@ +# Backup & restore + +Wraith is an indexer, so its Postgres data is always re-derivable by +re-indexing from chain — but a `pg_dump` restore is minutes, not days. The +[`db-backup.yml`](../.github/workflows/db-backup.yml) workflow runs nightly +(03:00 UTC) and on manual dispatch, and takes a logical backup of each +network's database independently, since mainnet and testnet each have their +own Postgres instance (see [`docs/DUAL_NETWORK.md`](DUAL_NETWORK.md)). + +## What it does + +For each network with a backup secret configured, the workflow: + +1. Runs `pg_dump --format=custom --no-owner --no-privileges` against that + network's database (via [`ops/backup/dump.sh`](../ops/backup/dump.sh)). +2. Gzips the dump. +3. Uploads it as a GitHub Actions artifact named + `wraith-db-backup--`, retained for 14 days. + +A network without its secret set is skipped with a workflow warning rather +than failing the run — this lets the workflow exist before both databases are +provisioned. + +## Required secrets + +| Secret | Purpose | +|---|---| +| `DATABASE_URL_TESTNET` | Connection string for the testnet database | +| `DATABASE_URL_MAINNET` | Connection string for the mainnet database | + +These are backup-only credentials, separate from the `DATABASE_URL` a given +deployment runs with (each deployment only knows about its own network's +database — see the env matrix in `docs/DUAL_NETWORK.md`). Use a role with +read access sufficient for `pg_dump`; it does not need write access. + +Set them in the repo's **Settings → Secrets and variables → Actions**. Never +commit a real connection string. + +## Running it manually + +Trigger the workflow from the **Actions** tab (`Nightly DB backup` → +**Run workflow**) and pick `both`, `testnet`, or `mainnet`. Useful right +before a risky migration or deploy. + +## Restoring from a backup + +1. Download the artifact from the workflow run (**Actions** → the run → + **Artifacts**), or via `gh run download -n wraith-db-backup--`. +2. Decompress it: + ```bash + gunzip wraith--.dump.gz + ``` +3. Provision (or reuse) a target Postgres instance — e.g. a fresh Neon + database — and restore into it: + ```bash + pg_restore --clean --if-exists --no-owner --no-privileges \ + --dbname="$TARGET_DATABASE_URL" \ + wraith--.dump + ``` + `--clean --if-exists` drops conflicting objects first, so this is also + safe to run against a database that already has the old schema in it. +4. Point the deployment at the restored database: update `DATABASE_URL` (and + `DIRECT_DATABASE_URL` if set separately, e.g. for a pooled connection) for + that network's service, then redeploy/restart it. +5. Sanity-check: hit `/status` and `/healthz` on the restored deployment and + confirm `IndexerState`/`BackfillCursor` rows look sane for that network. If + the restore lags behind the chain tip, the indexer will catch up on its + own via backfill — no manual action needed. + +### Per-network dumps + +Never restore a testnet dump into the mainnet database or vice versa. Every +row in this schema is tagged with a `network` column ([`docs/DUAL_NETWORK.md`](DUAL_NETWORK.md#L1)), +but the two networks still use physically separate database instances in the +recommended deployment, so cross-restoring would overwrite one network's +current data with the other's stale snapshot rather than merging anything. + +## Verifying the runbook + +Before relying on this in production, run the restore steps above once +against a scratch Postgres instance using a real nightly artifact, and +confirm the restored data matches what `/status` reports for that network. +This should be re-verified whenever the Prisma schema changes in a way that +affects `pg_restore` compatibility (e.g. extensions, custom types). diff --git a/ops/backup/dump.sh b/ops/backup/dump.sh new file mode 100644 index 00000000..3c47248b --- /dev/null +++ b/ops/backup/dump.sh @@ -0,0 +1,24 @@ +#!/usr/bin/env bash +# Dump a Postgres database and gzip the result. +# Usage: NETWORK=mainnet DATABASE_URL=postgresql://... dump.sh +set -euo pipefail + +NETWORK="${NETWORK:?NETWORK is required (testnet or mainnet)}" +DATABASE_URL="${DATABASE_URL:?DATABASE_URL is required}" +OUT_DIR="${OUT_DIR:-backups}" + +mkdir -p "$OUT_DIR" + +STAMP="$(date -u +%Y%m%dT%H%M%SZ)" +DUMP_FILE="${OUT_DIR}/wraith-${NETWORK}-${STAMP}.dump" + +echo "==> [backup/dump] dumping ${NETWORK} database" +pg_dump --format=custom --no-owner --no-privileges --dbname="$DATABASE_URL" --file="$DUMP_FILE" + +echo "==> [backup/dump] compressing ${DUMP_FILE}" +gzip "$DUMP_FILE" + +echo "==> [backup/dump] wrote ${DUMP_FILE}.gz" +if [[ -n "${GITHUB_OUTPUT:-}" ]]; then + echo "dump_path=${DUMP_FILE}.gz" >> "$GITHUB_OUTPUT" +fi