From 18b596928e430f2f659f32ba5d08159c6fb0aa80 Mon Sep 17 00:00:00 2001 From: namdamdoi68-oss Date: Wed, 26 Aug 2026 18:37:28 +0700 Subject: [PATCH 1/2] ci: nightly pg_dump artifacts and restore runbook Signed-off-by: namdamdoi68-oss --- .github/workflows/db-backup.yml | 62 ++++++++++++++++++++ docs/DUAL_NETWORK.md | 7 ++- docs/backup-restore.md | 74 ++++++++++++++++++++++++ src/__tests__/db-backup.workflow.test.ts | 72 +++++++++++++++++++++++ 4 files changed, 212 insertions(+), 3 deletions(-) create mode 100644 .github/workflows/db-backup.yml create mode 100644 docs/backup-restore.md create mode 100644 src/__tests__/db-backup.workflow.test.ts diff --git a/.github/workflows/db-backup.yml b/.github/workflows/db-backup.yml new file mode 100644 index 0000000..8718f3e --- /dev/null +++ b/.github/workflows/db-backup.yml @@ -0,0 +1,62 @@ +name: Nightly DB backup + +# Logical dumps of the per-network Postgres (Neon/direct URL). Price history is +# re-derivable from chain, but a nightly custom-format dump is the fast restore +# path. See docs/backup-restore.md. +on: + schedule: + # 03:00 UTC — one hour before the load test (04:00), so dump I/O and the + # 5k-RPS flood do not overlap. + - cron: '0 3 * * *' + workflow_dispatch: + +permissions: + contents: read + +concurrency: + group: db-backup + cancel-in-progress: false + +jobs: + dump: + name: pg_dump (${{ matrix.network }}) + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + network: [mainnet, testnet] + env: + DATABASE_URL_MAINNET: ${{ secrets.DATABASE_URL_MAINNET }} + DATABASE_URL_TESTNET: ${{ secrets.DATABASE_URL_TESTNET }} + steps: + - uses: actions/checkout@v4 + + - name: Install postgresql-client + run: sudo apt-get update && sudo apt-get install -y postgresql-client + + - name: Dump ${{ matrix.network }} + env: + NETWORK: ${{ matrix.network }} + run: | + set -euo pipefail + if [ "$NETWORK" = "mainnet" ]; then + DATABASE_URL="${DATABASE_URL_MAINNET:-}" + else + DATABASE_URL="${DATABASE_URL_TESTNET:-}" + fi + if [ -z "$DATABASE_URL" ]; then + echo "SKIP ${NETWORK}: secret not configured" + exit 0 + fi + DATE_UTC=$(date -u +%Y%m%d) + OUT="lens-${NETWORK}-${DATE_UTC}.dump" + pg_dump --format=custom --no-owner --no-acl --dbname="$DATABASE_URL" --file="$OUT" + gzip -n "$OUT" + + - uses: actions/upload-artifact@v4 + if: success() + with: + name: lens-${{ matrix.network }}-${{ github.run_id }} + path: lens-${{ matrix.network }}-*.dump.gz + retention-days: 14 + if-no-files-found: ignore diff --git a/docs/DUAL_NETWORK.md b/docs/DUAL_NETWORK.md index 9ef7632..6bf9da0 100644 --- a/docs/DUAL_NETWORK.md +++ b/docs/DUAL_NETWORK.md @@ -43,7 +43,7 @@ Dependencies: **#113 → #115 → #116 → #117**; #114 before the network selec | [#116](../../issues/116) | Per-network Soroswap/Reflector/Aquarius/token-list | #113 | | [#117](../../issues/117) | Launch ingesters per network | #113–#116 | | [#118](../../issues/118) | Network selector on routes + per-request x402 | #114 | -| [#119](../../issues/119) | Nightly `pg_dump` backup + restore runbook | — | +| [#119](../../issues/119) | Nightly `pg_dump` backup + [restore runbook](backup-restore.md) | — | | [#120](../../issues/120) | Mainnet deploy guide | the rest | ## Ops (Render + UptimeRobot + external Postgres) @@ -53,5 +53,6 @@ Dependencies: **#113 → #115 → #116 → #117**; #114 before the network selec mainnet always-on, testnet on-demand / a second account / a paid instance. - **Database:** use Neon or another managed Postgres. **Do not** use Render's free Postgres — it is **deleted after 90 days**. One DB per network. -- **Durability fallback:** nightly `pg_dump` (#119) for fast restore; Lens is an - aggregator, so the DB is also re-derivable by re-ingesting from chain. +- **Durability fallback:** nightly `pg_dump` (#119) for fast restore; see + [backup-restore.md](backup-restore.md). Lens is an aggregator, so the DB is + also re-derivable by re-ingesting from chain. diff --git a/docs/backup-restore.md b/docs/backup-restore.md new file mode 100644 index 0000000..0ffbda7 --- /dev/null +++ b/docs/backup-restore.md @@ -0,0 +1,74 @@ +# Backup and restore + +Nightly `pg_dump` of each network's Postgres. Lens stores price history in +**one database per network** (no `network` column); restoring a mainnet dump +into the testnet instance would mix pairs. See [DUAL_NETWORK.md](DUAL_NETWORK.md). + +The workflow is [`.github/workflows/db-backup.yml`](../.github/workflows/db-backup.yml): +cron `0 3 * * *` UTC plus `workflow_dispatch`. Dumps are GitHub Actions +artifacts, kept **14 days**. + +## Secrets + +Add these on the GitHub repo (Settings → Secrets and variables → Actions). +Use the **direct** connection string, not the pooler: + +| Secret | Database | +|--------|----------| +| `DATABASE_URL_MAINNET` | mainnet Postgres | +| `DATABASE_URL_TESTNET` | testnet Postgres | + +On Neon that means the host **without** `-pooler`, port **5432** (not 6543), +and `sslmode=require`. Prisma's pooled `DATABASE_URL` is the runtime URL; +`pg_dump` / `pg_restore` need the same class of URI as `DIRECT_DATABASE_URL`. + +A missing secret skips that network (`SKIP : secret not configured`) +so a deploy that only has mainnet still dumps. Configure both for full +coverage. + +Do not put these URIs in the repo, `.env.example`, or workflow YAML. + +## Download a dump + +1. Actions → **Nightly DB backup** → the run you want. +2. Artifacts: `lens-mainnet-` / `lens-testnet-`. +3. File inside: `lens--YYYYMMDD.dump.gz`. + +`workflow_dispatch` is the way to take a dump on demand (and to verify the +job once secrets are in place). + +## Restore into a fresh instance + +Do not `pg_restore` onto the live URL. Spin up an empty database, restore +there, then swap the deployment env. + +1. Create a new Neon project (or local Postgres). Match the dump's major + version; if `pg_restore` complains, install a client ≥ the server + (`postgresql-client-16` / `17` as needed). +2. Copy the **direct** URI for the new instance (`sslmode=require` on Neon). +3. Restore **one network per instance**: + +```bash +gzip -dc lens-mainnet-YYYYMMDD.dump.gz \ + | pg_restore --no-owner --no-acl --dbname="$NEW_DIRECT_URL" +``` + +`pg_restore -l` on the gunzipped file lists the TOC if you want to inspect +before loading. + +4. Point the matching Lens deployment at the new instance: + - `DATABASE_URL` — pooled URL is fine for the app + - `DIRECT_DATABASE_URL` — direct URL (Prisma migrations / `db push`) +5. Check `/status`, then one `/prices/history` (or GraphQL equivalent) for a + watched pair. +6. Rollback is swapping those two env vars back to the previous instance. + +Repeat independently for testnet with the testnet artifact and testnet +deployment. Never restore a mainnet dump into testnet, or the reverse. + +## Client notes + +Compose uses Postgres 15. Managed Neon may be 16/17. `pg_dump` / `pg_restore` +must be the same major as the server or newer. The workflow installs Ubuntu's +`postgresql-client`; bump it in the workflow if dumps start failing with a +version error. diff --git a/src/__tests__/db-backup.workflow.test.ts b/src/__tests__/db-backup.workflow.test.ts new file mode 100644 index 0000000..65b9563 --- /dev/null +++ b/src/__tests__/db-backup.workflow.test.ts @@ -0,0 +1,72 @@ +import { describe, it, expect } from 'vitest' +import { readFileSync } from 'node:fs' +import { resolve } from 'node:path' +import YAML from 'js-yaml' + +const root = resolve(__dirname, '../..') +const workflowPath = resolve(root, '.github/workflows/db-backup.yml') +const runbookPath = resolve(root, 'docs/backup-restore.md') + +function read(path: string): string { + return readFileSync(path, 'utf8') +} + +describe('nightly pg_dump workflow', () => { + it('exists and is valid YAML', () => { + const raw = read(workflowPath) + expect(raw.length).toBeGreaterThan(0) + expect(() => YAML.load(raw)).not.toThrow() + }) + + it('runs on a nightly schedule and workflow_dispatch only', () => { + const raw = read(workflowPath) + const doc = YAML.load(raw) as { + on?: { + schedule?: { cron: string }[] + workflow_dispatch?: unknown + pull_request?: unknown + push?: unknown + } + } + const crons = (doc.on?.schedule ?? []).map((s) => s.cron) + expect(crons).toContain('0 3 * * *') + expect(doc.on?.workflow_dispatch).toBeDefined() + expect(doc.on?.pull_request).toBeUndefined() + expect(doc.on?.push).toBeUndefined() + }) + + it('covers both networks via secrets and keeps artifacts 14 days', () => { + const raw = read(workflowPath) + expect(raw).toMatch(/secrets\.DATABASE_URL_MAINNET/) + expect(raw).toMatch(/secrets\.DATABASE_URL_TESTNET/) + expect(raw).toMatch(/mainnet/) + expect(raw).toMatch(/testnet/) + expect(raw).toMatch(/retention-days:\s*14/) + expect(raw).not.toMatch(/postgres:\/\/[^\s]+:[^\s]+@/) + }) + + it('dumps custom format, gzips, and does not echo the URL', () => { + const raw = read(workflowPath) + expect(raw).toMatch(/pg_dump/) + expect(raw).toMatch(/--format=custom/) + expect(raw).toMatch(/\bgzip\b/) + expect(raw).toMatch(/--no-owner/) + expect(raw).toMatch(/--no-acl/) + expect(raw).not.toMatch(/echo\s+"?\$\{?DATABASE_URL/) + expect(raw).not.toMatch(/set\s+-x/) + }) +}) + +describe('backup restore runbook', () => { + it('documents pg_restore into a fresh instance and URL swap for both networks', () => { + const raw = read(runbookPath) + expect(raw).toMatch(/pg_restore/) + expect(raw).toMatch(/Neon/i) + expect(raw).toMatch(/DATABASE_URL/) + expect(raw).toMatch(/DIRECT_DATABASE_URL/) + expect(raw).toMatch(/mainnet/i) + expect(raw).toMatch(/testnet/i) + expect(raw).toMatch(/pooler|direct/i) + expect(raw).toMatch(/fresh/i) + }) +}) From 0266fcd540391aa47e3c119a07ccd0a8efad8255 Mon Sep 17 00:00:00 2001 From: namdamdoi68-oss Date: Thu, 27 Aug 2026 02:06:11 +0700 Subject: [PATCH 2/2] fix: encrypt nightly dumps before uploading artifacts Signed-off-by: namdamdoi68-oss --- .github/workflows/db-backup.yml | 19 +++++--- docs/backup-restore.md | 56 +++++++++++++++++------- src/__tests__/db-backup.workflow.test.ts | 18 ++++++++ 3 files changed, 72 insertions(+), 21 deletions(-) diff --git a/.github/workflows/db-backup.yml b/.github/workflows/db-backup.yml index 8718f3e..b746260 100644 --- a/.github/workflows/db-backup.yml +++ b/.github/workflows/db-backup.yml @@ -1,8 +1,8 @@ name: Nightly DB backup -# Logical dumps of the per-network Postgres (Neon/direct URL). Price history is -# re-derivable from chain, but a nightly custom-format dump is the fast restore -# path. See docs/backup-restore.md. +# Logical dumps of the per-network Postgres (Neon/direct URL). This repo is +# public, so Actions artifacts are world-readable — only age ciphertext is +# uploaded. See docs/backup-restore.md. on: schedule: # 03:00 UTC — one hour before the load test (04:00), so dump I/O and the @@ -28,11 +28,12 @@ jobs: env: DATABASE_URL_MAINNET: ${{ secrets.DATABASE_URL_MAINNET }} DATABASE_URL_TESTNET: ${{ secrets.DATABASE_URL_TESTNET }} + AGE_RECIPIENT: ${{ secrets.AGE_RECIPIENT }} steps: - uses: actions/checkout@v4 - - name: Install postgresql-client - run: sudo apt-get update && sudo apt-get install -y postgresql-client + - name: Install postgresql-client and age + run: sudo apt-get update && sudo apt-get install -y postgresql-client age - name: Dump ${{ matrix.network }} env: @@ -48,15 +49,21 @@ jobs: echo "SKIP ${NETWORK}: secret not configured" exit 0 fi + if [ -z "${AGE_RECIPIENT:-}" ]; then + echo "AGE_RECIPIENT is required; refusing an unencrypted dump" + exit 1 + fi DATE_UTC=$(date -u +%Y%m%d) OUT="lens-${NETWORK}-${DATE_UTC}.dump" pg_dump --format=custom --no-owner --no-acl --dbname="$DATABASE_URL" --file="$OUT" gzip -n "$OUT" + age -r "$AGE_RECIPIENT" -o "${OUT}.gz.age" "${OUT}.gz" + rm -f "$OUT" "${OUT}.gz" - uses: actions/upload-artifact@v4 if: success() with: name: lens-${{ matrix.network }}-${{ github.run_id }} - path: lens-${{ matrix.network }}-*.dump.gz + path: lens-${{ matrix.network }}-*.dump.gz.age retention-days: 14 if-no-files-found: ignore diff --git a/docs/backup-restore.md b/docs/backup-restore.md index 0ffbda7..9f9359c 100644 --- a/docs/backup-restore.md +++ b/docs/backup-restore.md @@ -1,38 +1,59 @@ # Backup and restore +A dump of this database contains live credentials — `webhooks.secret` (HMAC +signing keys) and `api_keys` rows — and must never land anywhere +world-readable. GitHub Actions artifacts on this public repo are +world-readable, so the workflow only uploads an `age`-encrypted file. + Nightly `pg_dump` of each network's Postgres. Lens stores price history in **one database per network** (no `network` column); restoring a mainnet dump into the testnet instance would mix pairs. See [DUAL_NETWORK.md](DUAL_NETWORK.md). +Neon also has branching and point-in-time restore. Use that for short-window +accidents. The encrypted dump is the copy that survives a project delete or +a missed PITR window. + The workflow is [`.github/workflows/db-backup.yml`](../.github/workflows/db-backup.yml): -cron `0 3 * * *` UTC plus `workflow_dispatch`. Dumps are GitHub Actions -artifacts, kept **14 days**. +cron `0 3 * * *` UTC plus `workflow_dispatch`. Encrypted artifacts are kept +**14 days**. ## Secrets Add these on the GitHub repo (Settings → Secrets and variables → Actions). Use the **direct** connection string, not the pooler: -| Secret | Database | -|--------|----------| -| `DATABASE_URL_MAINNET` | mainnet Postgres | -| `DATABASE_URL_TESTNET` | testnet Postgres | +| Secret | What | +|--------|------| +| `DATABASE_URL_MAINNET` | mainnet Postgres (direct URI) | +| `DATABASE_URL_TESTNET` | testnet Postgres (direct URI) | +| `AGE_RECIPIENT` | `age` public key (`age1…`) | On Neon that means the host **without** `-pooler`, port **5432** (not 6543), and `sslmode=require`. Prisma's pooled `DATABASE_URL` is the runtime URL; `pg_dump` / `pg_restore` need the same class of URI as `DIRECT_DATABASE_URL`. -A missing secret skips that network (`SKIP : secret not configured`) -so a deploy that only has mainnet still dumps. Configure both for full -coverage. +Generate the keypair on a machine that is not the runner: + +```bash +age-keygen -o lens-backup.agekey +# public key line: age1… +``` + +Put only the public key in `AGE_RECIPIENT`. The identity file stays offline +(password manager / disk not in this repo). GitHub never sees it. -Do not put these URIs in the repo, `.env.example`, or workflow YAML. +A missing database URL skips that network (`SKIP : secret not +configured`). A missing `AGE_RECIPIENT` fails the job — it will not upload +plaintext. + +Do not put these URIs, the identity file, or a plaintext dump in the repo, +`.env.example`, or workflow YAML. ## Download a dump 1. Actions → **Nightly DB backup** → the run you want. 2. Artifacts: `lens-mainnet-` / `lens-testnet-`. -3. File inside: `lens--YYYYMMDD.dump.gz`. +3. File inside: `lens--YYYYMMDD.dump.gz.age`. `workflow_dispatch` is the way to take a dump on demand (and to verify the job once secrets are in place). @@ -46,15 +67,20 @@ there, then swap the deployment env. version; if `pg_restore` complains, install a client ≥ the server (`postgresql-client-16` / `17` as needed). 2. Copy the **direct** URI for the new instance (`sslmode=require` on Neon). -3. Restore **one network per instance**: +3. Decrypt with the offline identity, then restore **one network per + instance**: ```bash +age -d -i lens-backup.agekey \ + -o lens-mainnet-YYYYMMDD.dump.gz \ + lens-mainnet-YYYYMMDD.dump.gz.age + gzip -dc lens-mainnet-YYYYMMDD.dump.gz \ | pg_restore --no-owner --no-acl --dbname="$NEW_DIRECT_URL" ``` `pg_restore -l` on the gunzipped file lists the TOC if you want to inspect -before loading. +before loading. Delete the decrypted `.dump.gz` when you are done. 4. Point the matching Lens deployment at the new instance: - `DATABASE_URL` — pooled URL is fine for the app @@ -70,5 +96,5 @@ deployment. Never restore a mainnet dump into testnet, or the reverse. Compose uses Postgres 15. Managed Neon may be 16/17. `pg_dump` / `pg_restore` must be the same major as the server or newer. The workflow installs Ubuntu's -`postgresql-client`; bump it in the workflow if dumps start failing with a -version error. +`postgresql-client` and `age`; bump the client package if dumps start failing +with a version error. diff --git a/src/__tests__/db-backup.workflow.test.ts b/src/__tests__/db-backup.workflow.test.ts index 65b9563..e2ef418 100644 --- a/src/__tests__/db-backup.workflow.test.ts +++ b/src/__tests__/db-backup.workflow.test.ts @@ -55,6 +55,16 @@ describe('nightly pg_dump workflow', () => { expect(raw).not.toMatch(/echo\s+"?\$\{?DATABASE_URL/) expect(raw).not.toMatch(/set\s+-x/) }) + + it('age-encrypts before upload and refuses a missing recipient', () => { + const raw = read(workflowPath) + expect(raw).toMatch(/secrets\.AGE_RECIPIENT/) + expect(raw).toMatch(/\bage -r\b/) + expect(raw).toMatch(/\.dump\.gz\.age/) + expect(raw).toMatch(/AGE_RECIPIENT is required|refusing.*unencrypted/i) + expect(raw).toMatch(/path:\s*lens-\$\{\{\s*matrix\.network\s*\}\}-\*\.dump\.gz\.age/) + expect(raw).not.toMatch(/path:\s*lens-\$\{\{\s*matrix\.network\s*\}\}-\*\.dump\.gz\s*$/m) + }) }) describe('backup restore runbook', () => { @@ -69,4 +79,12 @@ describe('backup restore runbook', () => { expect(raw).toMatch(/pooler|direct/i) expect(raw).toMatch(/fresh/i) }) + + it('says dumps hold credentials and must be decrypted off the public artifact', () => { + const raw = read(runbookPath) + expect(raw).toMatch(/webhooks?\.secret|HMAC/i) + expect(raw).toMatch(/world-readable|public/i) + expect(raw).toMatch(/age -d/) + expect(raw).toMatch(/AGE_RECIPIENT/) + }) })