From fd01aee47d911227088a9aa4b7ebb991e617184f Mon Sep 17 00:00:00 2001 From: Makisuo Date: Sat, 19 Sep 2026 20:22:51 +0200 Subject: [PATCH] feat(infra): apply prd migrations from the alchemy deploy The PlanetScale `main` branch is declared in alchemy.run.ts as `Planetscale.PostgresBranch` with `migrations` at packages/db/drizzle, adopted and retained, and carried on `MapleStack` for prd. The api, ai and alerting Workers put its name in their env so they upload after it. `ps:apply-schema` and `migrate:prod` are removed: the deploy is the migration. --- .github/workflows/deploy-prd.yml | 13 ++--- CLAUDE.md | 9 ++-- alchemy.run.ts | 13 +++++ apps/ai/src/worker.ts | 3 +- apps/alerting/src/worker.ts | 9 +++- apps/api/src/worker.ts | 3 +- bun.lock | 3 ++ docs/infra.md | 8 +++ docs/persistence.md | 13 +++-- knip.json | 4 +- package.json | 5 +- packages/db/package.json | 1 - packages/db/scripts/ensure-privileges.ts | 6 +-- .../db/scripts/planetscale-apply-schema.ts | 50 ------------------- .../planetscale-migrations-preflight.ts | 4 +- packages/db/src/migrate.ts | 3 +- packages/infra/src/cloudflare/stack.ts | 3 ++ 17 files changed, 68 insertions(+), 82 deletions(-) delete mode 100644 packages/db/scripts/planetscale-apply-schema.ts diff --git a/.github/workflows/deploy-prd.yml b/.github/workflows/deploy-prd.yml index 45553b151..c07039eab 100644 --- a/.github/workflows/deploy-prd.yml +++ b/.github/workflows/deploy-prd.yml @@ -59,15 +59,10 @@ jobs: infisical-project-slug: ${{ vars.INFISICAL_PROJECT_SLUG }} aws-role-arn: ${{ vars.AWS_DEPLOY_ROLE_ARN }} - # NOTE: prod schema migrations are applied OUT OF BAND (manually, via - # `bun run migrate:prod` → `ps:apply-schema main`, against the direct 5432 - # port), NOT by this workflow — so a deploy never touches the prod database - # and needs no MAPLE_PG_URL/admin credential. The worker binds to the - # pre-configured `maple-prd` Hyperdrive (`MapleDb` in packages/infra). - # apply-schema installs default privileges granting PUBLIC before it - # migrates, so new and rebuilt tables are readable by every consumer — - # including the ingest gateway, which reaches Postgres through PSBouncer as - # a role that does NOT inherit `postgres`. + # Schema migrations run in this deploy (`Planetscale.PostgresBranch` in alchemy.run.ts), + # over PLANETSCALE_API_TOKEN_ID / PLANETSCALE_API_TOKEN / PLANETSCALE_ORGANIZATION + # from Infisical. The Workers still bind the dashboard-managed `maple-prd` + # Hyperdrive (`MapleDb`); the deploy needs no database credential of its own. # alchemy's env-credential path (CI=true) otherwise discovers the account # with an STS GetCallerIdentity issued while its own AWSEnvironment is diff --git a/CLAUDE.md b/CLAUDE.md index 655ecce41..dbbf781c6 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -177,10 +177,11 @@ database), reached from Workers via the Hyperdrive binding `MAPLE_DB`. `connectionTimeoutMillis` to queue waits too. A stalled dial lands as `error.type = ConnectionError` (a refused one carries the socket code, `ECONNREFUSED`). Fork DB work off a request only with `forkRequestScoped`, which interrupts it at the response but lets a DB call already under way finish. -- Migrations: `bun run --cwd packages/db db:generate`. Production is applied BY HAND before the - Worker deploy: `bun run --cwd packages/db ps:migrations-preflight main` (read-only; the v1 - migrator refuses unmatched rows and replays unrecorded folders), then `bun run migrate:prod` - against the DIRECT port 5432 (never a pooler). PGlite applies them at layer build. +- Migrations: `bun run --cwd packages/db db:generate`. **The prd deploy applies them**: the + PlanetScale `main` branch is an alchemy `Planetscale.PostgresBranch` in `alchemy.run.ts` with + `migrations` pointed at `packages/db/drizzle`; never run `drizzle-kit migrate` against prd. It + migrates as a temporary role, so a migration creating a table must `GRANT` it `TO PUBLIC` itself + (the ingest gateway reads only through PUBLIC). PGlite applies them at layer build. - **PR preview deploys are label-gated** (2026-08, cost — re-enabled by `fd00bcd412`). A PR gets a preview only while it carries the `preview` label; `deploy-pr-preview.yml` triggers on `opened, reopened, synchronize, labeled, unlabeled, closed` and tears the stack down the moment diff --git a/alchemy.run.ts b/alchemy.run.ts index 61e283e70..3e5e16654 100644 --- a/alchemy.run.ts +++ b/alchemy.run.ts @@ -14,6 +14,8 @@ import * as AWS from "alchemy/AWS" import * as Cloudflare from "alchemy/Cloudflare" import * as Command from "alchemy/Command" import * as Output from "alchemy/Output" +import * as Planetscale from "alchemy/Planetscale" +import * as RemovalPolicy from "alchemy/RemovalPolicy" import * as Effect from "effect/Effect" import * as Layer from "effect/Layer" import { @@ -130,6 +132,16 @@ const MapleStackLive = Layer.effect( }, workerDev, devEnv, + // prd's database: the PlanetScale `main` branch, adopted, whose deploy applies the drizzle + // migrations. The Workers that bind it put its name in their env so they upload after it. + dbSchema: + resolveDatabaseMode(stage) === "ref" + ? yield* Planetscale.PostgresBranch("maple-db-main", { + database: "maple", + name: "main", + migrations: "packages/db/drizzle", + }).pipe(RemovalPolicy.retain()) + : undefined, } return context }), @@ -181,6 +193,7 @@ const providers = Acm.providers().pipe( Layer.provideMerge(Cloudflare.providers()), Layer.provideMerge(AWS.providers()), + Layer.provideMerge(Planetscale.providers()), Layer.provideMerge(Portless.providers()), ) diff --git a/apps/ai/src/worker.ts b/apps/ai/src/worker.ts index bfaa5db0c..620e9e15f 100644 --- a/apps/ai/src/worker.ts +++ b/apps/ai/src/worker.ts @@ -131,7 +131,7 @@ const configuredEnv = (stage: MapleStage) => */ const props = Effect.gen(function* () { if (globalThis.__ALCHEMY_RUNTIME__) return { main: import.meta.url } - const { stage, workerDev, devEnv } = yield* MapleStack + const { stage, workerDev, devEnv, dbSchema } = yield* MapleStack // The agents' repository sandbox, reached only over this binding. Absent on // the stages that do not deploy it, where `SandboxClient` reports the tools // as unavailable rather than failing. @@ -155,6 +155,7 @@ const props = Effect.gen(function* () { // `devEnv` last, so `.env.local` cannot override the inter-app URLs. env: { ...makeWorkerBindings({ stage }), + ...(dbSchema && { MAPLE_DB_BRANCH: dbSchema.name }), ...(Option.isSome(sandbox) ? { SANDBOX: sandbox.value } : undefined), ...env, ...devEnv, diff --git a/apps/alerting/src/worker.ts b/apps/alerting/src/worker.ts index a3d2a20b4..d5213173c 100644 --- a/apps/alerting/src/worker.ts +++ b/apps/alerting/src/worker.ts @@ -110,7 +110,7 @@ const configuredEnv = (stage: MapleStage) => */ const props = Effect.gen(function* () { if (globalThis.__ALCHEMY_RUNTIME__) return { main: import.meta.url } - const { stage, workerDev, devEnv } = yield* MapleStack + const { stage, workerDev, devEnv, dbSchema } = yield* MapleStack const env = yield* configuredEnv(stage) return { main: import.meta.url, @@ -121,7 +121,12 @@ const props = Effect.gen(function* () { dev: workerDev("alerting"), workersDev: false, // `devEnv` last, so `.env.local` cannot override the inter-app URLs. - env: { ...makeWorkerBindings({ stage }), ...env, ...devEnv }, + env: { + ...makeWorkerBindings({ stage }), + ...(dbSchema && { MAPLE_DB_BRANCH: dbSchema.name }), + ...env, + ...devEnv, + }, } }) diff --git a/apps/api/src/worker.ts b/apps/api/src/worker.ts index e111bc36a..75684a652 100644 --- a/apps/api/src/worker.ts +++ b/apps/api/src/worker.ts @@ -66,7 +66,7 @@ const makeWorkerBindings = ({ stage }: { stage: MapleStage }) => ({ */ const props = Effect.gen(function* () { if (globalThis.__ALCHEMY_RUNTIME__) return { main: import.meta.url } - const { stage, domains, workerDev, devEnv } = yield* MapleStack + const { stage, domains, workerDev, devEnv, dbSchema } = yield* MapleStack // maple-ai, which serves `/mcp` and the chat surface. api keeps the hostname // and forwards, so the public address and the OAuth identity do not move. const ai = yield* AiWorker @@ -100,6 +100,7 @@ const props = Effect.gen(function* () { // `devEnv` last, so `.env.local` cannot override the inter-app URLs. env: { ...makeWorkerBindings({ stage }), + ...(dbSchema && { MAPLE_DB_BRANCH: dbSchema.name }), AI_WORKER: ai, ...configuredEnv, ...devEnv, diff --git a/bun.lock b/bun.lock index 58fbdc53a..de098e409 100644 --- a/bun.lock +++ b/bun.lock @@ -4,6 +4,9 @@ "workspaces": { "": { "name": "maple", + "dependencies": { + "pg": "^8.23.0", + }, "devDependencies": { "@cloudflare/workers-types": "catalog:alchemy", "@effect/platform-node": "4.0.0-rc.112", diff --git a/docs/infra.md b/docs/infra.md index 71ec193da..79c8d2f9e 100644 --- a/docs/infra.md +++ b/docs/infra.md @@ -405,6 +405,14 @@ the number as load-bearing. The workflow compiles inside `rust:1.94-bookworm` ra on the runner because the runtime base is `debian:bookworm-slim` (glibc 2.36) while `ubuntu-24.04` ships 2.39 — a host-built binary dies with `version 'GLIBC_2.39' not found`. +## Schema migrations run in the deploy + +The PlanetScale `main` branch is a `Planetscale.PostgresBranch` yielded into `MapleStack` on prd +(`dbSchema`), with `migrations` at `packages/db/drizzle`. Alchemy orders resources only by the +Outputs their props reference, and a Hyperdrive bound by id references nothing, so the api, ai and +alerting Workers put `dbSchema.name` in their env (`MAPLE_DB_BRANCH`) to upload after it. Details in +`docs/persistence.md`. + ## Hyperdrive: why api and alerting have separate configs Measured over 6h on prd: `alerting` issued 60,688 Postgres queries/hour against the api's diff --git a/docs/persistence.md b/docs/persistence.md index 5009fa4bf..746c8a8d2 100644 --- a/docs/persistence.md +++ b/docs/persistence.md @@ -88,10 +88,15 @@ bun run --cwd packages/db db:studio ## Deployment and tests -Production migrations are applied by hand, before the Worker deploy: `bun run --cwd packages/db -ps:migrations-preflight main` (read-only, below), then `bun run migrate:prod`, which runs -`drizzle-kit migrate` against PlanetScale's **direct** port 5432. Never run migrations through a -pooler or Hyperdrive. The deployed Worker does not migrate on boot. +The prd deploy applies migrations: `alchemy.run.ts` declares the PlanetScale `main` branch as +`Planetscale.PostgresBranch` with `migrations` pointed at `packages/db/drizzle`, and the api, ai and +alerting Workers carry its name in their env so they upload after it. Bookkeeping is alchemy's +`__alchemy_migrations`; `drizzle.__drizzle_migrations` was copied in once and is frozen, so never run +`drizzle-kit migrate` against prd. The deploy migrates as a temporary role, not `postgres`, so the +branch's default privileges do not cover the tables it creates: a migration that creates one grants +it `TO PUBLIC` itself. The stack registers `Planetscale.providers()`, so `alchemy` commands need +PlanetScale in the alchemy profile or `PLANETSCALE_API_TOKEN_ID` / `PLANETSCALE_API_TOKEN` / +`PLANETSCALE_ORGANIZATION` in the environment. The first v1 migrate on a database migrated by drizzle 0.x upgrades `drizzle.__drizzle_migrations` in place (adds `name` and `applied_at`), matching every existing row to a local folder by diff --git a/knip.json b/knip.json index 193ae61bb..93621cf1b 100644 --- a/knip.json +++ b/knip.json @@ -6,7 +6,9 @@ "workspaces": { ".": { "entry": ["alchemy.run.ts", "scripts/**/*.ts", "scripts/oxlint-plugins/*.mjs"], - "ignoreBinaries": ["tb", "pscale"] + "ignoreBinaries": ["tb", "pscale"], + // alchemy's optional peer, dynamic-imported when the deploy applies migrations. + "ignoreDependencies": ["pg"] }, "apps/web": { // `src/worker-entry.ts` is the deployed Worker entry, named by the vite diff --git a/package.json b/package.json index 10258d4fe..677c0ec69 100644 --- a/package.json +++ b/package.json @@ -21,7 +21,6 @@ "bench:queries": "bun apps/api/scripts/bench-queries.ts", "ch:test": "CLICKHOUSE_E2E=1 CLICKHOUSE_E2E_URL=http://127.0.0.1:8123 bun run --cwd apps/api test scripts/query-bench/catalog.clickhouse.e2e.test.ts && CLICKHOUSE_E2E=1 CLICKHOUSE_E2E_URL=http://127.0.0.1:8123 bun run --cwd packages/backend test src/services/warehouse/WarehouseQueryService.clickhouse.e2e.test.ts src/services/warehouse/web-analytics-parity.clickhouse.e2e.test.ts", "db:migrate:local": "DATABASE_URL=postgres://maple:maple@localhost:5499/maple bun run --cwd packages/db db:migrate", - "migrate:prod": "bun run --cwd packages/db ps:apply-schema main", "backup:restore-test": "bun run --cwd packages/db db:restore-test", "format": "oxfmt", "format:check": "oxfmt --check", @@ -49,7 +48,9 @@ "knip:fix": "bun run --cwd apps/landing sync:i18n && knip --fix --allow-remove-files", "typecheck": "turbo typecheck && tsc -p tsconfig.alchemy.json" }, - "dependencies": {}, + "dependencies": { + "pg": "^8.23.0" + }, "devDependencies": { "@cloudflare/workers-types": "catalog:alchemy", "@effect/platform-node": "4.0.0-rc.112", diff --git a/packages/db/package.json b/packages/db/package.json index b161cc399..95d97ed1e 100644 --- a/packages/db/package.json +++ b/packages/db/package.json @@ -22,7 +22,6 @@ "db:reset-preview": "bun scripts/reset-preview-branch.ts", "db:audit-raw-sql": "bun scripts/audit-raw-sql.ts", "db:normalize-preview": "bun scripts/normalize-preview-ownership.ts", - "ps:apply-schema": "bun scripts/planetscale-apply-schema.ts", "ps:migrations-preflight": "bun scripts/planetscale-migrations-preflight.ts", "db:restore-test": "bun scripts/restore-test.ts", "db:backfill:dashboards-v3": "bun scripts/backfill-dashboard-datasource-v3.ts" diff --git a/packages/db/scripts/ensure-privileges.ts b/packages/db/scripts/ensure-privileges.ts index 33bacc22b..7d2ba0742 100644 --- a/packages/db/scripts/ensure-privileges.ts +++ b/packages/db/scripts/ensure-privileges.ts @@ -9,8 +9,8 @@ * * DATABASE_URL="$MAPLE_PG_URL" bun packages/db/scripts/ensure-privileges.ts * - * `ps:apply-schema` calls `ensureRuntimePrivileges` directly, so the prod path - * needs no separate invocation. + * The deploy migrates as a temporary role these defaults never key to: a + * migration that creates a table must GRANT it to PUBLIC itself. * * ── Why PUBLIC, and why no runtime-role name ────────────────────────────── * Prod has four `pscale_api_*` login roles. Three are members of `postgres` @@ -141,7 +141,7 @@ export const ensureRuntimePrivileges = async (connectionUrl: string): Promise bun packages/db/scripts/planetscale-apply-schema.ts - * - * # examples - * bun packages/db/scripts/planetscale-apply-schema.ts main # prd - * bun packages/db/scripts/planetscale-apply-schema.ts pr-123 - * - * Mints an ephemeral credential for the branch (direct port 5432 — DDL must NOT - * go through the PSBouncer/Hyperdrive poolers), runs `drizzle-kit migrate`, then - * revokes the credential. Idempotent: drizzle skips migrations already recorded - * in `drizzle.__drizzle_migrations`, so re-running is a no-op on an up-to-date - * branch. - */ -import { spawnSync } from "node:child_process" -import { resolve } from "node:path" -import { ensureRuntimePrivileges } from "./ensure-privileges" -import { fail, resolveDatabase, withBranchConnection } from "./planetscale-connection" - -const branch = process.argv[2]?.trim() -if (!branch) { - fail("Usage: bun packages/db/scripts/planetscale-apply-schema.ts ") -} - -const packageDir = resolve(import.meta.dir, "..") - -await withBranchConnection(branch as string, async (connectionUrl) => { - // BEFORE migrate, not after: this installs default privileges, which only - // apply to objects created after they are set. A fresh `CREATE TABLE` carries - // OWNER privileges only and a table-rebuild migration DROPs grants outright, - // so without this the ingest gateway — the one consumer that reads through - // PUBLIC rather than by inheriting `postgres` — hits "permission denied for - // table …" on anything the migration creates. A missing runtime grant makes - // the gateway's startup probe fail and the process exit before serving traffic. - await ensureRuntimePrivileges(connectionUrl) - - console.log(`\n→ Applying schema to ${resolveDatabase()}/${branch} via drizzle-kit migrate\n`) - const proc = spawnSync("bun", ["run", "db:migrate"], { - cwd: packageDir, - env: { ...process.env, DATABASE_URL: connectionUrl }, - stdio: "inherit", - }) - if (proc.status !== 0) { - fail("drizzle-kit migrate failed") - } - console.log(`\n✓ Schema applied to ${resolveDatabase()}/${branch}`) -}) diff --git a/packages/db/scripts/planetscale-migrations-preflight.ts b/packages/db/scripts/planetscale-migrations-preflight.ts index 65691cafd..516965c8c 100644 --- a/packages/db/scripts/planetscale-migrations-preflight.ts +++ b/packages/db/scripts/planetscale-migrations-preflight.ts @@ -1,6 +1,6 @@ /** - * `migrations-preflight.ts` against a PlanetScale branch, over the same - * ephemeral credential `ps:apply-schema` uses. Read-only. + * `migrations-preflight.ts` against a PlanetScale branch, over an ephemeral + * credential. Read-only. * * bun run --cwd packages/db ps:migrations-preflight main */ diff --git a/packages/db/src/migrate.ts b/packages/db/src/migrate.ts index 896c7390b..be5db806d 100644 --- a/packages/db/src/migrate.ts +++ b/packages/db/src/migrate.ts @@ -9,8 +9,7 @@ const migrationsFolder = () => resolve(dirname(fileURLToPath(import.meta.url)), /** * Applies the bundled drizzle migrations to an embedded PGlite instance. - * Local-dev and test path only — production runs `drizzle-kit migrate` by hand - * (`bun run migrate:prod`) before the Worker deploy. + * Local-dev and test path only — prd is migrated by the deploy (`alchemy.run.ts`). */ export const runMigrations = async (pglite: PGlite): Promise => { const db = drizzle({ client: pglite }) diff --git a/packages/infra/src/cloudflare/stack.ts b/packages/infra/src/cloudflare/stack.ts index d2654e864..a73e8154d 100644 --- a/packages/infra/src/cloudflare/stack.ts +++ b/packages/infra/src/cloudflare/stack.ts @@ -1,4 +1,5 @@ import type * as Cloudflare from "alchemy/Cloudflare" +import type * as Planetscale from "alchemy/Planetscale" import * as Context from "effect/Context" import * as Effect from "effect/Effect" import type { WorkerDev } from "@maple/alchemy-portless" @@ -28,6 +29,8 @@ export interface MapleStackContext { * so `.env.local` cannot override them; undefined on a deploy. */ readonly devEnv: Record | undefined + /** prd's PlanetScale branch, whose deploy applies the migrations; undefined on the other stages. */ + readonly dbSchema: Planetscale.PostgresBranch | undefined } /**