Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
13 changes: 4 additions & 9 deletions .github/workflows/deploy-prd.yml
Original file line number Diff line number Diff line change
Expand Up @@ -59,15 +59,10 @@ jobs:
infisical-project-slug: ${{ vars.INFISICAL_PROJECT_SLUG }}
aws-role-arn: ${{ vars.AWS_DEPLOY_ROLE_ARN }}

# NOTE: prod schema migrations are applied OUT OF BAND (manually, via
# `bun run migrate:prod` → `ps:apply-schema main`, against the direct 5432
# port), NOT by this workflow — so a deploy never touches the prod database
# and needs no MAPLE_PG_URL/admin credential. The worker binds to the
# pre-configured `maple-prd` Hyperdrive (`MapleDb` in packages/infra).
# apply-schema installs default privileges granting PUBLIC before it
# migrates, so new and rebuilt tables are readable by every consumer —
# including the ingest gateway, which reaches Postgres through PSBouncer as
# a role that does NOT inherit `postgres`.
# Schema migrations run in this deploy (`Planetscale.PostgresBranch` in alchemy.run.ts),
# over PLANETSCALE_API_TOKEN_ID / PLANETSCALE_API_TOKEN / PLANETSCALE_ORGANIZATION
# from Infisical. The Workers still bind the dashboard-managed `maple-prd`
# Hyperdrive (`MapleDb`); the deploy needs no database credential of its own.

# alchemy's env-credential path (CI=true) otherwise discovers the account
# with an STS GetCallerIdentity issued while its own AWSEnvironment is
Expand Down
9 changes: 5 additions & 4 deletions CLAUDE.md
Original file line number Diff line number Diff line change
Expand Up @@ -177,10 +177,11 @@ database), reached from Workers via the Hyperdrive binding `MAPLE_DB`.
`connectionTimeoutMillis` to queue waits too. A stalled dial lands as `error.type = ConnectionError`
(a refused one carries the socket code, `ECONNREFUSED`). Fork DB work off a request only with
`forkRequestScoped`, which interrupts it at the response but lets a DB call already under way finish.
- Migrations: `bun run --cwd packages/db db:generate`. Production is applied BY HAND before the
Worker deploy: `bun run --cwd packages/db ps:migrations-preflight main` (read-only; the v1
migrator refuses unmatched rows and replays unrecorded folders), then `bun run migrate:prod`
against the DIRECT port 5432 (never a pooler). PGlite applies them at layer build.
- Migrations: `bun run --cwd packages/db db:generate`. **The prd deploy applies them**: the
PlanetScale `main` branch is an alchemy `Planetscale.PostgresBranch` in `alchemy.run.ts` with
`migrations` pointed at `packages/db/drizzle`; never run `drizzle-kit migrate` against prd. It
migrates as a temporary role, so a migration creating a table must `GRANT` it `TO PUBLIC` itself
(the ingest gateway reads only through PUBLIC). PGlite applies them at layer build.
- **PR preview deploys are label-gated** (2026-08, cost — re-enabled by `fd00bcd412`). A PR gets a
preview only while it carries the `preview` label; `deploy-pr-preview.yml` triggers on
`opened, reopened, synchronize, labeled, unlabeled, closed` and tears the stack down the moment
Expand Down
13 changes: 13 additions & 0 deletions alchemy.run.ts
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,8 @@ import * as AWS from "alchemy/AWS"
import * as Cloudflare from "alchemy/Cloudflare"
import * as Command from "alchemy/Command"
import * as Output from "alchemy/Output"
import * as Planetscale from "alchemy/Planetscale"
import * as RemovalPolicy from "alchemy/RemovalPolicy"
import * as Effect from "effect/Effect"
import * as Layer from "effect/Layer"
import {
Expand Down Expand Up @@ -130,6 +132,16 @@ const MapleStackLive = Layer.effect(
},
workerDev,
devEnv,
// prd's database: the PlanetScale `main` branch, adopted, whose deploy applies the drizzle
// migrations. The Workers that bind it put its name in their env so they upload after it.
dbSchema:
resolveDatabaseMode(stage) === "ref"
? yield* Planetscale.PostgresBranch("maple-db-main", {
database: "maple",
name: "main",
migrations: "packages/db/drizzle",
}).pipe(RemovalPolicy.retain())
: undefined,
}
return context
}),
Expand Down Expand Up @@ -181,6 +193,7 @@ const providers =
Acm.providers().pipe(
Layer.provideMerge(Cloudflare.providers()),
Layer.provideMerge(AWS.providers()),
Layer.provideMerge(Planetscale.providers()),
Layer.provideMerge(Portless.providers()),
)

Expand Down
3 changes: 2 additions & 1 deletion apps/ai/src/worker.ts
Original file line number Diff line number Diff line change
Expand Up @@ -131,7 +131,7 @@ const configuredEnv = (stage: MapleStage) =>
*/
const props = Effect.gen(function* () {
if (globalThis.__ALCHEMY_RUNTIME__) return { main: import.meta.url }
const { stage, workerDev, devEnv } = yield* MapleStack
const { stage, workerDev, devEnv, dbSchema } = yield* MapleStack
// The agents' repository sandbox, reached only over this binding. Absent on
// the stages that do not deploy it, where `SandboxClient` reports the tools
// as unavailable rather than failing.
Expand All @@ -155,6 +155,7 @@ const props = Effect.gen(function* () {
// `devEnv` last, so `.env.local` cannot override the inter-app URLs.
env: {
...makeWorkerBindings({ stage }),
...(dbSchema && { MAPLE_DB_BRANCH: dbSchema.name }),
...(Option.isSome(sandbox) ? { SANDBOX: sandbox.value } : undefined),
...env,
...devEnv,
Expand Down
9 changes: 7 additions & 2 deletions apps/alerting/src/worker.ts
Original file line number Diff line number Diff line change
Expand Up @@ -110,7 +110,7 @@ const configuredEnv = (stage: MapleStage) =>
*/
const props = Effect.gen(function* () {
if (globalThis.__ALCHEMY_RUNTIME__) return { main: import.meta.url }
const { stage, workerDev, devEnv } = yield* MapleStack
const { stage, workerDev, devEnv, dbSchema } = yield* MapleStack
const env = yield* configuredEnv(stage)
return {
main: import.meta.url,
Expand All @@ -121,7 +121,12 @@ const props = Effect.gen(function* () {
dev: workerDev("alerting"),
workersDev: false,
// `devEnv` last, so `.env.local` cannot override the inter-app URLs.
env: { ...makeWorkerBindings({ stage }), ...env, ...devEnv },
env: {
...makeWorkerBindings({ stage }),
...(dbSchema && { MAPLE_DB_BRANCH: dbSchema.name }),
...env,
...devEnv,
},
}
})

Expand Down
3 changes: 2 additions & 1 deletion apps/api/src/worker.ts
Original file line number Diff line number Diff line change
Expand Up @@ -66,7 +66,7 @@ const makeWorkerBindings = ({ stage }: { stage: MapleStage }) => ({
*/
const props = Effect.gen(function* () {
if (globalThis.__ALCHEMY_RUNTIME__) return { main: import.meta.url }
const { stage, domains, workerDev, devEnv } = yield* MapleStack
const { stage, domains, workerDev, devEnv, dbSchema } = yield* MapleStack
// maple-ai, which serves `/mcp` and the chat surface. api keeps the hostname
// and forwards, so the public address and the OAuth identity do not move.
const ai = yield* AiWorker
Expand Down Expand Up @@ -100,6 +100,7 @@ const props = Effect.gen(function* () {
// `devEnv` last, so `.env.local` cannot override the inter-app URLs.
env: {
...makeWorkerBindings({ stage }),
...(dbSchema && { MAPLE_DB_BRANCH: dbSchema.name }),
AI_WORKER: ai,
...configuredEnv,
...devEnv,
Expand Down
3 changes: 3 additions & 0 deletions bun.lock

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

8 changes: 8 additions & 0 deletions docs/infra.md
Original file line number Diff line number Diff line change
Expand Up @@ -405,6 +405,14 @@ the number as load-bearing. The workflow compiles inside `rust:1.94-bookworm` ra
on the runner because the runtime base is `debian:bookworm-slim` (glibc 2.36) while
`ubuntu-24.04` ships 2.39 — a host-built binary dies with `version 'GLIBC_2.39' not found`.

## Schema migrations run in the deploy

The PlanetScale `main` branch is a `Planetscale.PostgresBranch` yielded into `MapleStack` on prd
(`dbSchema`), with `migrations` at `packages/db/drizzle`. Alchemy orders resources only by the
Outputs their props reference, and a Hyperdrive bound by id references nothing, so the api, ai and
alerting Workers put `dbSchema.name` in their env (`MAPLE_DB_BRANCH`) to upload after it. Details in
`docs/persistence.md`.

## Hyperdrive: why api and alerting have separate configs

Measured over 6h on prd: `alerting` issued 60,688 Postgres queries/hour against the api's
Expand Down
13 changes: 9 additions & 4 deletions docs/persistence.md
Original file line number Diff line number Diff line change
Expand Up @@ -88,10 +88,15 @@ bun run --cwd packages/db db:studio

## Deployment and tests

Production migrations are applied by hand, before the Worker deploy: `bun run --cwd packages/db
ps:migrations-preflight main` (read-only, below), then `bun run migrate:prod`, which runs
`drizzle-kit migrate` against PlanetScale's **direct** port 5432. Never run migrations through a
pooler or Hyperdrive. The deployed Worker does not migrate on boot.
The prd deploy applies migrations: `alchemy.run.ts` declares the PlanetScale `main` branch as
`Planetscale.PostgresBranch` with `migrations` pointed at `packages/db/drizzle`, and the api, ai and
alerting Workers carry its name in their env so they upload after it. Bookkeeping is alchemy's
`__alchemy_migrations`; `drizzle.__drizzle_migrations` was copied in once and is frozen, so never run
`drizzle-kit migrate` against prd. The deploy migrates as a temporary role, not `postgres`, so the
branch's default privileges do not cover the tables it creates: a migration that creates one grants
it `TO PUBLIC` itself. The stack registers `Planetscale.providers()`, so `alchemy` commands need
PlanetScale in the alchemy profile or `PLANETSCALE_API_TOKEN_ID` / `PLANETSCALE_API_TOKEN` /
`PLANETSCALE_ORGANIZATION` in the environment.

The first v1 migrate on a database migrated by drizzle 0.x upgrades `drizzle.__drizzle_migrations`
in place (adds `name` and `applied_at`), matching every existing row to a local folder by
Expand Down
4 changes: 3 additions & 1 deletion knip.json
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,9 @@
"workspaces": {
".": {
"entry": ["alchemy.run.ts", "scripts/**/*.ts", "scripts/oxlint-plugins/*.mjs"],
"ignoreBinaries": ["tb", "pscale"]
"ignoreBinaries": ["tb", "pscale"],
// alchemy's optional peer, dynamic-imported when the deploy applies migrations.
"ignoreDependencies": ["pg"]
},
"apps/web": {
// `src/worker-entry.ts` is the deployed Worker entry, named by the vite
Expand Down
5 changes: 3 additions & 2 deletions package.json
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,6 @@
"bench:queries": "bun apps/api/scripts/bench-queries.ts",
"ch:test": "CLICKHOUSE_E2E=1 CLICKHOUSE_E2E_URL=http://127.0.0.1:8123 bun run --cwd apps/api test scripts/query-bench/catalog.clickhouse.e2e.test.ts && CLICKHOUSE_E2E=1 CLICKHOUSE_E2E_URL=http://127.0.0.1:8123 bun run --cwd packages/backend test src/services/warehouse/WarehouseQueryService.clickhouse.e2e.test.ts src/services/warehouse/web-analytics-parity.clickhouse.e2e.test.ts",
"db:migrate:local": "DATABASE_URL=postgres://maple:maple@localhost:5499/maple bun run --cwd packages/db db:migrate",
"migrate:prod": "bun run --cwd packages/db ps:apply-schema main",
"backup:restore-test": "bun run --cwd packages/db db:restore-test",
"format": "oxfmt",
"format:check": "oxfmt --check",
Expand Down Expand Up @@ -49,7 +48,9 @@
"knip:fix": "bun run --cwd apps/landing sync:i18n && knip --fix --allow-remove-files",
"typecheck": "turbo typecheck && tsc -p tsconfig.alchemy.json"
},
"dependencies": {},
"dependencies": {
"pg": "^8.23.0"
},
"devDependencies": {
"@cloudflare/workers-types": "catalog:alchemy",
"@effect/platform-node": "4.0.0-rc.112",
Expand Down
1 change: 0 additions & 1 deletion packages/db/package.json
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,6 @@
"db:reset-preview": "bun scripts/reset-preview-branch.ts",
"db:audit-raw-sql": "bun scripts/audit-raw-sql.ts",
"db:normalize-preview": "bun scripts/normalize-preview-ownership.ts",
"ps:apply-schema": "bun scripts/planetscale-apply-schema.ts",
"ps:migrations-preflight": "bun scripts/planetscale-migrations-preflight.ts",
"db:restore-test": "bun scripts/restore-test.ts",
"db:backfill:dashboards-v3": "bun scripts/backfill-dashboard-datasource-v3.ts"
Expand Down
6 changes: 3 additions & 3 deletions packages/db/scripts/ensure-privileges.ts
Original file line number Diff line number Diff line change
Expand Up @@ -9,8 +9,8 @@
*
* DATABASE_URL="$MAPLE_PG_URL" bun packages/db/scripts/ensure-privileges.ts
*
* `ps:apply-schema` calls `ensureRuntimePrivileges` directly, so the prod path
* needs no separate invocation.
* The deploy migrates as a temporary role these defaults never key to: a
* migration that creates a table must GRANT it to PUBLIC itself.
*
* ── Why PUBLIC, and why no runtime-role name ──────────────────────────────
* Prod has four `pscale_api_*` login roles. Three are members of `postgres`
Expand Down Expand Up @@ -141,7 +141,7 @@ export const ensureRuntimePrivileges = async (connectionUrl: string): Promise<vo
}
}

// CLI entry (skipped when imported by ps:apply-schema).
// CLI entry (skipped when imported).
if (import.meta.main) {
const url = process.env.DATABASE_URL?.trim()
if (!url) {
Expand Down
50 changes: 0 additions & 50 deletions packages/db/scripts/planetscale-apply-schema.ts

This file was deleted.

4 changes: 2 additions & 2 deletions packages/db/scripts/planetscale-migrations-preflight.ts
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
/**
* `migrations-preflight.ts` against a PlanetScale branch, over the same
* ephemeral credential `ps:apply-schema` uses. Read-only.
* `migrations-preflight.ts` against a PlanetScale branch, over an ephemeral
* credential. Read-only.
*
* bun run --cwd packages/db ps:migrations-preflight main
*/
Expand Down
3 changes: 1 addition & 2 deletions packages/db/src/migrate.ts
Original file line number Diff line number Diff line change
Expand Up @@ -9,8 +9,7 @@ const migrationsFolder = () => resolve(dirname(fileURLToPath(import.meta.url)),

/**
* Applies the bundled drizzle migrations to an embedded PGlite instance.
* Local-dev and test path only — production runs `drizzle-kit migrate` by hand
* (`bun run migrate:prod`) before the Worker deploy.
* Local-dev and test path only — prd is migrated by the deploy (`alchemy.run.ts`).
*/
export const runMigrations = async (pglite: PGlite): Promise<void> => {
const db = drizzle({ client: pglite })
Expand Down
3 changes: 3 additions & 0 deletions packages/infra/src/cloudflare/stack.ts
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
import type * as Cloudflare from "alchemy/Cloudflare"
import type * as Planetscale from "alchemy/Planetscale"
import * as Context from "effect/Context"
import * as Effect from "effect/Effect"
import type { WorkerDev } from "@maple/alchemy-portless"
Expand Down Expand Up @@ -28,6 +29,8 @@ export interface MapleStackContext {
* so `.env.local` cannot override them; undefined on a deploy.
*/
readonly devEnv: Record<string, string> | undefined
/** prd's PlanetScale branch, whose deploy applies the migrations; undefined on the other stages. */
readonly dbSchema: Planetscale.PostgresBranch | undefined
}

/**
Expand Down
Loading