From ba7a7cba851cc0093f764a99326e09d514ac00f4 Mon Sep 17 00:00:00 2001 From: Lftobs Date: Thu, 6 Aug 2026 14:17:17 +0100 Subject: [PATCH 1/2] feat(api): implement leader election and migrate ports - Implement `LeaderElection` for multi-instance compatibility - Migrate API port to 17474 and Web port to 17476 - Enable WAL mode and busy timeouts for SQLite - Add Caddy upstream configuration files - Refactor API startup to gate background engines behind leadership --- .github/workflows/release.yml | 3 +- AGENTS.md | 6 +- README.md | 4 +- apps/api/Dockerfile | 2 +- apps/api/src/databases/manager.ts | 2 + apps/api/src/db/client.ts | 3 + apps/api/src/index.ts | 56 ++++- apps/api/src/monitoring/evaluator.ts | 2 + apps/api/src/orchestrator/cleanup.ts | 2 + apps/api/src/orchestrator/pipeline.ts | 7 + apps/api/src/orchestrator/queue.ts | 11 +- apps/api/src/scaling/engine.ts | 2 + apps/api/src/utils/__tests__/leader.test.ts | 91 +++++++ apps/api/src/utils/config-loader.ts | 2 - apps/api/src/utils/config.ts | 12 +- apps/api/src/utils/domain-verifier.ts | 2 + apps/api/src/utils/leader.ts | 62 +++++ apps/docs/src/content/docs/installation.md | 9 +- apps/web/Dockerfile | 4 +- docker-compose.yml | 8 +- infra/caddy/Caddyfile | 12 +- infra/caddy/upstreams/api.caddy | 3 + infra/caddy/upstreams/web.caddy | 1 + infra/monitoring/prometheus.yml | 2 +- scripts/dequel | 253 +++++++++++++++++++- scripts/install.sh | 4 +- 26 files changed, 509 insertions(+), 56 deletions(-) create mode 100644 apps/api/src/utils/__tests__/leader.test.ts create mode 100644 apps/api/src/utils/leader.ts create mode 100644 infra/caddy/upstreams/api.caddy create mode 100644 infra/caddy/upstreams/web.caddy diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 5ee0a56..580ced2 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -69,10 +69,11 @@ jobs: run: | VERSION="${{ steps.version.outputs.VERSION }}" TAR_DIR=_tar - mkdir -p "$TAR_DIR/infra/caddy" "$TAR_DIR/infra/monitoring/grafana/datasources" "$TAR_DIR/scripts/auth" + mkdir -p "$TAR_DIR/infra/caddy" "$TAR_DIR/infra/caddy/upstreams" "$TAR_DIR/infra/monitoring/grafana/datasources" "$TAR_DIR/scripts/auth" cp docker-compose.yml "$TAR_DIR/" cp scripts/dequel "$TAR_DIR/dequel" cp infra/caddy/Caddyfile "$TAR_DIR/infra/caddy/" + cp infra/caddy/upstreams/*.caddy "$TAR_DIR/infra/caddy/upstreams/" cp -r infra/monitoring "$TAR_DIR/infra/" cp -r scripts/auth/* "$TAR_DIR/scripts/auth/" cd "$TAR_DIR" && tar -czf "../dequel-config-${VERSION}.tar.gz" . diff --git a/AGENTS.md b/AGENTS.md index 0652f71..155cfb3 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -5,8 +5,8 @@ Self-hosted deployment platform. Deploy apps from Git, ZIP, or Docker Compose wi ## Tech Stack - **Runtime**: Bun -- **Backend**: ElysiaJS (`apps/api/`) — TypeScript, port 3001 -- **Frontend**: React 18 + Vite + TanStack Router + TanStack Query (`apps/web/`) — port 3000 +- **Backend**: ElysiaJS (`apps/api/`) — TypeScript, port 17474 (fixed) +- **Frontend**: React 18 + Vite + TanStack Router + TanStack Query (`apps/web/`) — port 17476 (fixed) - **Docs**: Astro 4 + Tailwind CSS (`apps/docs/`) — deployed to Vercel - **Database**: SQLite (`data/dequel.db`) — raw SQL - **Queue**: Redis (`ioredis`) for async job queue @@ -175,7 +175,7 @@ Requires secrets: `VERCEL_TOKEN`, `VERCEL_ORG_ID`, `VERCEL_PROJECT_ID` | Variable | Default | Description | |----------|---------|-------------| -| `PORT` | `3001` | API listen port | +| `PORT` | `17474` | API listen port (fixed — must match `infra/caddy/upstreams/api.caddy`) | | `DATABASE_PATH` | `./data/dequel.db` | SQLite database | | `WORKSPACE_ROOT` | `./workspace` | Build staging | | `CADDY_ROUTES_DIR` | `./infra/caddy/routes` | Caddy route output | diff --git a/README.md b/README.md index f447cdb..f4d1d3d 100644 --- a/README.md +++ b/README.md @@ -99,7 +99,7 @@ export DATABASE_PATH=./data/dequel.db \ WORKSPACE_ROOT=./workspace \ CADDY_ROUTES_DIR=./infra/caddy/routes \ DOCKER_NETWORK=dequel_net \ - APP_INTERNAL_PORT=3000 + APP_INTERNAL_PORT=17476 bun apps/api/src/index.ts # Terminal 2 - Web @@ -144,7 +144,7 @@ Key environment variables for the API service: | Variable | Default | Description | |----------|---------|-------------| -| `PORT` | `3001` | API listen port | +| `PORT` | `17474` | API listen port (fixed — must match `infra/caddy/upstreams/api.caddy`) | | `DATABASE_PATH` | `./data/dequel.db` | SQLite database location | | `WORKSPACE_ROOT` | `./workspace` | Build staging directory | | `CADDY_ROUTES_DIR` | `./infra/caddy/routes` | Caddy route output | diff --git a/apps/api/Dockerfile b/apps/api/Dockerfile index a91de2e..207c12f 100644 --- a/apps/api/Dockerfile +++ b/apps/api/Dockerfile @@ -39,6 +39,6 @@ COPY src ./src RUN mkdir -p /app/data /app/workspace /caddy/routes -EXPOSE 3001 +EXPOSE 17474 CMD ["bun", "src/index.ts"] diff --git a/apps/api/src/databases/manager.ts b/apps/api/src/databases/manager.ts index 13a811e..0d5bf98 100644 --- a/apps/api/src/databases/manager.ts +++ b/apps/api/src/databases/manager.ts @@ -2,6 +2,7 @@ import { spawn } from 'node:child_process'; import { config } from '../utils/config'; import { dockerBin } from '../utils/docker-bin'; import { DEQUEL_DATABASE_LABEL } from '../utils/dequel-labels'; +import { leader } from '../utils/leader'; import type { Database, DatabaseType } from '../types'; import { deleteDatabase, @@ -320,6 +321,7 @@ const reconcileMissingContainer = async (dbRecord: Database) => { export const startDatabaseMonitoring = () => { let checkInFlight = false; const check = async () => { + if (!leader.isLeader) return; if (checkInFlight) return; checkInFlight = true; try { diff --git a/apps/api/src/db/client.ts b/apps/api/src/db/client.ts index eac68e3..109338a 100644 --- a/apps/api/src/db/client.ts +++ b/apps/api/src/db/client.ts @@ -6,6 +6,9 @@ let db: Database | null = null; export const getDb = async () => { if (!db) { db = new Database(config.databasePath, { create: true }); + db.exec("PRAGMA journal_mode = WAL;"); + db.exec("PRAGMA busy_timeout = 15000;"); + db.exec("PRAGMA synchronous = NORMAL;"); } return db; }; diff --git a/apps/api/src/index.ts b/apps/api/src/index.ts index acf9ea4..4c3903e 100644 --- a/apps/api/src/index.ts +++ b/apps/api/src/index.ts @@ -15,6 +15,51 @@ import { loadOrCreateJwtSecret } from './utils/secrets'; import { initAuth, cleanupExpiredTokens } from './utils/auth'; import { startBuildCleanup } from './orchestrator/cleanup'; import { startDatabaseMonitoring } from './databases/manager'; +import { leader } from './utils/leader'; + +let enginesStarted = false; +let shuttingDown = false; +let reconciled = false; + +const startLeaderEngines = async () => { + if (enginesStarted || !leader.isLeader) return; + enginesStarted = true; + if (!reconciled) { + reconciled = true; + await orchestrator.reconcileState().catch((error) => + console.error('[API] Reconcile failed', error), + ); + } + console.log('[API] Leadership acquired, starting background engines'); + scalingEngine.start(); + serverManager.start(); + startDomainPolling(); + alertEvaluator.start(); + startBuildCleanup(); + startDatabaseMonitoring(); + setInterval(() => { + if (!leader.isLeader) return; + cleanupExpiredTokens().catch(() => {}); + }, 60_000); +}; + +const shutdown = async (signal: string) => { + if (shuttingDown) return; + shuttingDown = true; + console.log(`[API] ${signal} received, draining deployments and releasing leadership`); + const force = setTimeout(() => process.exit(1), 180_000); + force.unref(); + await Promise.allSettled([ + orchestrator.stopWorker(), + leader.stop(), + ]); + console.log('[API] Drained, shutting down'); + clearTimeout(force); + process.exit(0); +}; +process.on('SIGTERM', () => void shutdown('SIGTERM')); +process.on('SIGINT', () => void shutdown('SIGINT')); + const bootstrap = async () => { await mkdir(dirname(config.databasePath), { recursive: true }); await mkdir(config.workspaceRoot, { recursive: true }); @@ -24,15 +69,10 @@ const bootstrap = async () => { initAuth(jwtSecret); await migrate(); - await orchestrator.reconcileState(); + await leader.start(); + void startLeaderEngines(); + setInterval(() => void startLeaderEngines(), 2_000); orchestrator.startWorker(); - scalingEngine.start(); - serverManager.start(); - startDomainPolling(); - alertEvaluator.start(); - startBuildCleanup(); - startDatabaseMonitoring(); - setInterval(() => { cleanupExpiredTokens().catch(() => {}); }, 60_000); const metrics = { requestsTotal: 0, diff --git a/apps/api/src/monitoring/evaluator.ts b/apps/api/src/monitoring/evaluator.ts index 67cb831..5ef60d1 100644 --- a/apps/api/src/monitoring/evaluator.ts +++ b/apps/api/src/monitoring/evaluator.ts @@ -5,6 +5,7 @@ import { listDeployments, getProjectById } from '../db/repo'; import { sendNotification } from './notifier'; import { dockerBin } from '../utils/docker-bin'; import { run } from '../orchestrator/runtime'; +import { leader } from '../utils/leader'; const NOTIFICATION_KEY = 'dequel:alert:notified'; const NOTIFICATION_COOLDOWN_MS = 300_000; // 5 min between same alert @@ -83,6 +84,7 @@ class AlertEvaluator { } private async tick() { + if (!leader.isLeader) return; try { const db = await getDb(); const alertRows = db.query('SELECT * FROM alerts WHERE enabled = 1').all() as any[]; diff --git a/apps/api/src/orchestrator/cleanup.ts b/apps/api/src/orchestrator/cleanup.ts index 9521f93..d20d58d 100644 --- a/apps/api/src/orchestrator/cleanup.ts +++ b/apps/api/src/orchestrator/cleanup.ts @@ -3,6 +3,7 @@ import { config } from '../utils/config'; import { dockerBin } from '../utils/docker-bin'; import { tryRun } from './runtime'; import { DEQUEL_MANAGED_LABEL } from '../utils/dequel-labels'; +import { leader } from '../utils/leader'; const DLQ_KEY = 'dequel:deploy:dlq'; const GC_INTERVAL_MS = 1_800_000; @@ -38,6 +39,7 @@ export const startBuildCleanup = () => { if (interval) return; console.log('[Cleanup] Docker garbage collector started (every 30min)'); interval = setInterval(async () => { + if (!leader.isLeader) return; await pruneDocker().catch(e => console.warn('[Cleanup] Docker prune failed:', e)); await pruneDlq().catch(e => console.warn('[Cleanup] DLQ prune failed:', e)); }, GC_INTERVAL_MS); diff --git a/apps/api/src/orchestrator/pipeline.ts b/apps/api/src/orchestrator/pipeline.ts index 03e8296..a5f23f7 100644 --- a/apps/api/src/orchestrator/pipeline.ts +++ b/apps/api/src/orchestrator/pipeline.ts @@ -93,6 +93,13 @@ export class PipelineOrchestrator { ); } + async stopWorker() { + if (!this.started) return; + this.started = false; + this.queue.stop(); + await this.queue.drain(); + } + enqueue(deploymentId: string) { this.queue .enqueue(deploymentId) diff --git a/apps/api/src/orchestrator/queue.ts b/apps/api/src/orchestrator/queue.ts index c509642..b929699 100644 --- a/apps/api/src/orchestrator/queue.ts +++ b/apps/api/src/orchestrator/queue.ts @@ -27,6 +27,7 @@ const createRedis = () => new Redis(config.redisUrl, { maxRetriesPerRequest: nul export class DeploymentQueue { private redis: Redis; private shuttingDown = false; + private workers: Promise[] = []; constructor() { this.redis = createRedis(); @@ -49,13 +50,17 @@ export class DeploymentQueue { } async start(handler: (deploymentId: string) => Promise) { - const workers = Array.from({ length: config.queueConcurrency }, (_, i) => this.runWorker(i, handler)); - await Promise.all(workers); + this.workers = Array.from({ length: config.queueConcurrency }, (_, i) => this.runWorker(i, handler)); + await Promise.all(this.workers); } async stop() { this.shuttingDown = true; - await this.redis.quit(); + this.redis.quit().catch(() => {}); + } + + async drain() { + await Promise.allSettled(this.workers); } private async runWorker(workerId: number, handler: (deploymentId: string) => Promise) { diff --git a/apps/api/src/scaling/engine.ts b/apps/api/src/scaling/engine.ts index b527321..09db78c 100644 --- a/apps/api/src/scaling/engine.ts +++ b/apps/api/src/scaling/engine.ts @@ -6,6 +6,7 @@ import { dockerBin } from '../utils/docker-bin'; import { DEQUEL_MANAGED_LABEL } from '../utils/dequel-labels'; import { run, tryRun } from './docker-utils'; import { getScalingPolicy, listDeployments, updateDeploymentStatus, listEnvironmentVariablesForDeploy, getProjectById } from '../db/repo'; +import { leader } from '../utils/leader'; interface ContainerStats { containerName: string; @@ -41,6 +42,7 @@ class ScalingEngine { } private async tick() { + if (!leader.isLeader) return; try { const deployments = await listDeployments(); const running = deployments.filter(d => d.status === 'running' && d.projectId); diff --git a/apps/api/src/utils/__tests__/leader.test.ts b/apps/api/src/utils/__tests__/leader.test.ts new file mode 100644 index 0000000..8e163f2 --- /dev/null +++ b/apps/api/src/utils/__tests__/leader.test.ts @@ -0,0 +1,91 @@ +import { describe, it, expect, mock, beforeEach, afterEach } from 'bun:test'; +import { LeaderElection } from '../leader'; + +const leaderKey = 'dequel:leader'; + +class FakeRedis { + key: string | null = null; + value: string | null = null; + expiresAt = 0; + now = 0; + set = mock(async (_key: string, value: string, _px: string, ttl: number, _nx: string) => { + if (this.key !== null && this.now < this.expiresAt) return null; + this.key = value; + this.value = value; + this.expiresAt = this.now + ttl; + return 'OK'; + }); + pexpire = mock(async (_key: string, ttl: number) => { + if (this.key === null || this.now >= this.expiresAt) return 0; + this.expiresAt = this.now + ttl; + return 1; + }); + eval = mock(async (script: string, _n: number, key: string, token: string) => { + if (this.value === token) { + this.key = null; + this.value = null; + return 1; + } + return 0; + }); + quit = mock(async () => {}); +} + +describe('LeaderElection', () => { + let fake: FakeRedis; + let election: LeaderElection; + + beforeEach(() => { + fake = new FakeRedis(); + election = new LeaderElection(fake as any); + }); + + afterEach(async () => { + await election.stop(); + }); + + it('acquires leadership when the key is free', async () => { + await election.start(); + expect(election.isLeader).toBe(true); + expect(fake.set).toHaveBeenCalledWith(leaderKey, expect.any(String), 'PX', 10000, 'NX'); + }); + + it('fails to acquire when another instance holds the lock', async () => { + fake.key = 'other-token'; + fake.value = 'other-token'; + fake.expiresAt = fake.now + 10000; + await election.start(); + expect(election.isLeader).toBe(false); + }); + + it('renews its own lease and keeps leadership', async () => { + await election.start(); + expect(election.isLeader).toBe(true); + await election['acquire'](); + expect(election.isLeader).toBe(true); + expect(fake.pexpire).toHaveBeenCalledWith(leaderKey, 10000); + }); + + it('re-acquires after losing the lease', async () => { + await election.start(); + expect(election.isLeader).toBe(true); + fake.expiresAt = 0; + fake.now = 100_000; + await election['acquire'](); + expect(election.isLeader).toBe(false); + await election['acquire'](); + expect(election.isLeader).toBe(true); + }); + + it('only deletes the key it holds on release', async () => { + await election.start(); + const token = election['token']; + fake.key = 'replaced-token'; + fake.value = 'replaced-token'; + fake.expiresAt = fake.now + 10000; + await election.release(); + expect(fake.eval).toHaveBeenCalledWith(expect.stringContaining('del'), 1, leaderKey, token); + expect(fake.key).toBe('replaced-token'); + expect(election.isLeader).toBe(false); + }); +}); diff --git a/apps/api/src/utils/config-loader.ts b/apps/api/src/utils/config-loader.ts index f8155c7..3e8a3b4 100644 --- a/apps/api/src/utils/config-loader.ts +++ b/apps/api/src/utils/config-loader.ts @@ -5,13 +5,11 @@ import { homedir } from "node:os"; const XDG_CONFIG_HOME = process.env.XDG_CONFIG_HOME || `${homedir()}/.config`; export interface FileConfig { - port?: number; databasePath?: string; workspaceRoot?: string; caddyRoutesDir?: string; caddyBaseDomain?: string; dockerNetwork?: string; - appInternalPort?: number; buildkitHost?: string; envEncryptionKey?: string; redisUrl?: string; diff --git a/apps/api/src/utils/config.ts b/apps/api/src/utils/config.ts index d311de9..3c16ee0 100644 --- a/apps/api/src/utils/config.ts +++ b/apps/api/src/utils/config.ts @@ -34,20 +34,12 @@ export const config = { databasePath: withFile("DATABASE_PATH", "/app/data/dequel.db"), workspaceRoot: withFile("WORKSPACE_ROOT", "/app/workspace"), caddyRoutesDir: withFile("CADDY_ROUTES_DIR", "/caddy/routes"), - port: withFile( - "PORT", - "17474", - Number, - ), + port: 17474, caddyBaseDomain: withFile( "CADDY_BASE_DOMAIN", "localhost", ), - appInternalPort: withFile( - "APP_INTERNAL_PORT", - "17476", - Number, - ), + appInternalPort: 17476, envEncryptionKey: withFile( "ENV_ENCRYPTION_KEY", "dev-env-key-change-me", diff --git a/apps/api/src/utils/domain-verifier.ts b/apps/api/src/utils/domain-verifier.ts index cffaeeb..697655f 100644 --- a/apps/api/src/utils/domain-verifier.ts +++ b/apps/api/src/utils/domain-verifier.ts @@ -5,6 +5,7 @@ import { validateDomain, resolveServerIp } from './dns'; import { getDb } from '../db/client'; import { getProjectById, listDomains, updateDomainValidation, listEnvironmentVariablesForDeploy } from '../db/repo'; import { reloadCaddy } from '../orchestrator/runtime'; +import { leader } from './leader'; const POLL_INTERVAL = 30_000; @@ -44,6 +45,7 @@ const reconcileVerifiedDomains = async () => { }; const poll = async () => { + if (!leader.isLeader) return; try { const db = await getDb(); const rows = db.query( diff --git a/apps/api/src/utils/leader.ts b/apps/api/src/utils/leader.ts new file mode 100644 index 0000000..c1d431f --- /dev/null +++ b/apps/api/src/utils/leader.ts @@ -0,0 +1,62 @@ +import Redis from "ioredis"; +import { config } from "./config"; + +const LEADER_KEY = "dequel:leader"; +const LEADER_TTL_MS = 10_000; +const RENEW_INTERVAL_MS = 3_000; + +class LeaderElection { + private redis: Redis; + private timer: ReturnType | null = null; + private leadership = false; + private stopped = false; + + constructor() { + this.redis = new Redis(config.redisUrl, { + maxRetriesPerRequest: null, + enableOfflineQueue: false, + }); + } + + async start() { + if (this.timer) return; + await this.acquire(); + this.timer = setInterval(() => { + void this.acquire(); + }, RENEW_INTERVAL_MS); + } + + async stop() { + this.stopped = true; + if (this.timer) { + clearInterval(this.timer); + this.timer = null; + } + await this.release(); + await this.redis.quit().catch(() => {}); + } + + async release() { + await this.redis.del(LEADER_KEY).catch(() => {}); + this.leadership = false; + } + + get isLeader() { + return this.leadership; + } + + private async acquire() { + if (this.stopped) return; + try { + const acquired = await this.redis.set(LEADER_KEY, "1", "PX", LEADER_TTL_MS, "NX"); + this.leadership = acquired === "OK"; + if (this.leadership) { + await this.redis.pexpire(LEADER_KEY, LEADER_TTL_MS).catch(() => {}); + } + } catch { + this.leadership = false; + } + } +} + +export const leader = new LeaderElection(); diff --git a/apps/docs/src/content/docs/installation.md b/apps/docs/src/content/docs/installation.md index 72edfde..e419ef5 100644 --- a/apps/docs/src/content/docs/installation.md +++ b/apps/docs/src/content/docs/installation.md @@ -113,7 +113,7 @@ The `dequel` command manages the platform lifecycle: | `dequel stop` | Stop all services | | `dequel status` | Show service status | | `dequel logs` | Follow service logs | -| `dequel update` | Pull latest images and restart | +| `dequel update` | Pull latest images and roll services (zero-downtime) | | `dequel restart` | Restart all services | | `dequel --help` | Show all commands | @@ -148,7 +148,7 @@ export DATABASE_PATH=./data/dequel.db \ CADDY_ROUTES_DIR=./infra/caddy/routes \ CADDY_BASE_DOMAIN=localhost \ DOCKER_NETWORK=dequel_net \ - APP_INTERNAL_PORT=3000 + APP_INTERNAL_PORT=17476 bun apps/api/src/index.ts # Terminal 2 — Web @@ -161,5 +161,8 @@ bun apps/web/src/main.tsx dequel update ``` -This pulls the latest images from GitHub Container Registry and recreates the services. +This pulls the latest images from GitHub Container Registry and rolls the API and web services with a +zero-downtime blue-green swap. Caddy (the reverse proxy) is reloaded gracefully and never restarts, so +deployed apps and the dashboard keep serving throughout the update. Support services (monitoring, +BuildKit, Redis) are updated in a separate phase without affecting traffic. diff --git a/apps/web/Dockerfile b/apps/web/Dockerfile index dfc3ca2..c50e248 100644 --- a/apps/web/Dockerfile +++ b/apps/web/Dockerfile @@ -19,5 +19,5 @@ RUN bun run build FROM oven/bun:1 WORKDIR /app COPY --from=build /app/dist ./dist -EXPOSE 3000 -CMD ["bun", "x", "serve", "-s", "dist", "-l", "3000"] +EXPOSE 17476 +CMD ["bun", "x", "serve", "-s", "dist", "-l", "17476"] diff --git a/docker-compose.yml b/docker-compose.yml index e893ab2..037e6c7 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -34,13 +34,13 @@ services: # context: ./apps/api # dockerfile: Dockerfile environment: - PORT: 3001 + PORT: 17474 DATABASE_PATH: /app/data/dequel.db WORKSPACE_ROOT: /app/workspace CADDY_ROUTES_DIR: /caddy/routes CADDY_BASE_DOMAIN: ${CADDY_BASE_DOMAIN:-localhost} DOCKER_NETWORK: dequel_net - APP_INTERNAL_PORT: 3000 + APP_INTERNAL_PORT: 17476 BUILDKIT_HOST: tcp://buildkit:1234 DOCKER_BIN: /usr/bin/docker RAILPACK_VERBOSE: "1" @@ -63,7 +63,7 @@ services: "CMD", "curl", "-fsS", - "http://localhost:3001/api/health", + "http://localhost:17474/api/health", ] interval: 50s timeout: 3s @@ -118,7 +118,7 @@ services: "CMD", "bun", "-e", - "fetch('http://localhost:3000/').then(r=>{if(!r.ok)process.exit(1)}).catch(()=>process.exit(1))", + "fetch('http://localhost:17476/').then(r=>{if(!r.ok)process.exit(1)}).catch(()=>process.exit(1))", ] interval: 5s timeout: 3s diff --git a/infra/caddy/Caddyfile b/infra/caddy/Caddyfile index 4cd18df..2ad359b 100644 --- a/infra/caddy/Caddyfile +++ b/infra/caddy/Caddyfile @@ -13,9 +13,7 @@ import /etc/caddy/routes/*.caddy encode zstd gzip handle /api/* { - reverse_proxy api:3001 { - trusted_proxies private_ranges - } + import /etc/caddy/routes/upstreams/api.caddy } handle /metrics* { @@ -29,7 +27,7 @@ import /etc/caddy/routes/*.caddy } handle { - reverse_proxy web:3000 + import /etc/caddy/routes/upstreams/web.caddy } } @@ -42,9 +40,7 @@ import /etc/caddy/routes/*.caddy encode zstd gzip handle /api/* { - reverse_proxy api:3001 { - trusted_proxies private_ranges - } + import /etc/caddy/routes/upstreams/api.caddy } handle /metrics* { @@ -58,6 +54,6 @@ import /etc/caddy/routes/*.caddy } handle { - reverse_proxy web:3000 + import /etc/caddy/routes/upstreams/web.caddy } } diff --git a/infra/caddy/upstreams/api.caddy b/infra/caddy/upstreams/api.caddy new file mode 100644 index 0000000..c1a2784 --- /dev/null +++ b/infra/caddy/upstreams/api.caddy @@ -0,0 +1,3 @@ +reverse_proxy api:17474 { + trusted_proxies private_ranges +} \ No newline at end of file diff --git a/infra/caddy/upstreams/web.caddy b/infra/caddy/upstreams/web.caddy new file mode 100644 index 0000000..216f800 --- /dev/null +++ b/infra/caddy/upstreams/web.caddy @@ -0,0 +1 @@ +reverse_proxy web:17476 \ No newline at end of file diff --git a/infra/monitoring/prometheus.yml b/infra/monitoring/prometheus.yml index 01158b8..86d2def 100644 --- a/infra/monitoring/prometheus.yml +++ b/infra/monitoring/prometheus.yml @@ -10,5 +10,5 @@ scrape_configs: - job_name: 'dequel-api' static_configs: - - targets: ['api:3001'] + - targets: ['api:17474'] metrics_path: /metrics diff --git a/scripts/dequel b/scripts/dequel index e4fa25d..75beb5f 100755 --- a/scripts/dequel +++ b/scripts/dequel @@ -29,6 +29,7 @@ cmd() { } cmd_start() { + ensure_upstreams header "Starting Dequel" cmd up -d local domain="${CADDY_BASE_DOMAIN:-localhost}" @@ -41,12 +42,14 @@ cmd_start() { cmd_stop() { header "Stopping Dequel" + cleanup_greens cmd down success "Dequel stopped." } cmd_restart() { header "Restarting Dequel" + cleanup_greens cmd restart success "Dequel restarted." } @@ -60,8 +63,194 @@ cmd_logs() { cmd logs -f "$@" } +ensure_upstreams() { + local src_dir="$DEQUEL_HOME/infra/caddy/upstreams" + local dir="$DEQUEL_HOME/infra/caddy/routes/upstreams" + mkdir -p "$dir" + for f in api web; do + if [ ! -f "$dir/$f.caddy" ]; then + if [ -f "$src_dir/$f.caddy" ]; then + cp "$src_dir/$f.caddy" "$dir/$f.caddy" + else + write_upstream "$f" "$f" + fi + info "Initialized upstream $f" + fi + done +} + +write_upstream() { + local svc="$1" target="$2" + local port=17476 + [ "$svc" = "api" ] && port=17474 + local file="$DEQUEL_HOME/infra/caddy/routes/upstreams/$svc.caddy" + if [ "$svc" = "api" ]; then + printf 'reverse_proxy %s:%s {\n trusted_proxies private_ranges\n}\n' "$target" "$port" > "$file.tmp" + else + printf 'reverse_proxy %s:%s\n' "$target" "$port" > "$file.tmp" + fi + mv "$file.tmp" "$file" +} + +caddy_container_id() { + docker ps -q --filter "label=com.docker.compose.service=caddy" --filter "network=dequel_net" 2>/dev/null | head -1 +} + +caddy_reload() { + local caddy_id + caddy_id=$(caddy_container_id) + if [ -n "$caddy_id" ]; then + if ! docker exec "$caddy_id" caddy reload --config /etc/caddy/Caddyfile >/dev/null 2>&1; then + fail "Caddy reload failed - old containers still running, update aborted" + fi + else + warn "Caddy container not found, skipping reload" + fi +} + +compose_container_id() { + local svc="$1" + cmd ps -q "$svc" 2>/dev/null | head -1 +} + +service_image() { + local svc="$1" + local image + image=$(docker compose -f "$COMPOSE_FILE" config --format json 2>/dev/null \ + | python3 -c "import json,sys; print(json.load(sys.stdin)['services']['$svc'].get('image',''))" 2>/dev/null) || true + if [ -z "$image" ]; then + local cid + cid=$(compose_container_id "$svc") + [ -n "$cid" ] && image=$(docker inspect -f '{{.Config.Image}}' "$cid" 2>/dev/null || true) + fi + printf '%s' "$image" +} + +green_run_args() { + local svc="$1" + COMPOSE_FILE="$COMPOSE_FILE" python3 - "$svc" <<'PY' +import json, subprocess, sys, os +svc = sys.argv[1] +cfg = json.loads(subprocess.check_output( + ["docker", "compose", "-f", os.environ["COMPOSE_FILE"], "config", "--format", "json"], + text=True)) +s = cfg["services"][svc] +args = [] +for k, v in (s.get("environment") or {}).items(): + args += ["-e", f"{k}={v}"] +base_dir = os.path.dirname(os.path.abspath(os.environ["COMPOSE_FILE"])) +for vol in s.get("volumes") or []: + src = vol.get("source", "") + if vol.get("type") == "bind": + src = src if os.path.isabs(src) else os.path.join(base_dir, src) + dest = vol.get("target", "") + if vol.get("type") == "bind" and not src: + continue + opts = "ro" if vol.get("read_only") else None + args += ["-v", f"{src}:{dest}" + (f":{opts}" if opts else "")] +net = next(iter(s.get("networks") or {}), None) +if net is None: + net = "dequel_net" +else: + net = cfg.get("networks", {}).get(net, {}).get("name", net) +print(json.dumps({"args": args, "network": net})) +PY +} + +wait_healthy() { + local name="$1" svc="$2" timeout="$3" elapsed=0 + while [ "$elapsed" -lt "$timeout" ]; do + if [ "$svc" = "api" ]; then + if docker exec "$name" curl -fsS http://localhost:17474/api/health >/dev/null 2>&1; then + return 0 + fi + else + if docker exec "$name" sh -c "bun -e \"fetch('http://localhost:17476/').then(r=>process.exit(r.ok?0:1)).catch(()=>process.exit(1))\"" >/dev/null 2>&1; then + return 0 + fi + fi + sleep 3 + elapsed=$((elapsed + 3)) + done + return 1 +} + +green_start() { + local svc="$1" image="$2" green="${svc}-green" + if [ -z "$image" ]; then + warn "No image resolved for $svc; will recreate via compose instead" + return 1 + fi + + local spec net args + spec=$(green_run_args "$svc") + net=$(printf '%s' "$spec" | python3 -c "import json,sys; print(json.load(sys.stdin)['network'])") + args=$(printf '%s' "$spec" | python3 -c "import json,sys; print(' '.join(json.load(sys.stdin)['args']))") + + info "Starting $green from new image" + docker rm -f "$green" >/dev/null 2>&1 || true + # shellcheck disable=SC2086 + if ! docker run -d --name "$green" --restart unless-stopped \ + --network "$net" --network-alias "$green" \ + $args \ + "$image" >/dev/null 2>&1; then + warn "Failed to start $green" + return 1 + fi + + if ! wait_healthy "$green" "$svc" 150; then + warn "$svc green container failed health check" + docker rm -f "$green" >/dev/null 2>&1 || true + return 1 + fi + success "$green healthy" + return 0 +} + +green_flip() { + local svc="$1" green="${svc}-green" old + old=$(compose_container_id "$svc") + if [ -z "$old" ]; then + warn "No compose-managed $svc container found; leaving $green in service" + return 1 + fi + + info "Switching Caddy to $green" + write_upstream "$svc" "$green" + caddy_reload + + info "Stopping old $svc (draining in-flight work)" + docker stop -t 120 "$old" >/dev/null 2>&1 || true + docker rm -f "$old" >/dev/null 2>&1 || true + + info "Recreating compose-managed $svc" + cmd up -d --no-deps "$svc" + local new + new=$(compose_container_id "$svc") + if [ -n "$new" ] && wait_healthy "$new" "$svc" 150; then + info "Switching Caddy back to compose-managed $svc" + write_upstream "$svc" "$svc" + caddy_reload + docker stop -t 30 "$green" >/dev/null 2>&1 || true + docker rm -f "$green" >/dev/null 2>&1 || true + success "$svc updated" + else + warn "Compose-managed $svc not healthy; keeping $green in service" + fi +} + +cleanup_greens() { + for g in web api; do + if docker inspect "$g-green" >/dev/null 2>&1; then + docker stop -t 30 "$g-green" >/dev/null 2>&1 || true + docker rm -f "$g-green" >/dev/null 2>&1 || true + info "Removed leftover $g-green" + fi + done +} + cmd_update() { - header "Updating Dequel" + header "Updating Dequel (zero-downtime)" local repo="Lftobs/dequel" local tag="" @@ -89,6 +278,8 @@ cmd_update() { curl -fsSL "$base_url/infra/caddy/Caddyfile" -o "$tmp_dir/Caddyfile" curl -fsSL "$base_url/scripts/dequel" -o "$tmp_dir/dequel" curl -fsSL "$base_url/VERSION" -o "$tmp_dir/VERSION" + curl -fsSL "$base_url/infra/caddy/upstreams/api.caddy" -o "$tmp_dir/api-upstream.caddy" + curl -fsSL "$base_url/infra/caddy/upstreams/web.caddy" -o "$tmp_dir/web-upstream.caddy" mkdir -p "$tmp_dir/infra/monitoring/grafana/datasources" \ "$tmp_dir/infra/monitoring/grafana/dashboards" \ @@ -122,13 +313,17 @@ with open('$tmp_dir/docker-compose.yml', 'w') as f: f.write(content) " 2>/dev/null || true - mkdir -p "$DEQUEL_HOME/infra/caddy" \ + ensure_upstreams + + mkdir -p "$DEQUEL_HOME/infra/caddy/upstreams" \ + "$DEQUEL_HOME/infra/caddy" \ "$DEQUEL_HOME/infra/monitoring/grafana/datasources" \ "$DEQUEL_HOME/infra/monitoring/grafana/dashboards" \ "$DEQUEL_HOME/scripts/auth" mv "$tmp_dir/docker-compose.yml" "$DEQUEL_HOME/docker-compose.yml" - mv "$tmp_dir/Caddyfile" "$DEQUEL_HOME/infra/caddy/Caddyfile" + mv "$tmp_dir/api-upstream.caddy" "$DEQUEL_HOME/infra/caddy/upstreams/api.caddy" + mv "$tmp_dir/web-upstream.caddy" "$DEQUEL_HOME/infra/caddy/upstreams/web.caddy" for f in prometheus.yml loki-config.yml promtail-config.yml; do mv "$tmp_dir/infra/monitoring/$f" "$DEQUEL_HOME/infra/monitoring/$f" @@ -152,10 +347,54 @@ with open('$tmp_dir/docker-compose.yml', 'w') as f: header "Pulling Docker images" cmd pull - header "Recreating services" - cmd up -d + header "Updating support services" + cmd up -d --no-deps buildkit redis pam-auth cadvisor prometheus loki promtail grafana || true + + local web_ready=0 api_ready=0 + header "Starting green containers (current traffic unaffected)" + if green_start web "$(service_image web)"; then + web_ready=1 + else + warn "Web green unavailable; will recreate web in place" + fi + if green_start api "$(service_image api)"; then + api_ready=1 + else + warn "API green unavailable; will recreate api in place" + fi + + local web_target=web api_target=api + [ "$web_ready" = 1 ] && web_target=web-green + [ "$api_ready" = 1 ] && api_target=api-green + + if [ "$web_ready" = 0 ]; then + header "Recreating web in place" + cmd up -d --no-deps web || true + fi + if [ "$api_ready" = 0 ]; then + header "Recreating api in place" + cmd up -d --no-deps api || true + fi + + header "Activating new Caddy configuration" + write_upstream web "$web_target" + write_upstream api "$api_target" + mv "$tmp_dir/Caddyfile" "$DEQUEL_HOME/infra/caddy/Caddyfile" + caddy_reload + + if [ "$web_ready" = 1 ]; then + header "Finishing web roll" + green_flip web + fi + if [ "$api_ready" = 1 ]; then + header "Finishing api roll" + green_flip api + fi + + header "Reconciling stack" + cmd up -d --no-recreate || true - success "Dequel updated to ${tag:-main}" + success "Dequel updated to ${tag:-main} (zero-downtime)" } cmd_uninstall() { @@ -224,7 +463,7 @@ cmd_help() { echo " restart Restart all Dequel services" echo " status Show service status" echo " logs Follow service logs" - echo " update Download latest config, pull images, and recreate services" + echo " update Download latest config, pull images, and roll services (zero-downtime)" echo " uninstall Remove Dequel completely (config, images, volumes)" echo " --version Show version" echo " --help Show this help" diff --git a/scripts/install.sh b/scripts/install.sh index 128882f..201dade 100755 --- a/scripts/install.sh +++ b/scripts/install.sh @@ -68,7 +68,7 @@ check_prerequisites() { setup_directories() { header "Setting up installation directory" - mkdir -p "$INSTALL_DIR/data" "$INSTALL_DIR/workspace" "$INSTALL_DIR/infra/caddy/routes" "$INSTALL_DIR/infra/monitoring/grafana/datasources" "$INSTALL_DIR/infra/monitoring/grafana/dashboards" "$INSTALL_DIR/scripts/auth" + mkdir -p "$INSTALL_DIR/data" "$INSTALL_DIR/workspace" "$INSTALL_DIR/infra/caddy/routes" "$INSTALL_DIR/infra/caddy/upstreams" "$INSTALL_DIR/infra/monitoring/grafana/datasources" "$INSTALL_DIR/infra/monitoring/grafana/dashboards" "$INSTALL_DIR/scripts/auth" info "Installing to: $INSTALL_DIR" } @@ -118,6 +118,8 @@ download_configs() { download_if_missing "$BASE_URL/docker-compose.yml" "$INSTALL_DIR/docker-compose.yml" download_if_missing "$BASE_URL/infra/caddy/Caddyfile" "$INSTALL_DIR/infra/caddy/Caddyfile" + download_if_missing "$BASE_URL/infra/caddy/upstreams/api.caddy" "$INSTALL_DIR/infra/caddy/upstreams/api.caddy" + download_if_missing "$BASE_URL/infra/caddy/upstreams/web.caddy" "$INSTALL_DIR/infra/caddy/upstreams/web.caddy" download_if_missing "$BASE_URL/scripts/dequel" "$INSTALL_DIR/dequel" download_if_missing "$BASE_URL/scripts/auth/pam-server.py" "$INSTALL_DIR/scripts/auth/pam-server.py" From 2f125f4c72c3c5e715bcc959faa4a2fea39d19b6 Mon Sep 17 00:00:00 2001 From: Lftobs Date: Thu, 6 Aug 2026 14:18:20 +0100 Subject: [PATCH 2/2] fix(api): add implement safe leader election with tokens file --- apps/api/src/utils/leader.ts | 55 ++++++++++++++++++++++++++++-------- 1 file changed, 44 insertions(+), 11 deletions(-) diff --git a/apps/api/src/utils/leader.ts b/apps/api/src/utils/leader.ts index c1d431f..8ee05cc 100644 --- a/apps/api/src/utils/leader.ts +++ b/apps/api/src/utils/leader.ts @@ -1,21 +1,41 @@ import Redis from "ioredis"; +import { randomUUID } from "node:crypto"; import { config } from "./config"; const LEADER_KEY = "dequel:leader"; const LEADER_TTL_MS = 10_000; const RENEW_INTERVAL_MS = 3_000; -class LeaderElection { - private redis: Redis; +const RELEASE_SCRIPT = ` +if redis.call('get', KEYS[1]) == ARGV[1] then + return redis.call('del', KEYS[1]) +else + return 0 +end +`; + +const createClient = () => + new Redis(config.redisUrl, { + maxRetriesPerRequest: null, + enableOfflineQueue: false, + }); + +export class LeaderElection { + private redis: Redis | null; private timer: ReturnType | null = null; + private token: string | null = null; private leadership = false; private stopped = false; - constructor() { - this.redis = new Redis(config.redisUrl, { - maxRetriesPerRequest: null, - enableOfflineQueue: false, - }); + constructor(redis?: Redis) { + this.redis = redis ?? null; + } + + private client() { + if (!this.redis) { + this.redis = createClient(); + } + return this.redis; } async start() { @@ -33,12 +53,17 @@ class LeaderElection { this.timer = null; } await this.release(); - await this.redis.quit().catch(() => {}); + await this.client().quit().catch(() => {}); } async release() { - await this.redis.del(LEADER_KEY).catch(() => {}); + if (!this.token) return; + const token = this.token; + this.token = null; this.leadership = false; + await this.client() + .eval(RELEASE_SCRIPT, 1, LEADER_KEY, token) + .catch(() => {}); } get isLeader() { @@ -48,13 +73,21 @@ class LeaderElection { private async acquire() { if (this.stopped) return; try { - const acquired = await this.redis.set(LEADER_KEY, "1", "PX", LEADER_TTL_MS, "NX"); + if (this.token) { + const renewed = await this.client().pexpire(LEADER_KEY, LEADER_TTL_MS); + this.leadership = renewed === 1; + if (!this.leadership) this.token = null; + return; + } + const token = randomUUID(); + const acquired = await this.client().set(LEADER_KEY, token, "PX", LEADER_TTL_MS, "NX"); this.leadership = acquired === "OK"; if (this.leadership) { - await this.redis.pexpire(LEADER_KEY, LEADER_TTL_MS).catch(() => {}); + this.token = token; } } catch { this.leadership = false; + this.token = null; } } }