Chore/deq update

.github/workflows/release.yml

@@ -69,10 +69,11 @@ jobs:
         run: |
           VERSION="${{ steps.version.outputs.VERSION }}"
           TAR_DIR=_tar
-          mkdir -p "$TAR_DIR/infra/caddy" "$TAR_DIR/infra/monitoring/grafana/datasources" "$TAR_DIR/scripts/auth"
+          mkdir -p "$TAR_DIR/infra/caddy" "$TAR_DIR/infra/caddy/upstreams" "$TAR_DIR/infra/monitoring/grafana/datasources" "$TAR_DIR/scripts/auth"
           cp docker-compose.yml "$TAR_DIR/"
           cp scripts/dequel "$TAR_DIR/dequel"
           cp infra/caddy/Caddyfile "$TAR_DIR/infra/caddy/"
+          cp infra/caddy/upstreams/*.caddy "$TAR_DIR/infra/caddy/upstreams/"
           cp -r infra/monitoring "$TAR_DIR/infra/"
           cp -r scripts/auth/* "$TAR_DIR/scripts/auth/"
           cd "$TAR_DIR" && tar -czf "../dequel-config-${VERSION}.tar.gz" .

AGENTS.md

@@ -5,8 +5,8 @@ Self-hosted deployment platform. Deploy apps from Git, ZIP, or Docker Compose wi
 ## Tech Stack
 
 - **Runtime**: Bun
-- **Backend**: ElysiaJS (`apps/api/`) — TypeScript, port 3001
-- **Frontend**: React 18 + Vite + TanStack Router + TanStack Query (`apps/web/`) — port 3000
+- **Backend**: ElysiaJS (`apps/api/`) — TypeScript, port 17474 (fixed)
+- **Frontend**: React 18 + Vite + TanStack Router + TanStack Query (`apps/web/`) — port 17476 (fixed)
 - **Docs**: Astro 4 + Tailwind CSS (`apps/docs/`) — deployed to Vercel
 - **Database**: SQLite (`data/dequel.db`) — raw SQL
 - **Queue**: Redis (`ioredis`) for async job queue
@@ -175,7 +175,7 @@ Requires secrets: `VERCEL_TOKEN`, `VERCEL_ORG_ID`, `VERCEL_PROJECT_ID`
 
 | Variable | Default | Description |
 |----------|---------|-------------|
-| `PORT` | `3001` | API listen port |
+| `PORT` | `17474` | API listen port (fixed — must match `infra/caddy/upstreams/api.caddy`) |
 | `DATABASE_PATH` | `./data/dequel.db` | SQLite database |
 | `WORKSPACE_ROOT` | `./workspace` | Build staging |
 | `CADDY_ROUTES_DIR` | `./infra/caddy/routes` | Caddy route output |

README.md

@@ -99,7 +99,7 @@ export DATABASE_PATH=./data/dequel.db \
        WORKSPACE_ROOT=./workspace \
        CADDY_ROUTES_DIR=./infra/caddy/routes \
        DOCKER_NETWORK=dequel_net \
-       APP_INTERNAL_PORT=3000
+       APP_INTERNAL_PORT=17476
 bun apps/api/src/index.ts
 
 # Terminal 2 - Web
@@ -144,7 +144,7 @@ Key environment variables for the API service:
 
 | Variable | Default | Description |
 |----------|---------|-------------|
-| `PORT` | `3001` | API listen port |
+| `PORT` | `17474` | API listen port (fixed — must match `infra/caddy/upstreams/api.caddy`) |
 | `DATABASE_PATH` | `./data/dequel.db` | SQLite database location |
 | `WORKSPACE_ROOT` | `./workspace` | Build staging directory |
 | `CADDY_ROUTES_DIR` | `./infra/caddy/routes` | Caddy route output |

apps/api/Dockerfile

@@ -39,6 +39,6 @@ COPY src ./src
 
 RUN mkdir -p /app/data /app/workspace /caddy/routes
 
-EXPOSE 3001
+EXPOSE 17474
 
 CMD ["bun", "src/index.ts"]

apps/api/src/databases/manager.ts

@@ -2,6 +2,7 @@ import { spawn } from 'node:child_process';
 import { config } from '../utils/config';
 import { dockerBin } from '../utils/docker-bin';
 import { DEQUEL_DATABASE_LABEL } from '../utils/dequel-labels';
+import { leader } from '../utils/leader';
 import type { Database, DatabaseType } from '../types';
 import {
   deleteDatabase,
@@ -320,6 +321,7 @@ const reconcileMissingContainer = async (dbRecord: Database) => {
 export const startDatabaseMonitoring = () => {
   let checkInFlight = false;
   const check = async () => {
+    if (!leader.isLeader) return;
     if (checkInFlight) return;
     checkInFlight = true;
     try {

apps/api/src/db/client.ts

@@ -6,6 +6,9 @@ let db: Database | null = null;
 export const getDb = async () => {
   if (!db) {
     db = new Database(config.databasePath, { create: true });
+    db.exec("PRAGMA journal_mode = WAL;");
+    db.exec("PRAGMA busy_timeout = 15000;");
+    db.exec("PRAGMA synchronous = NORMAL;");
   }
   return db;
 };

apps/api/src/index.ts

@@ -15,6 +15,51 @@ import { loadOrCreateJwtSecret } from './utils/secrets';
 import { initAuth, cleanupExpiredTokens } from './utils/auth';
 import { startBuildCleanup } from './orchestrator/cleanup';
 import { startDatabaseMonitoring } from './databases/manager';
+import { leader } from './utils/leader';
+
+let enginesStarted = false;
+let shuttingDown = false;
+let reconciled = false;
+
+const startLeaderEngines = async () => {
+  if (enginesStarted || !leader.isLeader) return;
+  enginesStarted = true;
+  if (!reconciled) {
+    reconciled = true;
+    await orchestrator.reconcileState().catch((error) =>
+      console.error('[API] Reconcile failed', error),
+    );
+  }
+  console.log('[API] Leadership acquired, starting background engines');
+  scalingEngine.start();
+  serverManager.start();
+  startDomainPolling();
+  alertEvaluator.start();
+  startBuildCleanup();
+  startDatabaseMonitoring();
+  setInterval(() => {
+    if (!leader.isLeader) return;
+    cleanupExpiredTokens().catch(() => {});
+  }, 60_000);
+};
+
+const shutdown = async (signal: string) => {
+  if (shuttingDown) return;
+  shuttingDown = true;
+  console.log(`[API] ${signal} received, draining deployments and releasing leadership`);
+  const force = setTimeout(() => process.exit(1), 180_000);
+  force.unref();
+  await Promise.allSettled([
+    orchestrator.stopWorker(),
+    leader.stop(),
+  ]);
+  console.log('[API] Drained, shutting down');
+  clearTimeout(force);
+  process.exit(0);
+};
+process.on('SIGTERM', () => void shutdown('SIGTERM'));
+process.on('SIGINT', () => void shutdown('SIGINT'));
+
 const bootstrap = async () => {
   await mkdir(dirname(config.databasePath), { recursive: true });
   await mkdir(config.workspaceRoot, { recursive: true });
@@ -24,15 +69,10 @@ const bootstrap = async () => {
   initAuth(jwtSecret);
 
   await migrate();
-  await orchestrator.reconcileState();
+  await leader.start();
+  void startLeaderEngines();
+  setInterval(() => void startLeaderEngines(), 2_000);
   orchestrator.startWorker();
-  scalingEngine.start();
-  serverManager.start();
-  startDomainPolling();
-  alertEvaluator.start();
-  startBuildCleanup();
-  startDatabaseMonitoring();
-  setInterval(() => { cleanupExpiredTokens().catch(() => {}); }, 60_000);
 
   const metrics = {
     requestsTotal: 0,

apps/api/src/monitoring/evaluator.ts

@@ -5,6 +5,7 @@ import { listDeployments, getProjectById } from '../db/repo';
 import { sendNotification } from './notifier';
 import { dockerBin } from '../utils/docker-bin';
 import { run } from '../orchestrator/runtime';
+import { leader } from '../utils/leader';
 
 const NOTIFICATION_KEY = 'dequel:alert:notified';
 const NOTIFICATION_COOLDOWN_MS = 300_000; // 5 min between same alert
@@ -83,6 +84,7 @@ class AlertEvaluator {
   }
 
   private async tick() {
+    if (!leader.isLeader) return;
     try {
       const db = await getDb();
       const alertRows = db.query('SELECT * FROM alerts WHERE enabled = 1').all() as any[];

apps/api/src/orchestrator/cleanup.ts

@@ -3,6 +3,7 @@ import { config } from '../utils/config';
 import { dockerBin } from '../utils/docker-bin';
 import { tryRun } from './runtime';
 import { DEQUEL_MANAGED_LABEL } from '../utils/dequel-labels';
+import { leader } from '../utils/leader';
 
 const DLQ_KEY = 'dequel:deploy:dlq';
 const GC_INTERVAL_MS = 1_800_000;
@@ -38,6 +39,7 @@ export const startBuildCleanup = () => {
   if (interval) return;
   console.log('[Cleanup] Docker garbage collector started (every 30min)');
   interval = setInterval(async () => {
+    if (!leader.isLeader) return;
     await pruneDocker().catch(e => console.warn('[Cleanup] Docker prune failed:', e));
     await pruneDlq().catch(e => console.warn('[Cleanup] DLQ prune failed:', e));
   }, GC_INTERVAL_MS);

apps/api/src/orchestrator/pipeline.ts

@@ -93,6 +93,13 @@ export class PipelineOrchestrator {
 			);
 	}
 
+	async stopWorker() {
+		if (!this.started) return;
+		this.started = false;
+		this.queue.stop();
+		await this.queue.drain();
+	}
+
 	enqueue(deploymentId: string) {
 		this.queue
 			.enqueue(deploymentId)

apps/api/src/orchestrator/queue.ts

@@ -27,6 +27,7 @@ const createRedis = () => new Redis(config.redisUrl, { maxRetriesPerRequest: nul
 export class DeploymentQueue {
   private redis: Redis;
   private shuttingDown = false;
+  private workers: Promise<void>[] = [];
 
   constructor() {
     this.redis = createRedis();
@@ -49,13 +50,17 @@ export class DeploymentQueue {
   }
 
   async start(handler: (deploymentId: string) => Promise<boolean>) {
-    const workers = Array.from({ length: config.queueConcurrency }, (_, i) => this.runWorker(i, handler));
-    await Promise.all(workers);
+    this.workers = Array.from({ length: config.queueConcurrency }, (_, i) => this.runWorker(i, handler));
+    await Promise.all(this.workers);
   }
 
   async stop() {
     this.shuttingDown = true;
-    await this.redis.quit();
+    this.redis.quit().catch(() => {});
+  }
+
+  async drain() {
+    await Promise.allSettled(this.workers);
   }
 
   private async runWorker(workerId: number, handler: (deploymentId: string) => Promise<boolean>) {

apps/api/src/scaling/engine.ts

@@ -6,6 +6,7 @@ import { dockerBin } from '../utils/docker-bin';
 import { DEQUEL_MANAGED_LABEL } from '../utils/dequel-labels';
 import { run, tryRun } from './docker-utils';
 import { getScalingPolicy, listDeployments, updateDeploymentStatus, listEnvironmentVariablesForDeploy, getProjectById } from '../db/repo';
+import { leader } from '../utils/leader';
 
 interface ContainerStats {
   containerName: string;
@@ -41,6 +42,7 @@ class ScalingEngine {
   }
 
   private async tick() {
+    if (!leader.isLeader) return;
     try {
       const deployments = await listDeployments();
       const running = deployments.filter(d => d.status === 'running' && d.projectId);

apps/api/src/utils/__tests__/leader.test.ts

@@ -0,0 +1,91 @@
+import { describe, it, expect, mock, beforeEach, afterEach } from 'bun:test';
+import { LeaderElection } from '../leader';
+
+const leaderKey = 'dequel:leader';
+
+class FakeRedis {
+  key: string | null = null;
+  value: string | null = null;
+  expiresAt = 0;
+  now = 0;
+  set = mock(async (_key: string, value: string, _px: string, ttl: number, _nx: string) => {
+    if (this.key !== null && this.now < this.expiresAt) return null;
+    this.key = value;
+    this.value = value;
+    this.expiresAt = this.now + ttl;
+    return 'OK';
+  });
+  pexpire = mock(async (_key: string, ttl: number) => {
+    if (this.key === null || this.now >= this.expiresAt) return 0;
+    this.expiresAt = this.now + ttl;
+    return 1;
+  });
+  eval = mock(async (script: string, _n: number, key: string, token: string) => {
+    if (this.value === token) {
+      this.key = null;
+      this.value = null;
+      return 1;
+    }
+    return 0;
+  });
+  quit = mock(async () => {});
+}
+
+describe('LeaderElection', () => {
+  let fake: FakeRedis;
+  let election: LeaderElection;
+
+  beforeEach(() => {
+    fake = new FakeRedis();
+    election = new LeaderElection(fake as any);
+  });
+
+  afterEach(async () => {
+    await election.stop();
+  });
+
+  it('acquires leadership when the key is free', async () => {
+    await election.start();
+    expect(election.isLeader).toBe(true);
+    expect(fake.set).toHaveBeenCalledWith(leaderKey, expect.any(String), 'PX', 10000, 'NX');
+  });
+
+  it('fails to acquire when another instance holds the lock', async () => {
+    fake.key = 'other-token';
+    fake.value = 'other-token';
+    fake.expiresAt = fake.now + 10000;
+    await election.start();
+    expect(election.isLeader).toBe(false);
+  });
+
+  it('renews its own lease and keeps leadership', async () => {
+    await election.start();
+    expect(election.isLeader).toBe(true);
+    await election['acquire']();
+    expect(election.isLeader).toBe(true);
+    expect(fake.pexpire).toHaveBeenCalledWith(leaderKey, 10000);
+  });
+
+  it('re-acquires after losing the lease', async () => {
+    await election.start();
+    expect(election.isLeader).toBe(true);
+    fake.expiresAt = 0;
+    fake.now = 100_000;
+    await election['acquire']();
+    expect(election.isLeader).toBe(false);
+    await election['acquire']();
+    expect(election.isLeader).toBe(true);
+  });
+
+  it('only deletes the key it holds on release', async () => {
+    await election.start();
+    const token = election['token'];
+    fake.key = 'replaced-token';
+    fake.value = 'replaced-token';
+    fake.expiresAt = fake.now + 10000;
+    await election.release();
+    expect(fake.eval).toHaveBeenCalledWith(expect.stringContaining('del'), 1, leaderKey, token);
+    expect(fake.key).toBe('replaced-token');
+    expect(election.isLeader).toBe(false);
+  });
+});

apps/api/src/utils/config-loader.ts

@@ -5,13 +5,11 @@ import { homedir } from "node:os";
 const XDG_CONFIG_HOME = process.env.XDG_CONFIG_HOME || `${homedir()}/.config`;
 
 export interface FileConfig {
-  port?: number;
   databasePath?: string;
   workspaceRoot?: string;
   caddyRoutesDir?: string;
   caddyBaseDomain?: string;
   dockerNetwork?: string;
-  appInternalPort?: number;
   buildkitHost?: string;
   envEncryptionKey?: string;
   redisUrl?: string;

apps/api/src/utils/config.ts

@@ -34,20 +34,12 @@ export const config = {
 	databasePath: withFile<string>("DATABASE_PATH", "/app/data/dequel.db"),
 	workspaceRoot: withFile<string>("WORKSPACE_ROOT", "/app/workspace"),
 	caddyRoutesDir: withFile<string>("CADDY_ROUTES_DIR", "/caddy/routes"),
-	port: withFile<number>(
-		"PORT",
-		"17474",
-		Number,
-	),
+	port: 17474,
 	caddyBaseDomain: withFile<string>(
 		"CADDY_BASE_DOMAIN",
 		"localhost",
 	),
-	appInternalPort: withFile<number>(
-		"APP_INTERNAL_PORT",
-		"17476",
-		Number,
-	),
+	appInternalPort: 17476,
 	envEncryptionKey: withFile<string>(
 		"ENV_ENCRYPTION_KEY",
 		"dev-env-key-change-me",

apps/api/src/utils/domain-verifier.ts

@@ -5,6 +5,7 @@ import { validateDomain, resolveServerIp } from './dns';
 import { getDb } from '../db/client';
 import { getProjectById, listDomains, updateDomainValidation, listEnvironmentVariablesForDeploy } from '../db/repo';
 import { reloadCaddy } from '../orchestrator/runtime';
+import { leader } from './leader';
 
 const POLL_INTERVAL = 30_000;
 
@@ -44,6 +45,7 @@ const reconcileVerifiedDomains = async () => {
 };
 
 const poll = async () => {
+  if (!leader.isLeader) return;
   try {
     const db = await getDb();
     const rows = db.query(