diff --git a/.env.example b/.env.example index 146b9cf0..fe6a212b 100644 --- a/.env.example +++ b/.env.example @@ -97,8 +97,9 @@ CORS_ORIGINS=http://localhost:3000 # - failure: record only failed requests # RECORDER_MODE=all -# Directory to store recorded fixtures (default: tests/fixtures) -# RECORDER_FIXTURES_DIR=tests/fixtures +# Directory to store recorded fixtures. +# Production Docker default: /app/data/traffic-recordings (backed by autorouter-data) +# RECORDER_FIXTURES_DIR=/app/data/traffic-recordings # Code default: true # Deployment note: this repository's docker-compose.yml / deploy workflow default to false. diff --git a/Dockerfile b/Dockerfile index 4589958c..424a6d85 100644 --- a/Dockerfile +++ b/Dockerfile @@ -39,6 +39,7 @@ RUN pnpm build # Production stage FROM base AS runner +RUN apk add --no-cache su-exec WORKDIR /app ENV NODE_ENV=production @@ -54,6 +55,7 @@ COPY --from=builder /app/public ./public # Set correct permissions for prerender cache RUN mkdir .next RUN chown nextjs:nodejs .next +RUN mkdir -p /app/data && chown nextjs:nodejs /app/data # Copy standalone build COPY --from=builder --chown=nextjs:nodejs /app/.next/standalone ./ @@ -69,7 +71,7 @@ COPY --from=builder --chown=nextjs:nodejs /app/drizzle ./drizzle COPY --chown=nextjs:nodejs scripts/docker-entrypoint.sh /app/docker-entrypoint.sh RUN chmod +x /app/docker-entrypoint.sh -USER nextjs +USER root EXPOSE 3000 diff --git a/docker-compose.yml b/docker-compose.yml index 6ae15480..74e76a12 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -23,7 +23,7 @@ services: - LOG_RETENTION_DAYS=${LOG_RETENTION_DAYS:-90} - CORS_ORIGINS=${CORS_ORIGINS:-} - RECORDER_ENABLED=${RECORDER_ENABLED:-true} - - RECORDER_FIXTURES_DIR=${RECORDER_FIXTURES_DIR:-tests/fixtures} + - RECORDER_FIXTURES_DIR=${RECORDER_FIXTURES_DIR:-/app/data/traffic-recordings} - RECORDER_REDACT_SENSITIVE=${RECORDER_REDACT_SENSITIVE:-false} - RECORDER_MODE=${RECORDER_MODE:-failure} volumes: diff --git a/docs/guide/deployment/env-reference.md b/docs/guide/deployment/env-reference.md index 8e92af05..8bc67b07 100644 --- a/docs/guide/deployment/env-reference.md +++ b/docs/guide/deployment/env-reference.md @@ -106,9 +106,9 @@ openssl rand -hex 32 唯一仍生效的录制相关 env var 是文件目录： -| 变量 | 必填 | 默认值 | 重启 | 说明 | -| ----------------------- | ---- | ------------------------------------------------------------------------------ | ---- | ------------------------------------------------------------------------------------------------------------------- | -| `RECORDER_FIXTURES_DIR` | 否 | 代码默认 `data/traffic-recordings`；docker-compose.yml 中默认 `tests/fixtures` | 重启 | 录制文件落盘目录。由 `resolveRecordingRoot()` / `getTrafficRecordingRoot()` 直接读取 env var，未走 Runtime Settings | +| 变量 | 必填 | 默认值 | 重启 | 说明 | +| ----------------------- | ---- | --------------------------------------------------------------------------------------------------------------------------- | ---- | ------------------------------------------------------------------------------------------------------------------- | +| `RECORDER_FIXTURES_DIR` | 否 | 代码默认 `data/traffic-recordings`；docker-compose.yml 生产默认 `/app/data/traffic-recordings`（位于 `autorouter-data` 卷） | 重启 | 录制文件落盘目录。由 `resolveRecordingRoot()` / `getTrafficRecordingRoot()` 直接读取 env var，未走 Runtime Settings | ## CLIProxyAPI Sidecar（可选） diff --git a/docs/guide/deployment/persistence-backup.md b/docs/guide/deployment/persistence-backup.md index 59569a57..53b6e9f7 100644 --- a/docs/guide/deployment/persistence-backup.md +++ b/docs/guide/deployment/persistence-backup.md @@ -11,14 +11,14 @@ AutoRouter 的运行状态分布在四个位置：PostgreSQL 数据库、`autoro ## 持久化位置清单 -| 位置 | 形态 | 内容 | 丢失后果 | -| ---------------------------------------------- | -------------------------------- | ---------------------------------------------------------------------------------- | ---------------------------------------------------------------- | -| PostgreSQL 数据库（默认在 `postgres-data` 卷） | docker compose 命名卷 | 上游配置、客户端 Key、熔断状态、请求日志、计费快照、CLIProxy 实例与账号注册 | 系统状态归零，需要重新登记上游与 Key | -| `autorouter-data` 卷 | docker compose 命名卷 | 容器内 `/app/data`；当前主要承载 SQLite 模式的 `dev.sqlite`，PG 部署下该卷基本为空 | 仅 SQLite 模式有影响；PG 部署可忽略 | -| `cliproxy-auth` 卷 | docker compose 命名卷（sidecar） | Codex / Claude / Gemini 的 OAuth token 明文 | 所有账号需要在 CLIProxyAPI 管理端重新 OAuth 登录 | -| `cliproxy-logs` 卷 | docker compose 命名卷（sidecar） | CLIProxyAPI 的运行日志 | 仅丢历史日志，不影响运行 | -| 流量录制目录（`RECORDER_FIXTURES_DIR`） | 容器内目录或绑定挂载 | 已录制的请求 / 响应 fixture（JSON 文件）；数据库 `traffic_recordings` 表仅存索引 | 索引仍在，但 `fixture_path` 指向的文件已丢失，回放与详情查看失效 | -| `ENCRYPTION_KEY`（不在卷里，但同等关键） | `.env` 文件 | Fernet 加密密钥，用于解密上游 API Key、CLIProxy 凭据等敏感字段 | 数据库行还在，但所有加密字段都无法解密；上游配置必须逐条手工重填 | +| 位置 | 形态 | 内容 | 丢失后果 | +| ---------------------------------------------- | -------------------------------- | ---------------------------------------------------------------------------------------------------------------- | -------------------------------------------------------------------- | +| PostgreSQL 数据库（默认在 `postgres-data` 卷） | docker compose 命名卷 | 上游配置、客户端 Key、熔断状态、请求日志、计费快照、CLIProxy 实例与账号注册 | 系统状态归零，需要重新登记上游与 Key | +| `autorouter-data` 卷 | docker compose 命名卷 | 容器内 `/app/data`；生产 PG 部署默认包含 `/app/data/traffic-recordings` 录制文件，SQLite 模式还包含 `dev.sqlite` | 录制文件丢失会使详情 / 回放失效；SQLite 数据库丢失会导致本地数据归零 | +| `cliproxy-auth` 卷 | docker compose 命名卷（sidecar） | Codex / Claude / Gemini 的 OAuth token 明文 | 所有账号需要在 CLIProxyAPI 管理端重新 OAuth 登录 | +| `cliproxy-logs` 卷 | docker compose 命名卷（sidecar） | CLIProxyAPI 的运行日志 | 仅丢历史日志，不影响运行 | +| 自定义流量录制目录（`RECORDER_FIXTURES_DIR`） | 容器内目录或绑定挂载 | 当 env 覆盖默认路径时的请求 / 响应 fixture；数据库 `traffic_recordings` 表仅存索引 | 索引仍在，但 `fixture_path` 指向的文件已丢失，回放与详情查看失效 | +| `ENCRYPTION_KEY`（不在卷里，但同等关键） | `.env` 文件 | Fernet 加密密钥，用于解密上游 API Key、CLIProxy 凭据等敏感字段 | 数据库行还在，但所有加密字段都无法解密；上游配置必须逐条手工重填 | ::: danger 备份策略必须覆盖 .env `.env` 中的 `ENCRYPTION_KEY` 不存在于任何 named volume 中，标准的 `docker volume` 备份命令不会带上它。一旦 `.env` 丢失且没有离线副本，即使 PG 数据库完整恢复，所有上游凭据仍然不可读。`.env` 必须作为独立项纳入备份计划，建议在密码管理器或离线介质中保留至少一份。 @@ -203,27 +203,64 @@ docker compose -f docker-compose.yml -f docker-compose.cliproxy.yml up -d clipro ## 流量录制目录备份 -`recordTrafficFixture`（`src/lib/services/traffic-recorder.ts:517`）把录制内容以 JSON 写到 `RECORDER_FIXTURES_DIR` 指向的目录。数据库 `trafficRecordings` 表只存元数据与 `fixture_path` 路径。这意味着： +`recordTrafficFixture`（`src/lib/services/traffic-recorder.ts:533`）把录制内容以 JSON 写到 `RECORDER_FIXTURES_DIR` 指向的目录。数据库 `trafficRecordings` 表只存元数据与 `fixture_path` 路径。这意味着： - 单独备份 PG 不足以恢复录制；恢复后详情页打开会找不到文件。 - 单独备份录制目录也不够；查询索引、过滤、统计都依赖 PG。 完整的录制备份必须 PG 与录制目录一起做。 -::: danger 默认 RECORDER_FIXTURES_DIR 不是持久路径 -`docker-compose.yml` 中 `RECORDER_FIXTURES_DIR` 的默认值是 `tests/fixtures`（相对于容器内 `/app/`），实际写到 `/app/tests/fixtures`——这个目录在容器内部、**不在任何 named volume 上**。容器一旦重建（`docker compose up -d` 拉新镜像、`docker compose down && up -d` 等）所有录制文件即丢失。 +::: tip 生产默认路径 +`docker-compose.yml` 默认将 `RECORDER_FIXTURES_DIR` 指向 `/app/data/traffic-recordings`，该目录位于 `autorouter-data` named volume 中。容器重建后，录制文件会随卷保留。 +::: + +::: warning 旧部署迁移 +如果旧部署实际使用过 `tests/fixtures`（无论来自 `.env` 显式配置还是旧版本 Compose 默认值），不要直接重建容器。旧目录在容器的可写层中，容器重建后可能丢失；必须先备份仍存在的 fixture 文件和 PostgreSQL，再把文件迁入 `autorouter-data`，同步更新 PostgreSQL 中的 `traffic_recordings.fixture_path`，最后才切换配置。已经随旧容器丢失的文件无法由数据库索引恢复。 +::: + +### 从旧 `tests/fixtures` 切换 -要在生产环境保留录制，必须显式把 `RECORDER_FIXTURES_DIR` 指到挂在持久卷上的子目录。最少改动是把它指到 `autorouter-data` 卷下的子目录： +以下步骤使用仓库默认的 service/container 名称；如果项目名不同，按 `docker volume ls` 的实际卷名替换 `autorouter_autorouter-data`。迁移前先停止应用，避免备份期间继续写入旧目录。`pg_dump` 和 `psql` 从 `db` 容器内读取 Compose 注入的 `POSTGRES_USER`、`POSTGRES_DB`，不依赖宿主 shell 是否导出了 `.env`： -```env -# .env -RECORDER_FIXTURES_DIR=/app/data/traffic-recordings +```bash +mkdir -p backup/autorouter/legacy-fixtures +docker compose stop autorouter +docker cp autorouter:/app/tests/fixtures/. ./backup/autorouter/legacy-fixtures/ +docker compose exec -T db \ + sh -c 'pg_dump -U "$POSTGRES_USER" -d "$POSTGRES_DB"' \ + > backup/autorouter/postgres-before-recording-migration.sql ``` -`docker compose up -d` 让 autorouter 容器读到新值后，录制就会落到 `autorouter-data` 卷里，下面的备份命令才有意义。如果当前部署是默认值，需要在改 `.env` 之前接受「现存的容器内录制将随重建丢失」这一前提。 -::: +把仍存在的 fixture 文件复制到持久化卷，并保留 `tests/fixtures/` 后面的目录结构： + +```bash +docker run --rm \ + -v autorouter_autorouter-data:/target \ + -v "$PWD/backup/autorouter/legacy-fixtures:/source:ro" \ + alpine \ + sh -c 'mkdir -p /target/traffic-recordings && cp -a /source/. /target/traffic-recordings/' +``` + +确认旧路径格式后，将数据库索引改为新绝对路径： + +```bash +docker compose exec -T db \ + sh -c 'psql -U "$POSTGRES_USER" -d "$POSTGRES_DB"' <<'SQL' +UPDATE traffic_recordings +SET fixture_path = CASE + WHEN fixture_path LIKE 'tests/fixtures/%' THEN + '/app/data/traffic-recordings/' || + substring(fixture_path from length('tests/fixtures/') + 1) + WHEN fixture_path LIKE '/app/tests/fixtures/%' THEN + '/app/data/traffic-recordings/' || + substring(fixture_path from length('/app/tests/fixtures/') + 1) + ELSE fixture_path +END +WHERE fixture_path LIKE 'tests/fixtures/%' + OR fixture_path LIKE '/app/tests/fixtures/%'; +``` -把 `RECORDER_FIXTURES_DIR` 指到 `/app/data/...` 之后，录制目录就并入了 `autorouter-data` 卷，备份方式与 `cliproxy-auth` 同套路：用一次性容器 + `tar`。 +最后将 `.env` 改为 `RECORDER_FIXTURES_DIR=/app/data/traffic-recordings` 并执行 `docker compose up -d`。如果暂时必须保留旧 override，就必须另行把宿主机目录挂载到 `/app/tests/fixtures`；`autorouter-data:/app/data` 不会持久化容器可写层中的旧路径。 ```bash docker run --rm \ diff --git a/docs/guide/deployment/upgrade-rollback.md b/docs/guide/deployment/upgrade-rollback.md index ae59caf6..d4bb89d0 100644 --- a/docs/guide/deployment/upgrade-rollback.md +++ b/docs/guide/deployment/upgrade-rollback.md @@ -197,7 +197,7 @@ docker compose up -d ## `.env` 在升级 / 回滚时的最小变更 -正常的升级 / 回滚操作只动 `AUTOROUTER_IMAGE` 一行。其余字段保持原样： +正常的升级 / 回滚操作只动 `AUTOROUTER_IMAGE` 一行，前提是录制文件已经位于持久化的 `/app/data/traffic-recordings` 或其他显式持久化目录。旧部署如果实际使用过 `tests/fixtures`，必须先按[流量录制持久化迁移](./persistence-backup)完成备份、复制和路径迁移，再切换到新 compose；不要在未迁移时直接重建容器。 | 字段 | 升级 / 回滚时是否需要变更 | | ------------------------------------------------------------------ | ------------------------------------------------------------------------------------ | @@ -208,7 +208,7 @@ docker compose up -d | `PORT` | 否。除非有端口冲突需要换 | | `CLIPROXY_*` | 否。除非随版本调整凭据 | | `RECORDER_ENABLED` / `RECORDER_MODE` / `RECORDER_REDACT_SENSITIVE` | 否。这三个变量已迁移为运行时配置，由管理后台 Runtime Settings 控制，不再读取环境变量 | -| `RECORDER_FIXTURES_DIR` | 否。仍有效，控制录制文件存储目录；升级 / 回滚时通常无需改动 | +| `RECORDER_FIXTURES_DIR` | 已使用持久化目录时通常不变；旧 `tests/fixtures` 部署必须先迁移或显式挂载旧目录 | 任何「需要顺手改一下密码 / 密钥」的需求与升级 / 回滚解耦：先单独完成密钥轮换并验证可用，再做版本切换。混在一起做出问题时难以定位是版本还是密钥的问题。 diff --git a/docs/guide/usage/request-recording.md b/docs/guide/usage/request-recording.md index 36e4a172..4a7a0137 100644 --- a/docs/guide/usage/request-recording.md +++ b/docs/guide/usage/request-recording.md @@ -36,8 +36,8 @@ shouldRecordTraffic(outcome) === enabled && (mode === "all" || mode === outcome) 由 env 变量 `RECORDER_FIXTURES_DIR` 指定，未设置时默认 `data/traffic-recordings`（`src/lib/services/traffic-recording-service.ts:10`）。 -::: warning .env.example 注释与实际默认值不一致 -`.env.example:92` 的注释里把默认目录写成 `tests/fixtures`，但源码常量 `DEFAULT_TRAFFIC_RECORDING_ROOT` 是 `data/traffic-recordings`。实际生效值以源码为准——env 不设时录制写到 `data/traffic-recordings`，不是 `tests/fixtures`。 +::: tip +Docker Compose 的生产默认目录是 `/app/data/traffic-recordings`，位于 `autorouter-data` 持久卷中。未设置 env 时，应用代码默认使用 `data/traffic-recordings`。 ::: ### 文件路径与命名 @@ -120,7 +120,7 @@ x-codex-turn-metadata, x-codex-beta-features ### 保留期与清理 -后台任务 `traffic recording cleanup`（已注册到 background sync 注册表，`src/lib/services/background-sync-registry.ts`）按 `retention_days` 字段定期跑，删除超期的 DB 行与对应磁盘文件。 +后台任务 `traffic recording cleanup`（已注册到 background sync 注册表，`src/lib/services/background-sync-registry.ts`）按 `retention_days` 字段定期尝试清理超期录制：只有 fixture 文件存在、位于配置根目录且 DB 与文件删除都成功时，才会同时删除 DB 行和磁盘文件。文件缺失、路径越界或删除失败时保留索引，并在结果中报告失败，便于定位历史数据问题。 手动触发： @@ -128,20 +128,20 @@ x-codex-turn-metadata, x-codex-beta-features POST /api/admin/traffic-recordings/cleanup ``` -返回 `{deleted_count, failure_count, error_summary}`。 +返回 `{deleted_count, failure_count, error_summary}`；`failure_count > 0` 时表示仍有索引未清理，管理页面会以警告提示。 ## 管理 API 全部要求 `Authorization: Bearer `。 -| Method | Path | 行为 | -| -------- | --------------------------------------- | -------------------------------------------------------------------------------------------------------------------- | -| `GET` | `/api/admin/traffic-recording/settings` | 读 Runtime Settings | -| `PATCH` | `/api/admin/traffic-recording/settings` | 更新 Runtime Settings（字段任意可选） | -| `GET` | `/api/admin/traffic-recordings` | 分页列表；过滤 `api_key_id` / `upstream_id` / `request_log_id` / `status_code` / `model` / `start_time` / `end_time` | -| `GET` | `/api/admin/traffic-recordings/[id]` | 返回单条索引元数据 + 内联 `fixture` 字段（磁盘 JSON 内容） | -| `DELETE` | `/api/admin/traffic-recordings/[id]` | 删 DB 行 + 删磁盘文件 | -| `POST` | `/api/admin/traffic-recordings/cleanup` | 立即清理所有超 `retention_days` 的录制 | +| Method | Path | 行为 | +| -------- | --------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------ | +| `GET` | `/api/admin/traffic-recording/settings` | 读 Runtime Settings | +| `PATCH` | `/api/admin/traffic-recording/settings` | 更新 Runtime Settings（字段任意可选） | +| `GET` | `/api/admin/traffic-recordings` | 分页列表；过滤 `api_key_id` / `upstream_id` / `request_log_id` / `status_code` / `model` / `start_time` / `end_time` | +| `GET` | `/api/admin/traffic-recordings/[id]` | 返回单条索引元数据 + 内联 `fixture` 字段（磁盘 JSON 内容） | +| `DELETE` | `/api/admin/traffic-recordings/[id]` | 先删除 DB 行，再尝试删除对应磁盘文件 | +| `POST` | `/api/admin/traffic-recordings/cleanup` | 尝试清理所有超 `retention_days` 的录制；缺失/越界 fixture 的索引会保留并计入失败；文件在数据库索引删除后才消失时按幂等成功处理 | 源文件分别是 `src/app/api/admin/traffic-recording/settings/route.ts`、`src/app/api/admin/traffic-recordings/route.ts`、`.../[id]/route.ts`、`.../cleanup/route.ts`。 diff --git a/scripts/docker-entrypoint.sh b/scripts/docker-entrypoint.sh index 74dba6c5..11a3363d 100644 --- a/scripts/docker-entrypoint.sh +++ b/scripts/docker-entrypoint.sh @@ -1,10 +1,14 @@ #!/bin/sh set -e +DATA_DIR=/app/data +mkdir -p "$DATA_DIR" +chown -R nextjs:nodejs "$DATA_DIR" + echo "[AutoRouter] Running database migrations..." # Run migrations using Node.js with postgres package -node -e " +su-exec nextjs node -e " const postgres = require('postgres'); const fs = require('fs'); const path = require('path'); @@ -80,4 +84,4 @@ runMigrations(); " echo "[AutoRouter] Starting application..." -exec "$@" +exec su-exec nextjs "$@" diff --git a/src/hooks/use-traffic-recording.ts b/src/hooks/use-traffic-recording.ts index 056d3e03..c3fb687e 100644 --- a/src/hooks/use-traffic-recording.ts +++ b/src/hooks/use-traffic-recording.ts @@ -221,6 +221,16 @@ export function useCleanupTrafficRecordings() { onSuccess: (result) => { queryClient.invalidateQueries({ queryKey: ["traffic-recording"] }); queryClient.invalidateQueries({ queryKey: ["background-sync", "tasks"] }); + if (result.failure_count > 0) { + toast.warning( + t("cleanupPartial", { + deleted: result.deleted_count, + failed: result.failure_count, + message: result.error_summary ?? "", + }) + ); + return; + } toast.success(t("cleanupComplete", { count: result.deleted_count })); }, onError: (error: Error) => { diff --git a/src/lib/services/traffic-recording-service.ts b/src/lib/services/traffic-recording-service.ts index cc0905e0..cb05c192 100644 --- a/src/lib/services/traffic-recording-service.ts +++ b/src/lib/services/traffic-recording-service.ts @@ -159,6 +159,32 @@ function assertPathInsideRecordingRoot(filePath: string): string { return resolved; } +async function deleteRecordingFixtureFile(fixturePath: string): Promise { + await unlink(assertPathInsideRecordingRoot(fixturePath)); +} + +async function getExistingRecordingFixtureFile(fixturePath: string): Promise { + const filePath = assertPathInsideRecordingRoot(fixturePath); + await stat(filePath); + return filePath; +} + +async function restoreTrafficRecordingIndex(recording: TrafficRecording): Promise { + try { + await db.insert(trafficRecordings).values(recording).onConflictDoNothing(); + } catch { + await db + .insert(trafficRecordings) + .values({ + ...recording, + requestLogId: null, + apiKeyId: null, + upstreamId: null, + }) + .onConflictDoNothing(); + } +} + /** Return the configured fixture root for recorded traffic files. */ export function getTrafficRecordingRoot(): string { return process.env.RECORDER_FIXTURES_DIR || DEFAULT_TRAFFIC_RECORDING_ROOT; @@ -382,9 +408,14 @@ export async function deleteTrafficRecording(id: string): Promise { }); if (!row) return false; + const deletedRows = await db + .delete(trafficRecordings) + .where(eq(trafficRecordings.id, id)) + .returning({ id: trafficRecordings.id }); + if (deletedRows.length === 0) return false; + try { - const filePath = assertPathInsideRecordingRoot(row.fixturePath); - await unlink(filePath); + await deleteRecordingFixtureFile(row.fixturePath); } catch (error) { const code = error && typeof error === "object" && "code" in error ? error.code : null; if (code !== "ENOENT") { @@ -392,7 +423,6 @@ export async function deleteTrafficRecording(id: string): Promise { } } - await db.delete(trafficRecordings).where(eq(trafficRecordings.id, id)); return true; } @@ -412,12 +442,40 @@ export async function cleanupExpiredTrafficRecordings( const failures: string[] = []; for (const recording of expired) { - const deleted = await deleteTrafficRecording(recording.id); - if (deleted) { + let deletedFromDatabase = false; + try { + const filePath = await getExistingRecordingFixtureFile(recording.fixturePath); + const deletedRows = await db + .delete(trafficRecordings) + .where(eq(trafficRecordings.id, recording.id)) + .returning({ id: trafficRecordings.id }); + if (deletedRows.length === 0) continue; + deletedFromDatabase = true; + await unlink(filePath); deletedCount += 1; - } else { + } catch (error) { + const errorCode = error && typeof error === "object" && "code" in error ? error.code : null; + if (deletedFromDatabase && errorCode === "ENOENT") { + deletedCount += 1; + continue; + } + if (deletedFromDatabase) { + try { + await restoreTrafficRecordingIndex(recording); + } catch (restoreError) { + log.error( + { err: restoreError, recordingId: recording.id }, + "failed to restore traffic recording index after fixture cleanup failure" + ); + } + } failureCount += 1; - failures.push(recording.id); + const message = error instanceof Error ? error.message : String(error); + failures.push(`${recording.id}: ${message}`); + log.warn( + { err: error, recordingId: recording.id }, + "failed to clean up expired traffic recording" + ); } } diff --git a/src/messages/en.json b/src/messages/en.json index 1d1bfa6f..8263dce5 100644 --- a/src/messages/en.json +++ b/src/messages/en.json @@ -1631,6 +1631,7 @@ "filters": "Filters", "cleanupExpired": "Clean expired", "cleanupComplete": "Cleaned {count} expired recordings", + "cleanupPartial": "Cleaned {deleted} expired recordings; {failed} failed. {message}", "cleanupFailed": "Cleanup failed: {message}", "statusFilter": "Status filter", "statusAll": "All statuses", diff --git a/src/messages/zh-CN.json b/src/messages/zh-CN.json index 3ad2f83f..ad73ac6c 100644 --- a/src/messages/zh-CN.json +++ b/src/messages/zh-CN.json @@ -1631,6 +1631,7 @@ "filters": "筛选", "cleanupExpired": "清理过期记录", "cleanupComplete": "已清理 {count} 条过期录制", + "cleanupPartial": "已清理 {deleted} 条过期录制；{failed} 条失败。{message}", "cleanupFailed": "清理失败：{message}", "statusFilter": "状态码筛选", "statusAll": "全部状态", diff --git a/tests/unit/deployment/traffic-recording-persistence.test.ts b/tests/unit/deployment/traffic-recording-persistence.test.ts new file mode 100644 index 00000000..f0171358 --- /dev/null +++ b/tests/unit/deployment/traffic-recording-persistence.test.ts @@ -0,0 +1,96 @@ +import { execFileSync } from "node:child_process"; +import { chmodSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs"; +import { relative, resolve } from "node:path"; +import { describe, expect, it } from "vitest"; + +const entrypointPath = resolve(__dirname, "../../../scripts/docker-entrypoint.sh"); +const compose = readFileSync(resolve(__dirname, "../../../docker-compose.yml"), "utf8"); +const dockerfile = readFileSync(resolve(__dirname, "../../../Dockerfile"), "utf8"); +const entrypoint = readFileSync(entrypointPath, "utf8"); +const autorouterService = + compose.match(/\n autorouter:\n([\s\S]*?)(?=\n [a-zA-Z][\w-]*:\n|\nvolumes:\n)/)?.[1] ?? ""; + +describe("production traffic recording storage", () => { + it("defaults fixtures to the persistent application data volume", () => { + expect(autorouterService).not.toBe(""); + expect(autorouterService).toContain( + "RECORDER_FIXTURES_DIR=${RECORDER_FIXTURES_DIR:-/app/data/traffic-recordings}" + ); + expect(autorouterService).toMatch(/-\s+autorouter-data:\/app\/data/); + }); + + it("initializes the mounted data volume for the runtime user", () => { + const dataInitialization = dockerfile.indexOf( + "RUN mkdir -p /app/data && chown nextjs:nodejs /app/data" + ); + const rootUser = dockerfile.indexOf("USER root"); + + expect(dataInitialization).toBeGreaterThanOrEqual(0); + expect(dataInitialization).toBeLessThan(rootUser); + }); + + it("repairs existing volume ownership before starting as nextjs", () => { + const ownershipRepair = entrypoint.indexOf('chown -R nextjs:nodejs "$DATA_DIR"'); + const migrations = entrypoint.indexOf('su-exec nextjs node -e "'); + const application = entrypoint.indexOf('exec su-exec nextjs "$@"'); + + expect(dockerfile).toContain("RUN apk add --no-cache su-exec"); + expect(dockerfile).toContain("USER root"); + expect(ownershipRepair).toBeGreaterThanOrEqual(0); + expect(migrations).toBeGreaterThan(ownershipRepair); + expect(application).toBeGreaterThan(migrations); + }); + it("runs ownership repair, migrations, and app startup in order", () => { + const repoRoot = resolve(__dirname, "../../.."); + const tempRoot = mkdtempSync(resolve(repoRoot, ".tmp-entrypoint-")); + const relativeRoot = relative(repoRoot, tempRoot).replaceAll("\\", "/"); + const binRoot = resolve(tempRoot, "bin"); + const logPath = resolve(tempRoot, "calls.log"); + mkdirSync(binRoot, { recursive: true }); + const scriptPath = resolve(tempRoot, "entrypoint.sh"); + + try { + writeFileSync( + scriptPath, + entrypoint.replace("DATA_DIR=/app/data", `DATA_DIR=./${relativeRoot}/data`) + ); + writeFileSync( + resolve(binRoot, "chown"), + '#!/bin/sh\nprintf \'%s\\n\' "chown:$*" >> "$ENTRYPOINT_TEST_LOG"\n', + { mode: 0o755 } + ); + writeFileSync( + resolve(binRoot, "su-exec"), + [ + "#!/bin/sh", + 'printf \'%s\\n\' "su-exec:$1:$2" >> "$ENTRYPOINT_TEST_LOG"', + '[ "$2" = "node" ] && exit 0', + '[ "$1" = "nextjs" ] && [ "$2" = "app-smoke" ] && exit 0', + "exit 1", + "", + ].join("\n"), + { mode: 0o755 } + ); + chmodSync(resolve(binRoot, "chown"), 0o755); + chmodSync(resolve(binRoot, "su-exec"), 0o755); + + const pathSeparator = process.platform === "win32" ? ";" : ":"; + execFileSync("sh", [scriptPath, "app-smoke"], { + cwd: repoRoot, + env: { + ...process.env, + DATABASE_URL: "postgres://entrypoint-test", + ENTRYPOINT_TEST_LOG: `./${relativeRoot}/calls.log`, + PATH: `./${relativeRoot}/bin${pathSeparator}${process.env.PATH ?? ""}`, + }, + stdio: "pipe", + }); + + const calls = readFileSync(logPath, "utf8").trim().split(/\r?\n/); + expect(calls[0]).toContain("chown:-R nextjs:nodejs"); + expect(calls.slice(1)).toEqual(["su-exec:nextjs:node", "su-exec:nextjs:app-smoke"]); + } finally { + rmSync(tempRoot, { recursive: true, force: true }); + } + }); +}); diff --git a/tests/unit/hooks/use-traffic-recording.test.ts b/tests/unit/hooks/use-traffic-recording.test.ts new file mode 100644 index 00000000..cdd2a54f --- /dev/null +++ b/tests/unit/hooks/use-traffic-recording.test.ts @@ -0,0 +1,82 @@ +import { createElement } from "react"; +import { QueryClient, QueryClientProvider } from "@tanstack/react-query"; +import { renderHook, waitFor } from "@testing-library/react"; +import { beforeEach, describe, expect, it, vi } from "vitest"; +import { useCleanupTrafficRecordings } from "@/hooks/use-traffic-recording"; + +const { mockPost, toastSuccess, toastWarning, toastError } = vi.hoisted(() => ({ + mockPost: vi.fn(), + toastSuccess: vi.fn(), + toastWarning: vi.fn(), + toastError: vi.fn(), +})); + +vi.mock("@/providers/auth-provider", () => ({ + useAuth: () => ({ + apiClient: { post: mockPost }, + }), +})); + +vi.mock("next-intl", () => ({ + useTranslations: () => (key: string, values?: Record) => + `${key}:${JSON.stringify(values ?? {})}`, +})); + +vi.mock("sonner", () => ({ + toast: { + success: toastSuccess, + warning: toastWarning, + error: toastError, + }, +})); + +describe("useCleanupTrafficRecordings", () => { + let queryClient: QueryClient; + + const wrapper = ({ children }: { children: React.ReactNode }) => + createElement(QueryClientProvider, { client: queryClient }, children); + + beforeEach(() => { + queryClient = new QueryClient({ + defaultOptions: { + queries: { retry: false }, + mutations: { retry: false }, + }, + }); + vi.clearAllMocks(); + }); + + it("warns when cleanup reports retained indexes", async () => { + mockPost.mockResolvedValueOnce({ + deleted_count: 2, + failure_count: 1, + error_summary: "recording-1: fixture missing", + }); + + const { result } = renderHook(() => useCleanupTrafficRecordings(), { wrapper }); + result.current.mutate(); + + await waitFor(() => expect(result.current.isSuccess).toBe(true)); + expect(toastWarning).toHaveBeenCalledWith( + 'cleanupPartial:{"deleted":2,"failed":1,"message":"recording-1: fixture missing"}' + ); + expect(toastSuccess).not.toHaveBeenCalled(); + expect(toastError).not.toHaveBeenCalled(); + }); + + it("shows success when cleanup has no failures", async () => { + mockPost.mockResolvedValueOnce({ + deleted_count: 2, + failure_count: 0, + error_summary: null, + }); + + const { result } = renderHook(() => useCleanupTrafficRecordings(), { wrapper }); + result.current.mutate(); + + await waitFor(() => expect(result.current.isSuccess).toBe(true)); + expect(toastSuccess).toHaveBeenCalledWith('cleanupComplete:{"count":2}'); + expect(toastWarning).not.toHaveBeenCalled(); + expect(toastError).not.toHaveBeenCalled(); + }); +}); diff --git a/tests/unit/services/traffic-recording-service.test.ts b/tests/unit/services/traffic-recording-service.test.ts index 64ef9c32..1d24ccd4 100644 --- a/tests/unit/services/traffic-recording-service.test.ts +++ b/tests/unit/services/traffic-recording-service.test.ts @@ -330,13 +330,276 @@ describe("traffic-recording-service", () => { findFirstMock.mockResolvedValueOnce(recordingRow()); unlinkMock.mockResolvedValueOnce(undefined); dbDeleteMock.mockReturnValueOnce({ - where: vi.fn().mockResolvedValue(undefined), + where: vi.fn().mockReturnValue({ + returning: vi.fn().mockResolvedValue([{ id: "recording-1" }]), + }), }); await expect(deleteTrafficRecording("recording-1")).resolves.toBe(true); expect(unlinkMock).toHaveBeenCalledTimes(1); + + expect(dbDeleteMock).toHaveBeenCalledTimes(1); + }); + it("does not remove a fixture when another cleanup already claimed the row", async () => { + const { deleteTrafficRecording } = await import("@/lib/services/traffic-recording-service"); + + findFirstMock.mockResolvedValueOnce(recordingRow()); + dbDeleteMock.mockReturnValueOnce({ + where: vi.fn().mockReturnValue({ + returning: vi.fn().mockResolvedValue([]), + }), + }); + + await expect(deleteTrafficRecording("recording-1")).resolves.toBe(false); + expect(unlinkMock).not.toHaveBeenCalled(); + }); + it("preserves expired indexes when a fixture file is missing", async () => { + const { cleanupExpiredTrafficRecordings } = + await import("@/lib/services/traffic-recording-service"); + + dbInsertMock.mockReturnValueOnce({ + values: vi.fn().mockReturnValue({ + onConflictDoNothing: vi.fn().mockResolvedValue(undefined), + }), + }); + findFirstMock.mockResolvedValueOnce(settingsRow()); + findManyMock.mockResolvedValueOnce([ + recordingRow({ createdAt: new Date("2025-12-01T00:00:00.000Z") }), + ]); + statMock.mockRejectedValueOnce(Object.assign(new Error("fixture missing"), { code: "ENOENT" })); + + const result = await cleanupExpiredTrafficRecordings(new Date("2026-01-01T00:00:00.000Z")); + + expect(result.deletedCount).toBe(0); + expect(result.failureCount).toBe(1); + expect(result.errorSummary).toContain("recording-1"); + expect(unlinkMock).not.toHaveBeenCalled(); + expect(dbDeleteMock).not.toHaveBeenCalled(); + }); + + it("preserves expired indexes whose fixture path is outside the configured root", async () => { + const { cleanupExpiredTrafficRecordings } = + await import("@/lib/services/traffic-recording-service"); + + process.env.RECORDER_FIXTURES_DIR = "custom-fixtures"; + dbInsertMock.mockReturnValueOnce({ + values: vi.fn().mockReturnValue({ + onConflictDoNothing: vi.fn().mockResolvedValue(undefined), + }), + }); + findFirstMock.mockResolvedValueOnce(settingsRow()); + findManyMock.mockResolvedValueOnce([ + recordingRow({ createdAt: new Date("2025-12-01T00:00:00.000Z") }), + ]); + + const result = await cleanupExpiredTrafficRecordings(new Date("2026-01-01T00:00:00.000Z")); + + expect(result.deletedCount).toBe(0); + expect(result.failureCount).toBe(1); + expect(result.errorSummary).toContain("recording-1"); + expect(unlinkMock).not.toHaveBeenCalled(); + expect(dbDeleteMock).not.toHaveBeenCalled(); + }); + + it("deletes expired fixtures and indexes after a successful preflight", async () => { + const { cleanupExpiredTrafficRecordings } = + await import("@/lib/services/traffic-recording-service"); + + dbInsertMock.mockReturnValueOnce({ + values: vi.fn().mockReturnValue({ + onConflictDoNothing: vi.fn().mockResolvedValue(undefined), + }), + }); + findFirstMock.mockResolvedValueOnce(settingsRow()); + findManyMock.mockResolvedValueOnce([ + recordingRow({ createdAt: new Date("2025-12-01T00:00:00.000Z") }), + ]); + statMock.mockResolvedValueOnce({}); + dbDeleteMock.mockReturnValueOnce({ + where: vi.fn().mockReturnValue({ + returning: vi.fn().mockResolvedValue([{ id: "recording-1" }]), + }), + }); + unlinkMock.mockResolvedValueOnce(undefined); + + const result = await cleanupExpiredTrafficRecordings(new Date("2026-01-01T00:00:00.000Z")); + + expect(result.deletedCount).toBe(1); + expect(result.failureCount).toBe(0); + expect(dbDeleteMock).toHaveBeenCalledTimes(1); + expect(unlinkMock).toHaveBeenCalledTimes(1); + expect(dbDeleteMock.mock.invocationCallOrder[0]).toBeLessThan( + unlinkMock.mock.invocationCallOrder[0] + ); + }); + + it("preserves the index when database deletion fails before fixture removal", async () => { + const { cleanupExpiredTrafficRecordings } = + await import("@/lib/services/traffic-recording-service"); + + dbInsertMock.mockReturnValueOnce({ + values: vi.fn().mockReturnValue({ + onConflictDoNothing: vi.fn().mockResolvedValue(undefined), + }), + }); + findFirstMock.mockResolvedValueOnce(settingsRow()); + findManyMock.mockResolvedValueOnce([ + recordingRow({ createdAt: new Date("2025-12-01T00:00:00.000Z") }), + ]); + statMock.mockResolvedValueOnce({}); + dbDeleteMock.mockReturnValueOnce({ + where: vi.fn().mockReturnValue({ + returning: vi.fn().mockRejectedValueOnce(new Error("database unavailable")), + }), + }); + + const result = await cleanupExpiredTrafficRecordings(new Date("2026-01-01T00:00:00.000Z")); + + expect(result.deletedCount).toBe(0); + expect(result.failureCount).toBe(1); + + expect(result.errorSummary).toContain("database unavailable"); + expect(unlinkMock).not.toHaveBeenCalled(); expect(dbDeleteMock).toHaveBeenCalledTimes(1); }); + it("does not restore or remove a fixture when another cleanup wins", async () => { + const { cleanupExpiredTrafficRecordings } = + await import("@/lib/services/traffic-recording-service"); + + dbInsertMock.mockReturnValueOnce({ + values: vi.fn().mockReturnValue({ + onConflictDoNothing: vi.fn().mockResolvedValue(undefined), + }), + }); + findFirstMock.mockResolvedValueOnce(settingsRow()); + findManyMock.mockResolvedValueOnce([ + recordingRow({ createdAt: new Date("2025-12-01T00:00:00.000Z") }), + ]); + statMock.mockResolvedValueOnce({}); + dbDeleteMock.mockReturnValueOnce({ + where: vi.fn().mockReturnValue({ + returning: vi.fn().mockResolvedValue([]), + }), + }); + + const result = await cleanupExpiredTrafficRecordings(new Date("2026-01-01T00:00:00.000Z")); + + expect(result.deletedCount).toBe(0); + expect(result.failureCount).toBe(0); + expect(unlinkMock).not.toHaveBeenCalled(); + expect(dbInsertMock).toHaveBeenCalledTimes(1); + }); + + it("restores the index when fixture removal fails after database deletion", async () => { + const { cleanupExpiredTrafficRecordings } = + await import("@/lib/services/traffic-recording-service"); + + const insertSettings = { + values: vi.fn().mockReturnValue({ + onConflictDoNothing: vi.fn().mockResolvedValue(undefined), + }), + }; + const restoreIndex = { + values: vi.fn().mockReturnValue({ + onConflictDoNothing: vi.fn().mockResolvedValue(undefined), + }), + }; + dbInsertMock.mockReturnValueOnce(insertSettings).mockReturnValueOnce(restoreIndex); + findFirstMock.mockResolvedValueOnce(settingsRow()); + findManyMock.mockResolvedValueOnce([ + recordingRow({ createdAt: new Date("2025-12-01T00:00:00.000Z") }), + ]); + statMock.mockResolvedValueOnce({}); + dbDeleteMock.mockReturnValueOnce({ + where: vi.fn().mockReturnValue({ + returning: vi.fn().mockResolvedValue([{ id: "recording-1" }]), + }), + }); + unlinkMock.mockRejectedValueOnce(new Error("fixture is read-only")); + + const result = await cleanupExpiredTrafficRecordings(new Date("2026-01-01T00:00:00.000Z")); + + expect(result.deletedCount).toBe(0); + expect(result.failureCount).toBe(1); + expect(result.errorSummary).toContain("fixture is read-only"); + expect(dbInsertMock).toHaveBeenCalledTimes(2); + expect(unlinkMock).toHaveBeenCalledTimes(1); + }); + + it("detaches deleted foreign keys when restoring the index", async () => { + const { cleanupExpiredTrafficRecordings } = + await import("@/lib/services/traffic-recording-service"); + + const insertSettings = { + values: vi.fn().mockReturnValue({ + onConflictDoNothing: vi.fn().mockResolvedValue(undefined), + }), + }; + const restoreWithRelations = { + values: vi.fn().mockReturnValue({ + onConflictDoNothing: vi.fn().mockRejectedValueOnce(new Error("foreign key violation")), + }), + }; + const restoreDetached = { + values: vi.fn().mockReturnValue({ + onConflictDoNothing: vi.fn().mockResolvedValue(undefined), + }), + }; + dbInsertMock + .mockReturnValueOnce(insertSettings) + .mockReturnValueOnce(restoreWithRelations) + .mockReturnValueOnce(restoreDetached); + findFirstMock.mockResolvedValueOnce(settingsRow()); + findManyMock.mockResolvedValueOnce([ + recordingRow({ createdAt: new Date("2025-12-01T00:00:00.000Z") }), + ]); + statMock.mockResolvedValueOnce({}); + dbDeleteMock.mockReturnValueOnce({ + where: vi.fn().mockReturnValue({ + returning: vi.fn().mockResolvedValue([{ id: "recording-1" }]), + }), + }); + unlinkMock.mockRejectedValueOnce(new Error("fixture is read-only")); + + const result = await cleanupExpiredTrafficRecordings(new Date("2026-01-01T00:00:00.000Z")); + + expect(result.failureCount).toBe(1); + expect(restoreDetached.values).toHaveBeenCalledWith( + expect.objectContaining({ requestLogId: null, apiKeyId: null, upstreamId: null }) + ); + }); + + it("treats a fixture that vanishes after database deletion as cleaned", async () => { + const { cleanupExpiredTrafficRecordings } = + await import("@/lib/services/traffic-recording-service"); + + dbInsertMock.mockReturnValueOnce({ + values: vi.fn().mockReturnValue({ + onConflictDoNothing: vi.fn().mockResolvedValue(undefined), + }), + }); + findFirstMock.mockResolvedValueOnce(settingsRow()); + findManyMock.mockResolvedValueOnce([ + recordingRow({ createdAt: new Date("2025-12-01T00:00:00.000Z") }), + ]); + statMock.mockResolvedValueOnce({}); + dbDeleteMock.mockReturnValueOnce({ + where: vi.fn().mockReturnValue({ + returning: vi.fn().mockResolvedValue([{ id: "recording-1" }]), + }), + }); + unlinkMock.mockRejectedValueOnce( + Object.assign(new Error("fixture disappeared"), { code: "ENOENT" }) + ); + + const result = await cleanupExpiredTrafficRecordings(new Date("2026-01-01T00:00:00.000Z")); + + expect(result.deletedCount).toBe(1); + expect(result.failureCount).toBe(0); + expect(result.errorSummary).toBeNull(); + expect(dbInsertMock).toHaveBeenCalledTimes(1); + expect(unlinkMock).toHaveBeenCalledTimes(1); + }); it("uses runtime settings when deciding whether to record traffic", async () => { const { shouldRecordTraffic } = await import("@/lib/services/traffic-recording-service");