diff --git a/.github/workflows/api-benchmark-smoke.yml b/.github/workflows/api-benchmark-smoke.yml new file mode 100644 index 000000000..0376ca0dd --- /dev/null +++ b/.github/workflows/api-benchmark-smoke.yml @@ -0,0 +1,33 @@ +name: API Benchmark Smoke + +on: + pull_request: + paths: + - "apps/api/**" + - "benchmarks/**" + - "package.json" + - "package-lock.json" + - ".github/workflows/api-benchmark-smoke.yml" + +jobs: + benchmark-smoke: + runs-on: ubuntu-latest + + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Setup Node + uses: actions/setup-node@v4 + with: + node-version: 20 + cache: npm + + - name: Install dependencies + run: npm ci + + - name: Verify benchmark coverage + run: npm run benchmark:coverage + + - name: Run API benchmark smoke gate + run: npm run benchmark:smoke diff --git a/apps/api/package.json b/apps/api/package.json index 25fa0e90e..16556de8f 100644 --- a/apps/api/package.json +++ b/apps/api/package.json @@ -5,7 +5,7 @@ "scripts": { "dev": "node src/server.js", "start": "node src/server.js", - "test": "node --test src/tests" + "test": "node --test src/tests/*.test.js" }, "dependencies": { "cors": "^2.8.5", diff --git a/benchmarks/.env.benchmark.example b/benchmarks/.env.benchmark.example new file mode 100644 index 000000000..1a3ac3f15 --- /dev/null +++ b/benchmarks/.env.benchmark.example @@ -0,0 +1,10 @@ +# Optional. When omitted, the benchmark runner starts the local Express app on a random port. +BENCHMARK_TARGET_URL=http://127.0.0.1:4000 + +# Optional override for protected benchmark routes. The local runner generates an admin token automatically. +BENCHMARK_AUTH_TOKEN= + +# Defaults: smoke uses 1 concurrency and 2 requests per route; full uses 2 concurrency and 6 requests per route. +BENCHMARK_CONCURRENCY=2 +BENCHMARK_REQUESTS_PER_ROUTE=6 +BENCHMARK_REQUEST_TIMEOUT_MS=5000 diff --git a/benchmarks/README.md b/benchmarks/README.md new file mode 100644 index 000000000..8f81fd85d --- /dev/null +++ b/benchmarks/README.md @@ -0,0 +1,28 @@ +# API Benchmarks + +This directory contains a reproducible benchmark suite for the Express API. + +## Commands + +- `npm run benchmark:coverage` verifies that every mounted `/api/*` route, plus `/health`, is represented in the benchmark manifest. +- `npm run benchmark:smoke` runs a low-concurrency gate suitable for CI. +- `npm run benchmark` runs the fuller local suite and writes JSON plus Markdown results to `benchmarks/results/`. + +By default the runner starts the local Express app on a random loopback port and benchmarks that process. To benchmark another target, copy `.env.benchmark.example` to `.env.benchmark` and set `BENCHMARK_TARGET_URL`. + +Protected routes use a dedicated benchmark admin token when the local app is started by the runner. For external targets, set `BENCHMARK_AUTH_TOKEN` to a token created for benchmark-only use. + +## Metrics + +Each endpoint records: + +- p50, p95, and p99 total latency +- p50, p95, and p99 time to first byte +- sustained and peak requests per second +- status-code distribution +- error rate +- bytes received + +Thresholds for the CI smoke gate live in `benchmarks/thresholds.json`. + +The PR demo artifact is `demos/api-benchmark-demo.mp4`. diff --git a/benchmarks/api-route-manifest.mjs b/benchmarks/api-route-manifest.mjs new file mode 100644 index 000000000..eebdc8481 --- /dev/null +++ b/benchmarks/api-route-manifest.mjs @@ -0,0 +1,240 @@ +export const API_ROUTES = [ + { + id: "health", + method: "GET", + path: "/health", + pathTemplate: "/health", + description: "Service liveness check", + expectedStatus: 200 + }, + { + id: "auth-register", + method: "POST", + path: "/api/auth/register", + pathTemplate: "/api/auth/register", + description: "Register a benchmark client user", + expectedStatus: 201, + json: (index) => ({ + email: `bench-client-${Date.now()}-${index}@example.com`, + password: "benchmark-password", + role: "client" + }) + }, + { + id: "auth-login", + method: "POST", + path: "/api/auth/login", + pathTemplate: "/api/auth/login", + description: "Login with representative credentials", + expectedStatus: 200, + json: () => ({ + email: "benchmark@example.com", + password: "benchmark-password" + }) + }, + { + id: "auth-refresh", + method: "POST", + path: "/api/auth/refresh", + pathTemplate: "/api/auth/refresh", + description: "Refresh an access token", + expectedStatus: 200 + }, + { + id: "auth-oauth-callback", + method: "GET", + path: "/api/auth/oauth/github/callback", + pathTemplate: "/api/auth/oauth/:provider/callback", + description: "OAuth callback acknowledgement", + expectedStatus: 200 + }, + { + id: "users-list", + method: "GET", + path: "/api/users", + pathTemplate: "/api/users/", + description: "List users", + expectedStatus: 200 + }, + { + id: "users-create", + method: "POST", + path: "/api/users", + pathTemplate: "/api/users/", + description: "Create a representative user profile", + expectedStatus: 201, + json: (index) => ({ + name: `Benchmark User ${index}`, + email: `bench-user-${Date.now()}-${index}@example.com`, + role: index % 2 === 0 ? "client" : "freelancer", + status: "active", + bio: "Benchmark profile seeded with realistic text length for API regression tracking." + }) + }, + { + id: "jobs-list", + method: "GET", + path: "/api/jobs", + pathTemplate: "/api/jobs/", + description: "List jobs", + expectedStatus: 200 + }, + { + id: "jobs-create", + method: "POST", + path: "/api/jobs", + pathTemplate: "/api/jobs/", + description: "Create a representative marketplace job", + expectedStatus: 201, + json: (index) => ({ + title: `Benchmark API integration build ${index}`, + description: + "Build a production-ready integration with validation, webhook handling, retries, and clear operational docs.", + budgetMin: 750, + budgetMax: 2500, + categoryId: "cat_web_development", + skills: ["node", "express", "api", "testing"] + }) + }, + { + id: "proposals-list", + method: "GET", + path: "/api/proposals", + pathTemplate: "/api/proposals/", + description: "List proposals", + expectedStatus: 200 + }, + { + id: "proposals-create", + method: "POST", + path: "/api/proposals", + pathTemplate: "/api/proposals/", + description: "Submit a representative proposal", + expectedStatus: 201, + json: (index) => ({ + jobId: `job_benchmark_${index}`, + freelancerId: `usr_freelancer_${index}`, + amount: 1200, + timelineDays: 5, + coverLetter: + "I can deliver the integration with endpoint tests, retry handling, and a concise deployment checklist." + }) + }, + { + id: "payments-create", + method: "POST", + path: "/api/payments", + pathTemplate: "/api/payments/", + description: "Create a payment intent", + expectedStatus: 201, + json: (index) => ({ + jobId: `job_benchmark_${index}`, + amount: 1200, + currency: "usd", + payerId: `usr_client_${index}` + }) + }, + { + id: "reviews-list", + method: "GET", + path: "/api/reviews", + pathTemplate: "/api/reviews/", + description: "List reviews", + expectedStatus: 200 + }, + { + id: "reviews-create", + method: "POST", + path: "/api/reviews", + pathTemplate: "/api/reviews/", + description: "Create a representative review", + expectedStatus: 201, + json: (index) => ({ + jobId: `job_benchmark_${index}`, + reviewerId: `usr_client_${index}`, + revieweeId: `usr_freelancer_${index}`, + rating: 5, + comment: "Fast, clear delivery with strong tests and useful handoff notes." + }) + }, + { + id: "messages-list", + method: "GET", + path: "/api/messages", + pathTemplate: "/api/messages/", + description: "List messages", + expectedStatus: 200 + }, + { + id: "messages-create", + method: "POST", + path: "/api/messages", + pathTemplate: "/api/messages/", + description: "Send a representative project message", + expectedStatus: 201, + json: (index) => ({ + threadId: `thread_benchmark_${index}`, + senderId: `usr_client_${index}`, + recipientId: `usr_freelancer_${index}`, + body: "Can you confirm the API retry behavior and include the benchmark summary in the handoff?" + }) + }, + { + id: "notifications-list", + method: "GET", + path: "/api/notifications", + pathTemplate: "/api/notifications/", + description: "List notifications", + expectedStatus: 200 + }, + { + id: "notifications-create", + method: "POST", + path: "/api/notifications", + pathTemplate: "/api/notifications/", + description: "Create a representative notification", + expectedStatus: 201, + json: (index) => ({ + userId: `usr_client_${index}`, + type: "proposal_received", + title: "New proposal received", + body: "A freelancer submitted a proposal for your benchmark API integration job." + }) + }, + { + id: "uploads-create", + method: "POST", + path: "/api/uploads", + pathTemplate: "/api/uploads/", + description: "Upload a small representative attachment", + expectedStatus: 201, + multipart: { + fieldName: "file", + filename: "benchmark-brief.txt", + contentType: "text/plain", + content: + "Benchmark attachment body that simulates a short project brief uploaded by a client." + } + }, + { + id: "search-global", + method: "GET", + path: "/api/search?q=benchmark%20api%20developer", + pathTemplate: "/api/search/", + description: "Run global search with a realistic query", + expectedStatus: 200 + }, + { + id: "admin-metrics", + method: "GET", + path: "/api/admin/metrics", + pathTemplate: "/api/admin/metrics", + description: "Fetch protected admin metrics with benchmark token", + expectedStatus: 200, + auth: "admin" + } +]; + +export function routeLabels() { + return API_ROUTES.map((route) => `${route.method} ${route.pathTemplate}`); +} diff --git a/benchmarks/check-api-coverage.mjs b/benchmarks/check-api-coverage.mjs new file mode 100644 index 000000000..1ad8aa32b --- /dev/null +++ b/benchmarks/check-api-coverage.mjs @@ -0,0 +1,61 @@ +import fs from "node:fs"; +import path from "node:path"; +import { fileURLToPath } from "node:url"; +import { API_ROUTES, routeLabels } from "./api-route-manifest.mjs"; + +const rootDir = path.resolve(path.dirname(fileURLToPath(import.meta.url)), ".."); + +function read(filePath) { + return fs.readFileSync(path.join(rootDir, filePath), "utf8"); +} + +function normalizeMountedPath(basePath, routePath) { + const joined = `${basePath.replace(/\/$/, "")}/${routePath.replace(/^\//, "")}`; + return joined.replace(/\/$/, ""); +} + +function mountedRouters() { + const appSource = read("apps/api/src/app.js"); + const mounts = [...appSource.matchAll(/app\.use\("([^"]+)",\s*([a-zA-Z]+)Routes\)/g)]; + return mounts.map(([, basePath, routerName]) => ({ + basePath, + routerFile: `apps/api/src/routes/${routerName}Routes.js` + })); +} + +function routeDefinitions() { + const routes = new Set(["GET /health"]); + + for (const mount of mountedRouters()) { + const source = read(mount.routerFile); + const matches = [...source.matchAll(/(?:\w+Routes|Router\(\)|adminRoutes)\.(get|post|put|patch|delete)\("([^"]+)"/gi)]; + for (const [, method, routePath] of matches) { + routes.add(`${method.toUpperCase()} ${normalizeMountedPath(mount.basePath, routePath)}`); + } + } + + return routes; +} + +const implemented = new Set([...routeDefinitions()].map((route) => route.replace(/\/$/, ""))); +const benchmarked = new Set(routeLabels().map((label) => label.replace(/\/$/, ""))); +const missing = [...implemented].filter((route) => !benchmarked.has(route)); +const stale = [...benchmarked].filter((route) => !implemented.has(route)); + +if (missing.length || stale.length) { + console.error("Benchmark manifest is out of sync with the Express route surface."); + if (missing.length) { + console.error("\nMissing from benchmarks:"); + for (const route of missing) console.error(`- ${route}`); + } + if (stale.length) { + console.error("\nNo matching Express route:"); + for (const route of stale) console.error(`- ${route}`); + } + process.exit(1); +} + +console.log(`Benchmark manifest covers ${API_ROUTES.length} routes:`); +for (const route of routeLabels()) { + console.log(`- ${route}`); +} diff --git a/benchmarks/results/.gitkeep b/benchmarks/results/.gitkeep new file mode 100644 index 000000000..8b1378917 --- /dev/null +++ b/benchmarks/results/.gitkeep @@ -0,0 +1 @@ + diff --git a/benchmarks/results/api-benchmark-smoke.json b/benchmarks/results/api-benchmark-smoke.json new file mode 100644 index 000000000..3c8d95051 --- /dev/null +++ b/benchmarks/results/api-benchmark-smoke.json @@ -0,0 +1,571 @@ +{ + "mode": "smoke", + "target": "local-express", + "baseUrl": "http://127.0.0.1:58762", + "startedAt": "2026-05-21T03:25:23.773Z", + "finishedAt": "2026-05-21T03:25:23.877Z", + "config": { + "concurrency": 1, + "requestsPerRoute": 2 + }, + "environment": { + "platform": "darwin", + "release": "25.3.0", + "arch": "arm64", + "cpus": [ + "Apple M3", + "Apple M3", + "Apple M3", + "Apple M3", + "Apple M3", + "Apple M3", + "Apple M3", + "Apple M3" + ], + "cpuCount": 8, + "totalMemoryMb": 16384, + "freeMemoryMb": 202, + "nodeVersion": "v25.9.0" + }, + "routes": [ + { + "id": "health", + "method": "GET", + "path": "/health", + "pathTemplate": "/health", + "description": "Service liveness check", + "expectedStatus": 200, + "requestCount": 2, + "successCount": 2, + "errorCount": 0, + "errorRate": 0, + "statuses": { + "200": 2 + }, + "p50Ms": 3.66, + "p95Ms": 22.07, + "p99Ms": 22.07, + "ttfbP50Ms": 3.53, + "ttfbP95Ms": 21.31, + "ttfbP99Ms": 21.31, + "sustainedRps": 75.29, + "peakRps": 2, + "bytesReceived": 54, + "firstError": null + }, + { + "id": "auth-register", + "method": "POST", + "path": "/api/auth/register", + "pathTemplate": "/api/auth/register", + "description": "Register a benchmark client user", + "expectedStatus": 201, + "requestCount": 2, + "successCount": 2, + "errorCount": 0, + "errorRate": 0, + "statuses": { + "201": 2 + }, + "p50Ms": 2.03, + "p95Ms": 12.75, + "p99Ms": 12.75, + "ttfbP50Ms": 1.91, + "ttfbP95Ms": 12.64, + "ttfbP99Ms": 12.64, + "sustainedRps": 133.38, + "peakRps": 2, + "bytesReceived": 624, + "firstError": null + }, + { + "id": "auth-login", + "method": "POST", + "path": "/api/auth/login", + "pathTemplate": "/api/auth/login", + "description": "Login with representative credentials", + "expectedStatus": 200, + "requestCount": 2, + "successCount": 2, + "errorCount": 0, + "errorRate": 0, + "statuses": { + "200": 2 + }, + "p50Ms": 1.62, + "p95Ms": 1.97, + "p99Ms": 1.97, + "ttfbP50Ms": 1.34, + "ttfbP95Ms": 1.87, + "ttfbP99Ms": 1.87, + "sustainedRps": 512.28, + "peakRps": 2, + "bytesReceived": 490, + "firstError": null + }, + { + "id": "auth-refresh", + "method": "POST", + "path": "/api/auth/refresh", + "pathTemplate": "/api/auth/refresh", + "description": "Refresh an access token", + "expectedStatus": 200, + "requestCount": 2, + "successCount": 2, + "errorCount": 0, + "errorRate": 0, + "statuses": { + "200": 2 + }, + "p50Ms": 1.52, + "p95Ms": 1.6, + "p99Ms": 1.6, + "ttfbP50Ms": 1.33, + "ttfbP95Ms": 1.42, + "ttfbP99Ms": 1.42, + "sustainedRps": 575.33, + "peakRps": 2, + "bytesReceived": 426, + "firstError": null + }, + { + "id": "auth-oauth-callback", + "method": "GET", + "path": "/api/auth/oauth/github/callback", + "pathTemplate": "/api/auth/oauth/:provider/callback", + "description": "OAuth callback acknowledgement", + "expectedStatus": 200, + "requestCount": 2, + "successCount": 2, + "errorCount": 0, + "errorRate": 0, + "statuses": { + "200": 2 + }, + "p50Ms": 1.54, + "p95Ms": 1.85, + "p99Ms": 1.85, + "ttfbP50Ms": 1.44, + "ttfbP95Ms": 1.71, + "ttfbP99Ms": 1.71, + "sustainedRps": 546.84, + "peakRps": 2, + "bytesReceived": 148, + "firstError": null + }, + { + "id": "users-list", + "method": "GET", + "path": "/api/users", + "pathTemplate": "/api/users/", + "description": "List users", + "expectedStatus": 200, + "requestCount": 2, + "successCount": 2, + "errorCount": 0, + "errorRate": 0, + "statuses": { + "200": 2 + }, + "p50Ms": 1.1, + "p95Ms": 1.22, + "p99Ms": 1.22, + "ttfbP50Ms": 1.03, + "ttfbP95Ms": 1.14, + "ttfbP99Ms": 1.14, + "sustainedRps": 806.46, + "peakRps": 2, + "bytesReceived": 52, + "firstError": null + }, + { + "id": "users-create", + "method": "POST", + "path": "/api/users", + "pathTemplate": "/api/users/", + "description": "Create a representative user profile", + "expectedStatus": 201, + "requestCount": 2, + "successCount": 2, + "errorCount": 0, + "errorRate": 0, + "statuses": { + "201": 2 + }, + "p50Ms": 1.31, + "p95Ms": 1.7, + "p99Ms": 1.7, + "ttfbP50Ms": 1.24, + "ttfbP95Ms": 1.62, + "ttfbP99Ms": 1.62, + "sustainedRps": 626.48, + "peakRps": 2, + "bytesReceived": 500, + "firstError": null + }, + { + "id": "jobs-list", + "method": "GET", + "path": "/api/jobs", + "pathTemplate": "/api/jobs/", + "description": "List jobs", + "expectedStatus": 200, + "requestCount": 2, + "successCount": 2, + "errorCount": 0, + "errorRate": 0, + "statuses": { + "200": 2 + }, + "p50Ms": 1.35, + "p95Ms": 1.54, + "p99Ms": 1.54, + "ttfbP50Ms": 1.26, + "ttfbP95Ms": 1.45, + "ttfbP99Ms": 1.45, + "sustainedRps": 640.91, + "peakRps": 2, + "bytesReceived": 52, + "firstError": null + }, + { + "id": "jobs-create", + "method": "POST", + "path": "/api/jobs", + "pathTemplate": "/api/jobs/", + "description": "Create a representative marketplace job", + "expectedStatus": 201, + "requestCount": 2, + "successCount": 2, + "errorCount": 0, + "errorRate": 0, + "statuses": { + "201": 2 + }, + "p50Ms": 1.28, + "p95Ms": 2.35, + "p99Ms": 2.35, + "ttfbP50Ms": 1.2, + "ttfbP95Ms": 2.27, + "ttfbP99Ms": 2.27, + "sustainedRps": 528.39, + "peakRps": 2, + "bytesReceived": 694, + "firstError": null + }, + { + "id": "proposals-list", + "method": "GET", + "path": "/api/proposals", + "pathTemplate": "/api/proposals/", + "description": "List proposals", + "expectedStatus": 200, + "requestCount": 2, + "successCount": 2, + "errorCount": 0, + "errorRate": 0, + "statuses": { + "200": 2 + }, + "p50Ms": 1.35, + "p95Ms": 1.36, + "p99Ms": 1.36, + "ttfbP50Ms": 1.26, + "ttfbP95Ms": 1.28, + "ttfbP99Ms": 1.28, + "sustainedRps": 685.21, + "peakRps": 2, + "bytesReceived": 52, + "firstError": null + }, + { + "id": "proposals-create", + "method": "POST", + "path": "/api/proposals", + "pathTemplate": "/api/proposals/", + "description": "Submit a representative proposal", + "expectedStatus": 201, + "requestCount": 2, + "successCount": 2, + "errorCount": 0, + "errorRate": 0, + "statuses": { + "201": 2 + }, + "p50Ms": 0.71, + "p95Ms": 1.34, + "p99Ms": 1.34, + "ttfbP50Ms": 0.66, + "ttfbP95Ms": 1.26, + "ttfbP99Ms": 1.26, + "sustainedRps": 914.79, + "peakRps": 2, + "bytesReceived": 520, + "firstError": null + }, + { + "id": "payments-create", + "method": "POST", + "path": "/api/payments", + "pathTemplate": "/api/payments/", + "description": "Create a payment intent", + "expectedStatus": 201, + "requestCount": 2, + "successCount": 2, + "errorCount": 0, + "errorRate": 0, + "statuses": { + "201": 2 + }, + "p50Ms": 0.57, + "p95Ms": 0.72, + "p99Ms": 0.72, + "ttfbP50Ms": 0.52, + "ttfbP95Ms": 0.65, + "ttfbP99Ms": 0.65, + "sustainedRps": 1448.66, + "peakRps": 2, + "bytesReceived": 216, + "firstError": null + }, + { + "id": "reviews-list", + "method": "GET", + "path": "/api/reviews", + "pathTemplate": "/api/reviews/", + "description": "List reviews", + "expectedStatus": 200, + "requestCount": 2, + "successCount": 2, + "errorCount": 0, + "errorRate": 0, + "statuses": { + "200": 2 + }, + "p50Ms": 0.49, + "p95Ms": 0.5, + "p99Ms": 0.5, + "ttfbP50Ms": 0.42, + "ttfbP95Ms": 0.44, + "ttfbP99Ms": 0.44, + "sustainedRps": 1865.74, + "peakRps": 2, + "bytesReceived": 52, + "firstError": null + }, + { + "id": "reviews-create", + "method": "POST", + "path": "/api/reviews", + "pathTemplate": "/api/reviews/", + "description": "Create a representative review", + "expectedStatus": 201, + "requestCount": 2, + "successCount": 2, + "errorCount": 0, + "errorRate": 0, + "statuses": { + "201": 2 + }, + "p50Ms": 0.68, + "p95Ms": 1.32, + "p99Ms": 1.32, + "ttfbP50Ms": 0.64, + "ttfbP95Ms": 1.23, + "ttfbP99Ms": 1.23, + "sustainedRps": 935.11, + "peakRps": 2, + "bytesReceived": 448, + "firstError": null + }, + { + "id": "messages-list", + "method": "GET", + "path": "/api/messages", + "pathTemplate": "/api/messages/", + "description": "List messages", + "expectedStatus": 200, + "requestCount": 2, + "successCount": 2, + "errorCount": 0, + "errorRate": 0, + "statuses": { + "200": 2 + }, + "p50Ms": 1.08, + "p95Ms": 1.19, + "p99Ms": 1.19, + "ttfbP50Ms": 0.96, + "ttfbP95Ms": 1.1, + "ttfbP99Ms": 1.1, + "sustainedRps": 825.07, + "peakRps": 2, + "bytesReceived": 52, + "firstError": null + }, + { + "id": "messages-create", + "method": "POST", + "path": "/api/messages", + "pathTemplate": "/api/messages/", + "description": "Send a representative project message", + "expectedStatus": 201, + "requestCount": 2, + "successCount": 2, + "errorCount": 0, + "errorRate": 0, + "statuses": { + "201": 2 + }, + "p50Ms": 1.32, + "p95Ms": 1.58, + "p99Ms": 1.58, + "ttfbP50Ms": 1.27, + "ttfbP95Ms": 1.46, + "ttfbP99Ms": 1.46, + "sustainedRps": 656.01, + "peakRps": 2, + "bytesReceived": 550, + "firstError": null + }, + { + "id": "notifications-list", + "method": "GET", + "path": "/api/notifications", + "pathTemplate": "/api/notifications/", + "description": "List notifications", + "expectedStatus": 200, + "requestCount": 2, + "successCount": 2, + "errorCount": 0, + "errorRate": 0, + "statuses": { + "200": 2 + }, + "p50Ms": 0.46, + "p95Ms": 1.72, + "p99Ms": 1.72, + "ttfbP50Ms": 0.42, + "ttfbP95Ms": 1.67, + "ttfbP99Ms": 1.67, + "sustainedRps": 888.15, + "peakRps": 2, + "bytesReceived": 52, + "firstError": null + }, + { + "id": "notifications-create", + "method": "POST", + "path": "/api/notifications", + "pathTemplate": "/api/notifications/", + "description": "Create a representative notification", + "expectedStatus": 201, + "requestCount": 2, + "successCount": 2, + "errorCount": 0, + "errorRate": 0, + "statuses": { + "201": 2 + }, + "p50Ms": 0.37, + "p95Ms": 0.48, + "p99Ms": 0.48, + "ttfbP50Ms": 0.33, + "ttfbP95Ms": 0.44, + "ttfbP99Ms": 0.44, + "sustainedRps": 2000, + "peakRps": 2, + "bytesReceived": 458, + "firstError": null + }, + { + "id": "uploads-create", + "method": "POST", + "path": "/api/uploads", + "pathTemplate": "/api/uploads/", + "description": "Upload a small representative attachment", + "expectedStatus": 201, + "requestCount": 2, + "successCount": 2, + "errorCount": 0, + "errorRate": 0, + "statuses": { + "201": 2 + }, + "p50Ms": 1.93, + "p95Ms": 7.5, + "p99Ms": 7.5, + "ttfbP50Ms": 1.82, + "ttfbP95Ms": 7.41, + "ttfbP99Ms": 7.41, + "sustainedRps": 208.61, + "peakRps": 2, + "bytesReceived": 156, + "firstError": null + }, + { + "id": "search-global", + "method": "GET", + "path": "/api/search?q=benchmark%20api%20developer", + "pathTemplate": "/api/search/", + "description": "Run global search with a realistic query", + "expectedStatus": 200, + "requestCount": 2, + "successCount": 2, + "errorCount": 0, + "errorRate": 0, + "statuses": { + "200": 2 + }, + "p50Ms": 1.22, + "p95Ms": 1.77, + "p99Ms": 1.77, + "ttfbP50Ms": 1.17, + "ttfbP95Ms": 1.72, + "ttfbP99Ms": 1.72, + "sustainedRps": 616.93, + "peakRps": 2, + "bytesReceived": 194, + "firstError": null + }, + { + "id": "admin-metrics", + "method": "GET", + "path": "/api/admin/metrics", + "pathTemplate": "/api/admin/metrics", + "description": "Fetch protected admin metrics with benchmark token", + "expectedStatus": 200, + "requestCount": 2, + "successCount": 2, + "errorCount": 0, + "errorRate": 0, + "statuses": { + "200": 2 + }, + "p50Ms": 2.09, + "p95Ms": 3.26, + "p99Ms": 3.26, + "ttfbP50Ms": 2.02, + "ttfbP95Ms": 3.12, + "ttfbP99Ms": 3.12, + "sustainedRps": 362.4, + "peakRps": 2, + "bytesReceived": 212, + "firstError": null + } + ], + "totals": { + "routeCount": 21, + "requestCount": 42, + "errorCount": 0, + "errorRate": 0, + "maxP99Ms": 22.07, + "maxTtfbP99Ms": 21.31 + }, + "thresholds": { + "p99MaxMs": 1000, + "ttfbP99MaxMs": 1000, + "errorRateMax": 0 + }, + "thresholdFailures": [] +} diff --git a/benchmarks/results/api-benchmark-smoke.md b/benchmarks/results/api-benchmark-smoke.md new file mode 100644 index 000000000..c43e1699a --- /dev/null +++ b/benchmarks/results/api-benchmark-smoke.md @@ -0,0 +1,34 @@ +# API Benchmark Summary + +- Mode: smoke +- Target: local-express +- Started: 2026-05-21T03:25:23.773Z +- Routes covered: 21 +- Total requests: 42 +- Error rate: 0 +- Max p99 latency: 22.07 ms +- Max p99 TTFB: 21.31 ms + +| Endpoint | Requests | Error rate | p50 ms | p95 ms | p99 ms | p99 TTFB ms | Sustained RPS | +| --- | ---: | ---: | ---: | ---: | ---: | ---: | ---: | +| GET /health | 2 | 0 | 3.66 | 22.07 | 22.07 | 21.31 | 75.29 | +| POST /api/auth/register | 2 | 0 | 2.03 | 12.75 | 12.75 | 12.64 | 133.38 | +| POST /api/auth/login | 2 | 0 | 1.62 | 1.97 | 1.97 | 1.87 | 512.28 | +| POST /api/auth/refresh | 2 | 0 | 1.52 | 1.6 | 1.6 | 1.42 | 575.33 | +| GET /api/auth/oauth/github/callback | 2 | 0 | 1.54 | 1.85 | 1.85 | 1.71 | 546.84 | +| GET /api/users | 2 | 0 | 1.1 | 1.22 | 1.22 | 1.14 | 806.46 | +| POST /api/users | 2 | 0 | 1.31 | 1.7 | 1.7 | 1.62 | 626.48 | +| GET /api/jobs | 2 | 0 | 1.35 | 1.54 | 1.54 | 1.45 | 640.91 | +| POST /api/jobs | 2 | 0 | 1.28 | 2.35 | 2.35 | 2.27 | 528.39 | +| GET /api/proposals | 2 | 0 | 1.35 | 1.36 | 1.36 | 1.28 | 685.21 | +| POST /api/proposals | 2 | 0 | 0.71 | 1.34 | 1.34 | 1.26 | 914.79 | +| POST /api/payments | 2 | 0 | 0.57 | 0.72 | 0.72 | 0.65 | 1448.66 | +| GET /api/reviews | 2 | 0 | 0.49 | 0.5 | 0.5 | 0.44 | 1865.74 | +| POST /api/reviews | 2 | 0 | 0.68 | 1.32 | 1.32 | 1.23 | 935.11 | +| GET /api/messages | 2 | 0 | 1.08 | 1.19 | 1.19 | 1.1 | 825.07 | +| POST /api/messages | 2 | 0 | 1.32 | 1.58 | 1.58 | 1.46 | 656.01 | +| GET /api/notifications | 2 | 0 | 0.46 | 1.72 | 1.72 | 1.67 | 888.15 | +| POST /api/notifications | 2 | 0 | 0.37 | 0.48 | 0.48 | 0.44 | 2000 | +| POST /api/uploads | 2 | 0 | 1.93 | 7.5 | 7.5 | 7.41 | 208.61 | +| GET /api/search?q=benchmark%20api%20developer | 2 | 0 | 1.22 | 1.77 | 1.77 | 1.72 | 616.93 | +| GET /api/admin/metrics | 2 | 0 | 2.09 | 3.26 | 3.26 | 3.12 | 362.4 | diff --git a/benchmarks/results/api-benchmark.json b/benchmarks/results/api-benchmark.json new file mode 100644 index 000000000..f6d4454df --- /dev/null +++ b/benchmarks/results/api-benchmark.json @@ -0,0 +1,571 @@ +{ + "mode": "full", + "target": "local-express", + "baseUrl": "http://127.0.0.1:58782", + "startedAt": "2026-05-21T03:25:49.696Z", + "finishedAt": "2026-05-21T03:25:49.881Z", + "config": { + "concurrency": 2, + "requestsPerRoute": 6 + }, + "environment": { + "platform": "darwin", + "release": "25.3.0", + "arch": "arm64", + "cpus": [ + "Apple M3", + "Apple M3", + "Apple M3", + "Apple M3", + "Apple M3", + "Apple M3", + "Apple M3", + "Apple M3" + ], + "cpuCount": 8, + "totalMemoryMb": 16384, + "freeMemoryMb": 222, + "nodeVersion": "v25.9.0" + }, + "routes": [ + { + "id": "health", + "method": "GET", + "path": "/health", + "pathTemplate": "/health", + "description": "Service liveness check", + "expectedStatus": 200, + "requestCount": 6, + "successCount": 6, + "errorCount": 0, + "errorRate": 0, + "statuses": { + "200": 6 + }, + "p50Ms": 3.71, + "p95Ms": 31.96, + "p99Ms": 31.96, + "ttfbP50Ms": 3.56, + "ttfbP95Ms": 31.35, + "ttfbP99Ms": 31.35, + "sustainedRps": 144.37, + "peakRps": 6, + "bytesReceived": 162, + "firstError": null + }, + { + "id": "auth-register", + "method": "POST", + "path": "/api/auth/register", + "pathTemplate": "/api/auth/register", + "description": "Register a benchmark client user", + "expectedStatus": 201, + "requestCount": 6, + "successCount": 6, + "errorCount": 0, + "errorRate": 0, + "statuses": { + "201": 6 + }, + "p50Ms": 5.16, + "p95Ms": 26.83, + "p99Ms": 26.83, + "ttfbP50Ms": 5.07, + "ttfbP95Ms": 26.62, + "ttfbP99Ms": 26.62, + "sustainedRps": 161.1, + "peakRps": 6, + "bytesReceived": 1872, + "firstError": null + }, + { + "id": "auth-login", + "method": "POST", + "path": "/api/auth/login", + "pathTemplate": "/api/auth/login", + "description": "Login with representative credentials", + "expectedStatus": 200, + "requestCount": 6, + "successCount": 6, + "errorCount": 0, + "errorRate": 0, + "statuses": { + "200": 6 + }, + "p50Ms": 2.37, + "p95Ms": 3.36, + "p99Ms": 3.36, + "ttfbP50Ms": 2.32, + "ttfbP95Ms": 3.03, + "ttfbP99Ms": 3.03, + "sustainedRps": 652.96, + "peakRps": 6, + "bytesReceived": 1470, + "firstError": null + }, + { + "id": "auth-refresh", + "method": "POST", + "path": "/api/auth/refresh", + "pathTemplate": "/api/auth/refresh", + "description": "Refresh an access token", + "expectedStatus": 200, + "requestCount": 6, + "successCount": 6, + "errorCount": 0, + "errorRate": 0, + "statuses": { + "200": 6 + }, + "p50Ms": 2.09, + "p95Ms": 2.48, + "p99Ms": 2.48, + "ttfbP50Ms": 2.03, + "ttfbP95Ms": 2.43, + "ttfbP99Ms": 2.43, + "sustainedRps": 917.88, + "peakRps": 6, + "bytesReceived": 1278, + "firstError": null + }, + { + "id": "auth-oauth-callback", + "method": "GET", + "path": "/api/auth/oauth/github/callback", + "pathTemplate": "/api/auth/oauth/:provider/callback", + "description": "OAuth callback acknowledgement", + "expectedStatus": 200, + "requestCount": 6, + "successCount": 6, + "errorCount": 0, + "errorRate": 0, + "statuses": { + "200": 6 + }, + "p50Ms": 1.33, + "p95Ms": 1.49, + "p99Ms": 1.49, + "ttfbP50Ms": 1.27, + "ttfbP95Ms": 1.44, + "ttfbP99Ms": 1.44, + "sustainedRps": 1466.53, + "peakRps": 6, + "bytesReceived": 444, + "firstError": null + }, + { + "id": "users-list", + "method": "GET", + "path": "/api/users", + "pathTemplate": "/api/users/", + "description": "List users", + "expectedStatus": 200, + "requestCount": 6, + "successCount": 6, + "errorCount": 0, + "errorRate": 0, + "statuses": { + "200": 6 + }, + "p50Ms": 0.68, + "p95Ms": 1.16, + "p99Ms": 1.16, + "ttfbP50Ms": 0.63, + "ttfbP95Ms": 1.13, + "ttfbP99Ms": 1.13, + "sustainedRps": 2122.95, + "peakRps": 6, + "bytesReceived": 156, + "firstError": null + }, + { + "id": "users-create", + "method": "POST", + "path": "/api/users", + "pathTemplate": "/api/users/", + "description": "Create a representative user profile", + "expectedStatus": 201, + "requestCount": 6, + "successCount": 6, + "errorCount": 0, + "errorRate": 0, + "statuses": { + "201": 6 + }, + "p50Ms": 1.29, + "p95Ms": 1.49, + "p99Ms": 1.49, + "ttfbP50Ms": 1.24, + "ttfbP95Ms": 1.42, + "ttfbP99Ms": 1.42, + "sustainedRps": 1473.75, + "peakRps": 6, + "bytesReceived": 1500, + "firstError": null + }, + { + "id": "jobs-list", + "method": "GET", + "path": "/api/jobs", + "pathTemplate": "/api/jobs/", + "description": "List jobs", + "expectedStatus": 200, + "requestCount": 6, + "successCount": 6, + "errorCount": 0, + "errorRate": 0, + "statuses": { + "200": 6 + }, + "p50Ms": 0.72, + "p95Ms": 1.6, + "p99Ms": 1.6, + "ttfbP50Ms": 0.69, + "ttfbP95Ms": 1.57, + "ttfbP99Ms": 1.57, + "sustainedRps": 1817.81, + "peakRps": 6, + "bytesReceived": 156, + "firstError": null + }, + { + "id": "jobs-create", + "method": "POST", + "path": "/api/jobs", + "pathTemplate": "/api/jobs/", + "description": "Create a representative marketplace job", + "expectedStatus": 201, + "requestCount": 6, + "successCount": 6, + "errorCount": 0, + "errorRate": 0, + "statuses": { + "201": 6 + }, + "p50Ms": 1.91, + "p95Ms": 2.25, + "p99Ms": 2.25, + "ttfbP50Ms": 1.88, + "ttfbP95Ms": 2.19, + "ttfbP99Ms": 2.19, + "sustainedRps": 915.41, + "peakRps": 6, + "bytesReceived": 2082, + "firstError": null + }, + { + "id": "proposals-list", + "method": "GET", + "path": "/api/proposals", + "pathTemplate": "/api/proposals/", + "description": "List proposals", + "expectedStatus": 200, + "requestCount": 6, + "successCount": 6, + "errorCount": 0, + "errorRate": 0, + "statuses": { + "200": 6 + }, + "p50Ms": 0.92, + "p95Ms": 1.22, + "p99Ms": 1.22, + "ttfbP50Ms": 0.88, + "ttfbP95Ms": 1.19, + "ttfbP99Ms": 1.19, + "sustainedRps": 1870.69, + "peakRps": 6, + "bytesReceived": 156, + "firstError": null + }, + { + "id": "proposals-create", + "method": "POST", + "path": "/api/proposals", + "pathTemplate": "/api/proposals/", + "description": "Submit a representative proposal", + "expectedStatus": 201, + "requestCount": 6, + "successCount": 6, + "errorCount": 0, + "errorRate": 0, + "statuses": { + "201": 6 + }, + "p50Ms": 1.15, + "p95Ms": 1.49, + "p99Ms": 1.49, + "ttfbP50Ms": 1.11, + "ttfbP95Ms": 1.37, + "ttfbP99Ms": 1.37, + "sustainedRps": 1402.57, + "peakRps": 6, + "bytesReceived": 1560, + "firstError": null + }, + { + "id": "payments-create", + "method": "POST", + "path": "/api/payments", + "pathTemplate": "/api/payments/", + "description": "Create a payment intent", + "expectedStatus": 201, + "requestCount": 6, + "successCount": 6, + "errorCount": 0, + "errorRate": 0, + "statuses": { + "201": 6 + }, + "p50Ms": 0.97, + "p95Ms": 1.76, + "p99Ms": 1.76, + "ttfbP50Ms": 0.94, + "ttfbP95Ms": 1.68, + "ttfbP99Ms": 1.68, + "sustainedRps": 1568.78, + "peakRps": 6, + "bytesReceived": 648, + "firstError": null + }, + { + "id": "reviews-list", + "method": "GET", + "path": "/api/reviews", + "pathTemplate": "/api/reviews/", + "description": "List reviews", + "expectedStatus": 200, + "requestCount": 6, + "successCount": 6, + "errorCount": 0, + "errorRate": 0, + "statuses": { + "200": 6 + }, + "p50Ms": 1.2, + "p95Ms": 1.58, + "p99Ms": 1.58, + "ttfbP50Ms": 1.16, + "ttfbP95Ms": 1.54, + "ttfbP99Ms": 1.54, + "sustainedRps": 1551.72, + "peakRps": 6, + "bytesReceived": 156, + "firstError": null + }, + { + "id": "reviews-create", + "method": "POST", + "path": "/api/reviews", + "pathTemplate": "/api/reviews/", + "description": "Create a representative review", + "expectedStatus": 201, + "requestCount": 6, + "successCount": 6, + "errorCount": 0, + "errorRate": 0, + "statuses": { + "201": 6 + }, + "p50Ms": 0.66, + "p95Ms": 0.91, + "p99Ms": 0.91, + "ttfbP50Ms": 0.63, + "ttfbP95Ms": 0.87, + "ttfbP99Ms": 0.87, + "sustainedRps": 2366.16, + "peakRps": 6, + "bytesReceived": 1344, + "firstError": null + }, + { + "id": "messages-list", + "method": "GET", + "path": "/api/messages", + "pathTemplate": "/api/messages/", + "description": "List messages", + "expectedStatus": 200, + "requestCount": 6, + "successCount": 6, + "errorCount": 0, + "errorRate": 0, + "statuses": { + "200": 6 + }, + "p50Ms": 1.08, + "p95Ms": 1.48, + "p99Ms": 1.48, + "ttfbP50Ms": 1.04, + "ttfbP95Ms": 1.43, + "ttfbP99Ms": 1.43, + "sustainedRps": 1524.12, + "peakRps": 6, + "bytesReceived": 156, + "firstError": null + }, + { + "id": "messages-create", + "method": "POST", + "path": "/api/messages", + "pathTemplate": "/api/messages/", + "description": "Send a representative project message", + "expectedStatus": 201, + "requestCount": 6, + "successCount": 6, + "errorCount": 0, + "errorRate": 0, + "statuses": { + "201": 6 + }, + "p50Ms": 0.7, + "p95Ms": 1.26, + "p99Ms": 1.26, + "ttfbP50Ms": 0.66, + "ttfbP95Ms": 1.21, + "ttfbP99Ms": 1.21, + "sustainedRps": 2050.14, + "peakRps": 6, + "bytesReceived": 1650, + "firstError": null + }, + { + "id": "notifications-list", + "method": "GET", + "path": "/api/notifications", + "pathTemplate": "/api/notifications/", + "description": "List notifications", + "expectedStatus": 200, + "requestCount": 6, + "successCount": 6, + "errorCount": 0, + "errorRate": 0, + "statuses": { + "200": 6 + }, + "p50Ms": 0.56, + "p95Ms": 0.81, + "p99Ms": 0.81, + "ttfbP50Ms": 0.53, + "ttfbP95Ms": 0.79, + "ttfbP99Ms": 0.79, + "sustainedRps": 2727.07, + "peakRps": 6, + "bytesReceived": 156, + "firstError": null + }, + { + "id": "notifications-create", + "method": "POST", + "path": "/api/notifications", + "pathTemplate": "/api/notifications/", + "description": "Create a representative notification", + "expectedStatus": 201, + "requestCount": 6, + "successCount": 6, + "errorCount": 0, + "errorRate": 0, + "statuses": { + "201": 6 + }, + "p50Ms": 1.95, + "p95Ms": 2.18, + "p99Ms": 2.18, + "ttfbP50Ms": 1.75, + "ttfbP95Ms": 2.13, + "ttfbP99Ms": 2.13, + "sustainedRps": 1090.26, + "peakRps": 6, + "bytesReceived": 1374, + "firstError": null + }, + { + "id": "uploads-create", + "method": "POST", + "path": "/api/uploads", + "pathTemplate": "/api/uploads/", + "description": "Upload a small representative attachment", + "expectedStatus": 201, + "requestCount": 6, + "successCount": 6, + "errorCount": 0, + "errorRate": 0, + "statuses": { + "201": 6 + }, + "p50Ms": 1.68, + "p95Ms": 10.72, + "p99Ms": 10.72, + "ttfbP50Ms": 1.64, + "ttfbP95Ms": 10.64, + "ttfbP99Ms": 10.64, + "sustainedRps": 412.71, + "peakRps": 6, + "bytesReceived": 468, + "firstError": null + }, + { + "id": "search-global", + "method": "GET", + "path": "/api/search?q=benchmark%20api%20developer", + "pathTemplate": "/api/search/", + "description": "Run global search with a realistic query", + "expectedStatus": 200, + "requestCount": 6, + "successCount": 6, + "errorCount": 0, + "errorRate": 0, + "statuses": { + "200": 6 + }, + "p50Ms": 0.82, + "p95Ms": 2.04, + "p99Ms": 2.04, + "ttfbP50Ms": 0.77, + "ttfbP95Ms": 1.99, + "ttfbP99Ms": 1.99, + "sustainedRps": 1487, + "peakRps": 6, + "bytesReceived": 582, + "firstError": null + }, + { + "id": "admin-metrics", + "method": "GET", + "path": "/api/admin/metrics", + "pathTemplate": "/api/admin/metrics", + "description": "Fetch protected admin metrics with benchmark token", + "expectedStatus": 200, + "requestCount": 6, + "successCount": 6, + "errorCount": 0, + "errorRate": 0, + "statuses": { + "200": 6 + }, + "p50Ms": 1.46, + "p95Ms": 4.9, + "p99Ms": 4.9, + "ttfbP50Ms": 1.35, + "ttfbP95Ms": 4.76, + "ttfbP99Ms": 4.76, + "sustainedRps": 765.66, + "peakRps": 6, + "bytesReceived": 636, + "firstError": null + } + ], + "totals": { + "routeCount": 21, + "requestCount": 126, + "errorCount": 0, + "errorRate": 0, + "maxP99Ms": 31.96, + "maxTtfbP99Ms": 31.35 + }, + "thresholds": { + "p99MaxMs": 1000, + "ttfbP99MaxMs": 1000, + "errorRateMax": 0 + }, + "thresholdFailures": [] +} diff --git a/benchmarks/results/api-benchmark.md b/benchmarks/results/api-benchmark.md new file mode 100644 index 000000000..56990c92e --- /dev/null +++ b/benchmarks/results/api-benchmark.md @@ -0,0 +1,34 @@ +# API Benchmark Summary + +- Mode: full +- Target: local-express +- Started: 2026-05-21T03:25:49.696Z +- Routes covered: 21 +- Total requests: 126 +- Error rate: 0 +- Max p99 latency: 31.96 ms +- Max p99 TTFB: 31.35 ms + +| Endpoint | Requests | Error rate | p50 ms | p95 ms | p99 ms | p99 TTFB ms | Sustained RPS | +| --- | ---: | ---: | ---: | ---: | ---: | ---: | ---: | +| GET /health | 6 | 0 | 3.71 | 31.96 | 31.96 | 31.35 | 144.37 | +| POST /api/auth/register | 6 | 0 | 5.16 | 26.83 | 26.83 | 26.62 | 161.1 | +| POST /api/auth/login | 6 | 0 | 2.37 | 3.36 | 3.36 | 3.03 | 652.96 | +| POST /api/auth/refresh | 6 | 0 | 2.09 | 2.48 | 2.48 | 2.43 | 917.88 | +| GET /api/auth/oauth/github/callback | 6 | 0 | 1.33 | 1.49 | 1.49 | 1.44 | 1466.53 | +| GET /api/users | 6 | 0 | 0.68 | 1.16 | 1.16 | 1.13 | 2122.95 | +| POST /api/users | 6 | 0 | 1.29 | 1.49 | 1.49 | 1.42 | 1473.75 | +| GET /api/jobs | 6 | 0 | 0.72 | 1.6 | 1.6 | 1.57 | 1817.81 | +| POST /api/jobs | 6 | 0 | 1.91 | 2.25 | 2.25 | 2.19 | 915.41 | +| GET /api/proposals | 6 | 0 | 0.92 | 1.22 | 1.22 | 1.19 | 1870.69 | +| POST /api/proposals | 6 | 0 | 1.15 | 1.49 | 1.49 | 1.37 | 1402.57 | +| POST /api/payments | 6 | 0 | 0.97 | 1.76 | 1.76 | 1.68 | 1568.78 | +| GET /api/reviews | 6 | 0 | 1.2 | 1.58 | 1.58 | 1.54 | 1551.72 | +| POST /api/reviews | 6 | 0 | 0.66 | 0.91 | 0.91 | 0.87 | 2366.16 | +| GET /api/messages | 6 | 0 | 1.08 | 1.48 | 1.48 | 1.43 | 1524.12 | +| POST /api/messages | 6 | 0 | 0.7 | 1.26 | 1.26 | 1.21 | 2050.14 | +| GET /api/notifications | 6 | 0 | 0.56 | 0.81 | 0.81 | 0.79 | 2727.07 | +| POST /api/notifications | 6 | 0 | 1.95 | 2.18 | 2.18 | 2.13 | 1090.26 | +| POST /api/uploads | 6 | 0 | 1.68 | 10.72 | 10.72 | 10.64 | 412.71 | +| GET /api/search?q=benchmark%20api%20developer | 6 | 0 | 0.82 | 2.04 | 2.04 | 1.99 | 1487 | +| GET /api/admin/metrics | 6 | 0 | 1.46 | 4.9 | 4.9 | 4.76 | 765.66 | diff --git a/benchmarks/run-api-benchmark.mjs b/benchmarks/run-api-benchmark.mjs new file mode 100644 index 000000000..14d8bdae9 --- /dev/null +++ b/benchmarks/run-api-benchmark.mjs @@ -0,0 +1,347 @@ +import fs from "node:fs/promises"; +import http from "node:http"; +import https from "node:https"; +import os from "node:os"; +import path from "node:path"; +import { performance } from "node:perf_hooks"; +import { fileURLToPath } from "node:url"; +import { createApp } from "../apps/api/src/app.js"; +import { signAccessToken } from "../apps/api/src/utils/jwt.js"; +import { API_ROUTES } from "./api-route-manifest.mjs"; + +const rootDir = path.resolve(path.dirname(fileURLToPath(import.meta.url)), ".."); +const resultsDir = path.join(rootDir, "benchmarks", "results"); +const thresholdsPath = path.join(rootDir, "benchmarks", "thresholds.json"); + +const args = new Set(process.argv.slice(2)); +const smoke = args.has("--smoke"); +const outPrefix = smoke ? "api-benchmark-smoke" : "api-benchmark"; + +function applyDotenv(text) { + for (const line of text.split(/\r?\n/)) { + const trimmed = line.trim(); + if (!trimmed || trimmed.startsWith("#") || !trimmed.includes("=")) continue; + const [key, ...valueParts] = trimmed.split("="); + if (!process.env[key]) { + process.env[key] = valueParts.join("=").replace(/^"|"$/g, ""); + } + } +} + +async function loadBenchmarkEnv() { + for (const fileName of [".env.benchmark", ".env.benchmark.local"]) { + const filePath = path.join(rootDir, "benchmarks", fileName); + try { + applyDotenv(await fs.readFile(filePath, "utf8")); + } catch (error) { + if (error.code !== "ENOENT") throw error; + } + } +} + +function percentile(values, p) { + if (!values.length) return 0; + const sorted = [...values].sort((a, b) => a - b); + const index = Math.ceil((p / 100) * sorted.length) - 1; + return sorted[Math.max(0, Math.min(index, sorted.length - 1))]; +} + +function round(value, decimals = 2) { + return Number(value.toFixed(decimals)); +} + +function multipartBody(route) { + const boundary = `----freelanceflow-benchmark-${Math.random().toString(16).slice(2)}`; + const part = route.multipart; + const body = [ + `--${boundary}`, + `Content-Disposition: form-data; name="${part.fieldName}"; filename="${part.filename}"`, + `Content-Type: ${part.contentType}`, + "", + part.content, + `--${boundary}--`, + "" + ].join("\r\n"); + + return { + body, + headers: { + "content-type": `multipart/form-data; boundary=${boundary}`, + "content-length": Buffer.byteLength(body) + } + }; +} + +function requestPayload(route, index, adminToken) { + const headers = {}; + let body; + + if (route.auth === "admin") { + headers.authorization = `Bearer ${process.env.BENCHMARK_AUTH_TOKEN || adminToken}`; + } + + if (route.multipart) { + const multipart = multipartBody(route); + Object.assign(headers, multipart.headers); + body = multipart.body; + } else if (route.json) { + body = JSON.stringify(route.json(index)); + headers["content-type"] = "application/json"; + headers["content-length"] = Buffer.byteLength(body); + } + + return { body, headers }; +} + +function requestOnce(baseUrl, route, index, adminToken) { + const startedAt = performance.now(); + const url = new URL(route.path, baseUrl); + const protocol = url.protocol === "https:" ? https : http; + const payload = requestPayload(route, index, adminToken); + + return new Promise((resolve) => { + const req = protocol.request( + url, + { + method: route.method, + headers: payload.headers, + timeout: Number(process.env.BENCHMARK_REQUEST_TIMEOUT_MS || 5000) + }, + (res) => { + const firstByteAt = performance.now(); + let bytes = 0; + + res.on("data", (chunk) => { + bytes += chunk.length; + }); + res.on("end", () => { + const endedAt = performance.now(); + resolve({ + status: res.statusCode, + ok: res.statusCode === route.expectedStatus, + latencyMs: endedAt - startedAt, + ttfbMs: firstByteAt - startedAt, + bytes, + error: null + }); + }); + } + ); + + req.on("timeout", () => { + req.destroy(new Error("request timeout")); + }); + req.on("error", (error) => { + const endedAt = performance.now(); + resolve({ + status: 0, + ok: false, + latencyMs: endedAt - startedAt, + ttfbMs: endedAt - startedAt, + bytes: 0, + error: error.message + }); + }); + + if (payload.body) req.write(payload.body); + req.end(); + }); +} + +async function benchmarkRoute(baseUrl, route, adminToken, config) { + const samples = []; + const startedAt = performance.now(); + const requestTimeline = []; + + for (let i = 0; i < config.requestsPerRoute; i += config.concurrency) { + const batchSize = Math.min(config.concurrency, config.requestsPerRoute - i); + const batchStartedAt = performance.now(); + const batch = await Promise.all( + Array.from({ length: batchSize }, (_, offset) => + requestOnce(baseUrl, route, i + offset, adminToken) + ) + ); + for (const sample of batch) { + samples.push(sample); + requestTimeline.push(batchStartedAt); + } + } + + const endedAt = performance.now(); + const latencies = samples.map((sample) => sample.latencyMs); + const ttfb = samples.map((sample) => sample.ttfbMs); + const errors = samples.filter((sample) => !sample.ok); + const durationSeconds = Math.max((endedAt - startedAt) / 1000, 0.001); + const buckets = new Map(); + + for (const timestamp of requestTimeline) { + const second = Math.floor((timestamp - startedAt) / 1000); + buckets.set(second, (buckets.get(second) || 0) + 1); + } + + return { + id: route.id, + method: route.method, + path: route.path, + pathTemplate: route.pathTemplate, + description: route.description, + expectedStatus: route.expectedStatus, + requestCount: samples.length, + successCount: samples.length - errors.length, + errorCount: errors.length, + errorRate: round(errors.length / Math.max(samples.length, 1), 4), + statuses: samples.reduce((acc, sample) => { + acc[sample.status] = (acc[sample.status] || 0) + 1; + return acc; + }, {}), + p50Ms: round(percentile(latencies, 50)), + p95Ms: round(percentile(latencies, 95)), + p99Ms: round(percentile(latencies, 99)), + ttfbP50Ms: round(percentile(ttfb, 50)), + ttfbP95Ms: round(percentile(ttfb, 95)), + ttfbP99Ms: round(percentile(ttfb, 99)), + sustainedRps: round(samples.length / durationSeconds), + peakRps: Math.max(...buckets.values(), samples.length), + bytesReceived: samples.reduce((sum, sample) => sum + sample.bytes, 0), + firstError: errors[0]?.error ?? null + }; +} + +function summaryMarkdown(result) { + const rows = result.routes + .map( + (route) => + `| ${route.method} ${route.path} | ${route.requestCount} | ${route.errorRate} | ${route.p50Ms} | ${route.p95Ms} | ${route.p99Ms} | ${route.ttfbP99Ms} | ${route.sustainedRps} |` + ) + .join("\n"); + + return `# API Benchmark Summary + +- Mode: ${result.mode} +- Target: ${result.target} +- Started: ${result.startedAt} +- Routes covered: ${result.routes.length} +- Total requests: ${result.totals.requestCount} +- Error rate: ${result.totals.errorRate} +- Max p99 latency: ${result.totals.maxP99Ms} ms +- Max p99 TTFB: ${result.totals.maxTtfbP99Ms} ms + +| Endpoint | Requests | Error rate | p50 ms | p95 ms | p99 ms | p99 TTFB ms | Sustained RPS | +| --- | ---: | ---: | ---: | ---: | ---: | ---: | ---: | +${rows} +`; +} + +function enforceThresholds(result, thresholds) { + const failures = []; + for (const route of result.routes) { + if (route.p99Ms > thresholds.p99MaxMs) { + failures.push(`${route.method} ${route.path} p99 ${route.p99Ms}ms > ${thresholds.p99MaxMs}ms`); + } + if (route.ttfbP99Ms > thresholds.ttfbP99MaxMs) { + failures.push( + `${route.method} ${route.path} p99 TTFB ${route.ttfbP99Ms}ms > ${thresholds.ttfbP99MaxMs}ms` + ); + } + if (route.errorRate > thresholds.errorRateMax) { + failures.push( + `${route.method} ${route.path} error rate ${route.errorRate} > ${thresholds.errorRateMax}` + ); + } + } + return failures; +} + +async function startLocalServer() { + const app = createApp(); + const server = app.listen(0); + await new Promise((resolve, reject) => { + server.once("listening", resolve); + server.once("error", reject); + }); + const { port } = server.address(); + return { + baseUrl: `http://127.0.0.1:${port}`, + close: () => new Promise((resolve, reject) => server.close((error) => (error ? reject(error) : resolve()))) + }; +} + +async function main() { + await loadBenchmarkEnv(); + await fs.mkdir(resultsDir, { recursive: true }); + + const config = { + concurrency: Number(process.env.BENCHMARK_CONCURRENCY || (smoke ? 1 : 2)), + requestsPerRoute: Number(process.env.BENCHMARK_REQUESTS_PER_ROUTE || (smoke ? 2 : 6)) + }; + + const localServer = process.env.BENCHMARK_TARGET_URL + ? null + : await startLocalServer(); + const baseUrl = process.env.BENCHMARK_TARGET_URL || localServer.baseUrl; + const adminToken = signAccessToken({ sub: "benchmark_admin", role: "admin" }); + const startedAt = new Date().toISOString(); + + try { + const routes = []; + for (const route of API_ROUTES) { + routes.push(await benchmarkRoute(baseUrl, route, adminToken, config)); + } + + const result = { + mode: smoke ? "smoke" : "full", + target: process.env.BENCHMARK_TARGET_URL ? "external" : "local-express", + baseUrl, + startedAt, + finishedAt: new Date().toISOString(), + config, + environment: { + platform: os.platform(), + release: os.release(), + arch: os.arch(), + cpus: os.cpus().map((cpu) => cpu.model), + cpuCount: os.cpus().length, + totalMemoryMb: Math.round(os.totalmem() / 1024 / 1024), + freeMemoryMb: Math.round(os.freemem() / 1024 / 1024), + nodeVersion: process.version + }, + routes, + totals: { + routeCount: routes.length, + requestCount: routes.reduce((sum, route) => sum + route.requestCount, 0), + errorCount: routes.reduce((sum, route) => sum + route.errorCount, 0), + errorRate: round( + routes.reduce((sum, route) => sum + route.errorCount, 0) / + Math.max(routes.reduce((sum, route) => sum + route.requestCount, 0), 1), + 4 + ), + maxP99Ms: Math.max(...routes.map((route) => route.p99Ms)), + maxTtfbP99Ms: Math.max(...routes.map((route) => route.ttfbP99Ms)) + } + }; + + const thresholds = JSON.parse(await fs.readFile(thresholdsPath, "utf8")); + const failures = enforceThresholds(result, thresholds); + result.thresholds = thresholds; + result.thresholdFailures = failures; + + const jsonPath = path.join(resultsDir, `${outPrefix}.json`); + const markdownPath = path.join(resultsDir, `${outPrefix}.md`); + await fs.writeFile(jsonPath, `${JSON.stringify(result, null, 2)}\n`); + await fs.writeFile(markdownPath, summaryMarkdown(result)); + + console.log(summaryMarkdown(result)); + if (failures.length) { + console.error("\nBenchmark threshold failures:"); + for (const failure of failures) console.error(`- ${failure}`); + process.exit(1); + } + } finally { + if (localServer) await localServer.close(); + } +} + +main().catch((error) => { + console.error(error); + process.exit(1); +}); diff --git a/benchmarks/thresholds.json b/benchmarks/thresholds.json new file mode 100644 index 000000000..c787509aa --- /dev/null +++ b/benchmarks/thresholds.json @@ -0,0 +1,5 @@ +{ + "p99MaxMs": 1000, + "ttfbP99MaxMs": 1000, + "errorRateMax": 0 +} diff --git a/demos/api-benchmark-demo.mp4 b/demos/api-benchmark-demo.mp4 new file mode 100644 index 000000000..a64f3f42f Binary files /dev/null and b/demos/api-benchmark-demo.mp4 differ diff --git a/package.json b/package.json index 675e6e69d..5a2854706 100644 --- a/package.json +++ b/package.json @@ -7,6 +7,9 @@ ], "scripts": { "build": "echo \"Run package-specific builds (e.g. npm run build -w apps/web)\"", + "benchmark": "node benchmarks/run-api-benchmark.mjs", + "benchmark:coverage": "node benchmarks/check-api-coverage.mjs", + "benchmark:smoke": "node benchmarks/run-api-benchmark.mjs --smoke", "lint": "echo \"No root lint configured\"", "test": "npm run test -w apps/api" }