diff --git a/.env.benchmark.example b/.env.benchmark.example new file mode 100644 index 000000000..6377e1660 --- /dev/null +++ b/.env.benchmark.example @@ -0,0 +1,5 @@ +BENCHMARK_TARGET_URL=http://127.0.0.1:4000 +BENCHMARK_ADMIN_TOKEN= +BENCHMARK_CONCURRENCY=2 +BENCHMARK_REQUESTS_PER_ENDPOINT=8 +BENCHMARK_OUTPUT_DIR=benchmarks/results diff --git a/.gitignore b/.gitignore index 1e3ce10dd..e02a5dc48 100644 --- a/.gitignore +++ b/.gitignore @@ -3,5 +3,6 @@ node_modules dist .env .env.* +!.env.benchmark.example coverage *.log diff --git a/apps/api/package.json b/apps/api/package.json index 25fa0e90e..d32d03beb 100644 --- a/apps/api/package.json +++ b/apps/api/package.json @@ -5,7 +5,7 @@ "scripts": { "dev": "node src/server.js", "start": "node src/server.js", - "test": "node --test src/tests" + "test": "node --test src/tests/*.js" }, "dependencies": { "cors": "^2.8.5", diff --git a/benchmarks/README.md b/benchmarks/README.md new file mode 100644 index 000000000..b5f582152 --- /dev/null +++ b/benchmarks/README.md @@ -0,0 +1,38 @@ +# API Benchmarks + +This benchmark suite covers every currently mounted API endpoint in `apps/api/src/app.js`, plus `/health`. + +## Commands + +```sh +npm run benchmark +npm run benchmark:smoke +npm run benchmark:coverage +``` + +By default the runner starts the Express app in-process on a random loopback port. To run against an already running local or staging server, set `BENCHMARK_TARGET_URL`. + +## Configuration + +Copy `.env.benchmark.example` into your shell environment or CI secret configuration and adjust values as needed: + +- `BENCHMARK_TARGET_URL`: optional target base URL. If unset, the runner starts the local app. +- `BENCHMARK_ADMIN_TOKEN`: optional bearer token for protected admin routes. If unset for local runs, the runner creates a short-lived benchmark token. +- `BENCHMARK_CONCURRENCY`: parallel requests per endpoint. +- `BENCHMARK_REQUESTS_PER_ENDPOINT`: measured requests per endpoint. +- `BENCHMARK_OUTPUT_DIR`: output directory for JSON and Markdown reports. + +Thresholds live in `benchmarks/thresholds.json`. `npm run benchmark:smoke` is the low-concurrency gate command intended for CI, while local full runs provide broader baseline data. + +`npm run benchmark:coverage` checks the benchmark endpoint inventory against the API route mounts in `apps/api/src/app.js`. + +## Output + +Each run writes: + +- `benchmarks/results/full-latest.json` +- `benchmarks/results/full-latest.md` +- `benchmarks/results/smoke-latest.json` +- `benchmarks/results/smoke-latest.md` + +Reports include p50, p95, p99 latency, p99 time to first byte, requests per second, peak requests per second, and error rate for each endpoint. diff --git a/benchmarks/endpoints.mjs b/benchmarks/endpoints.mjs new file mode 100644 index 000000000..9f0a8ae69 --- /dev/null +++ b/benchmarks/endpoints.mjs @@ -0,0 +1,186 @@ +export const benchmarkEndpoints = [ + { + name: "health", + method: "GET", + path: "/health", + expectedStatus: 200 + }, + { + name: "auth-register", + method: "POST", + path: "/api/auth/register", + expectedStatus: 201, + json: ({ iteration }) => ({ + email: `benchmark-user-${Date.now()}-${iteration}@example.com`, + password: "benchmark-pass-123", + role: "freelancer" + }) + }, + { + name: "auth-login", + method: "POST", + path: "/api/auth/login", + expectedStatus: 200, + json: { + email: "benchmark-user@example.com", + password: "benchmark-pass-123" + } + }, + { + name: "auth-oauth-callback", + method: "GET", + path: "/api/auth/oauth/benchmark/callback", + expectedStatus: 200 + }, + { + name: "auth-refresh", + method: "POST", + path: "/api/auth/refresh", + expectedStatus: 200 + }, + { + name: "users-list", + method: "GET", + path: "/api/users", + expectedStatus: 200 + }, + { + name: "users-create", + method: "POST", + path: "/api/users", + expectedStatus: 201, + json: ({ iteration }) => ({ + email: `benchmark-client-${iteration}@example.com`, + name: "Benchmark Client", + role: "client" + }) + }, + { + name: "jobs-list", + method: "GET", + path: "/api/jobs", + expectedStatus: 200 + }, + { + name: "jobs-create", + method: "POST", + path: "/api/jobs", + expectedStatus: 201, + json: { + title: "Benchmark API implementation", + description: "Create a benchmark suite with realistic local payloads.", + budgetMin: 500, + budgetMax: 1500, + categoryId: "software-development", + skills: ["node", "express", "benchmarking"] + } + }, + { + name: "proposals-list", + method: "GET", + path: "/api/proposals", + expectedStatus: 200 + }, + { + name: "proposals-create", + method: "POST", + path: "/api/proposals", + expectedStatus: 201, + json: { + jobId: "job_benchmark", + freelancerId: "usr_benchmark", + coverLetter: "I can deliver a reproducible API benchmark suite.", + price: 950, + timelineDays: 5 + } + }, + { + name: "payments-create", + method: "POST", + path: "/api/payments", + expectedStatus: 201, + json: { + amount: 95000, + currency: "usd", + jobId: "job_benchmark" + } + }, + { + name: "reviews-list", + method: "GET", + path: "/api/reviews", + expectedStatus: 200 + }, + { + name: "reviews-create", + method: "POST", + path: "/api/reviews", + expectedStatus: 201, + json: { + jobId: "job_benchmark", + reviewerId: "usr_client", + revieweeId: "usr_freelancer", + rating: 5, + comment: "Fast delivery and clear reporting." + } + }, + { + name: "messages-list", + method: "GET", + path: "/api/messages", + expectedStatus: 200 + }, + { + name: "messages-create", + method: "POST", + path: "/api/messages", + expectedStatus: 201, + json: { + fromUserId: "usr_client", + toUserId: "usr_freelancer", + body: "Can you share the latest benchmark report?" + } + }, + { + name: "notifications-list", + method: "GET", + path: "/api/notifications", + expectedStatus: 200 + }, + { + name: "notifications-create", + method: "POST", + path: "/api/notifications", + expectedStatus: 201, + json: { + userId: "usr_client", + type: "proposal_received", + message: "A freelancer submitted a proposal." + } + }, + { + name: "uploads-create", + method: "POST", + path: "/api/uploads", + expectedStatus: 201, + multipart: { + fieldName: "file", + filename: "benchmark-profile.txt", + type: "text/plain", + body: "Synthetic benchmark upload payload for API route coverage." + } + }, + { + name: "search", + method: "GET", + path: "/api/search?q=benchmark", + expectedStatus: 200 + }, + { + name: "admin-metrics", + method: "GET", + path: "/api/admin/metrics", + expectedStatus: 200, + auth: true + } +]; diff --git a/benchmarks/results/full-latest.json b/benchmarks/results/full-latest.json new file mode 100644 index 000000000..5eed1522f --- /dev/null +++ b/benchmarks/results/full-latest.json @@ -0,0 +1,651 @@ +{ + "mode": "full", + "target": "http://127.0.0.1:51582", + "generatedAt": "2026-05-20T20:29:40.993Z", + "configuration": { + "concurrency": 2, + "requestsPerEndpoint": 8, + "thresholdsFile": "benchmarks/thresholds.json" + }, + "summary": { + "endpointCount": 21, + "totalRequests": 168, + "totalErrors": 0, + "errorRate": 0, + "maxP99LatencyMs": 13.01, + "maxP99TtfbMs": 12.74, + "passed": true + }, + "results": [ + { + "name": "health", + "method": "GET", + "path": "/health", + "expectedStatus": 200, + "requests": 8, + "errors": 0, + "errorRate": 0, + "requestsPerSecond": 390.66, + "peakRequestsPerSecond": 2000, + "latencyMs": { + "mean": 4.74, + "p50": 1.87, + "p95": 13.01, + "p99": 13.01 + }, + "ttfbMs": { + "mean": 4.58, + "p50": 1.72, + "p95": 12.74, + "p99": 12.74 + }, + "threshold": { + "p99LatencyMs": 1000, + "p99TtfbMs": 1000, + "errorRate": 0 + }, + "passedThresholds": true, + "failures": [] + }, + { + "name": "auth-register", + "method": "POST", + "path": "/api/auth/register", + "expectedStatus": 201, + "requests": 8, + "errors": 0, + "errorRate": 0, + "requestsPerSecond": 574.19, + "peakRequestsPerSecond": 2000, + "latencyMs": { + "mean": 3.34, + "p50": 1.37, + "p95": 8.73, + "p99": 8.73 + }, + "ttfbMs": { + "mean": 3.29, + "p50": 1.34, + "p95": 8.55, + "p99": 8.55 + }, + "threshold": { + "p99LatencyMs": 1000, + "p99TtfbMs": 1000, + "errorRate": 0 + }, + "passedThresholds": true, + "failures": [] + }, + { + "name": "auth-login", + "method": "POST", + "path": "/api/auth/login", + "expectedStatus": 200, + "requests": 8, + "errors": 0, + "errorRate": 0, + "requestsPerSecond": 1491.7, + "peakRequestsPerSecond": 2000, + "latencyMs": { + "mean": 1.3, + "p50": 1.32, + "p95": 1.64, + "p99": 1.64 + }, + "ttfbMs": { + "mean": 1.26, + "p50": 1.29, + "p95": 1.58, + "p99": 1.58 + }, + "threshold": { + "p99LatencyMs": 1000, + "p99TtfbMs": 1000, + "errorRate": 0 + }, + "passedThresholds": true, + "failures": [] + }, + { + "name": "auth-oauth-callback", + "method": "GET", + "path": "/api/auth/oauth/benchmark/callback", + "expectedStatus": 200, + "requests": 8, + "errors": 0, + "errorRate": 0, + "requestsPerSecond": 3102.73, + "peakRequestsPerSecond": 2000, + "latencyMs": { + "mean": 0.61, + "p50": 0.53, + "p95": 0.9, + "p99": 0.9 + }, + "ttfbMs": { + "mean": 0.58, + "p50": 0.5, + "p95": 0.86, + "p99": 0.86 + }, + "threshold": { + "p99LatencyMs": 1000, + "p99TtfbMs": 1000, + "errorRate": 0 + }, + "passedThresholds": true, + "failures": [] + }, + { + "name": "auth-refresh", + "method": "POST", + "path": "/api/auth/refresh", + "expectedStatus": 200, + "requests": 8, + "errors": 0, + "errorRate": 0, + "requestsPerSecond": 3310.46, + "peakRequestsPerSecond": 2000, + "latencyMs": { + "mean": 0.59, + "p50": 0.55, + "p95": 0.75, + "p99": 0.75 + }, + "ttfbMs": { + "mean": 0.56, + "p50": 0.53, + "p95": 0.72, + "p99": 0.72 + }, + "threshold": { + "p99LatencyMs": 1000, + "p99TtfbMs": 1000, + "errorRate": 0 + }, + "passedThresholds": true, + "failures": [] + }, + { + "name": "users-list", + "method": "GET", + "path": "/api/users", + "expectedStatus": 200, + "requests": 8, + "errors": 0, + "errorRate": 0, + "requestsPerSecond": 3269.87, + "peakRequestsPerSecond": 2000, + "latencyMs": { + "mean": 0.58, + "p50": 0.47, + "p95": 0.91, + "p99": 0.91 + }, + "ttfbMs": { + "mean": 0.54, + "p50": 0.44, + "p95": 0.89, + "p99": 0.89 + }, + "threshold": { + "p99LatencyMs": 1000, + "p99TtfbMs": 1000, + "errorRate": 0 + }, + "passedThresholds": true, + "failures": [] + }, + { + "name": "users-create", + "method": "POST", + "path": "/api/users", + "expectedStatus": 201, + "requests": 8, + "errors": 0, + "errorRate": 0, + "requestsPerSecond": 2616.98, + "peakRequestsPerSecond": 2000, + "latencyMs": { + "mean": 0.73, + "p50": 0.62, + "p95": 1.14, + "p99": 1.14 + }, + "ttfbMs": { + "mean": 0.7, + "p50": 0.6, + "p95": 1.12, + "p99": 1.12 + }, + "threshold": { + "p99LatencyMs": 1000, + "p99TtfbMs": 1000, + "errorRate": 0 + }, + "passedThresholds": true, + "failures": [] + }, + { + "name": "jobs-list", + "method": "GET", + "path": "/api/jobs", + "expectedStatus": 200, + "requests": 8, + "errors": 0, + "errorRate": 0, + "requestsPerSecond": 3787.51, + "peakRequestsPerSecond": 2000, + "latencyMs": { + "mean": 0.51, + "p50": 0.5, + "p95": 0.57, + "p99": 0.57 + }, + "ttfbMs": { + "mean": 0.48, + "p50": 0.47, + "p95": 0.55, + "p99": 0.55 + }, + "threshold": { + "p99LatencyMs": 1000, + "p99TtfbMs": 1000, + "errorRate": 0 + }, + "passedThresholds": true, + "failures": [] + }, + { + "name": "jobs-create", + "method": "POST", + "path": "/api/jobs", + "expectedStatus": 201, + "requests": 8, + "errors": 0, + "errorRate": 0, + "requestsPerSecond": 2067.54, + "peakRequestsPerSecond": 2000, + "latencyMs": { + "mean": 0.95, + "p50": 0.78, + "p95": 1.57, + "p99": 1.57 + }, + "ttfbMs": { + "mean": 0.92, + "p50": 0.75, + "p95": 1.54, + "p99": 1.54 + }, + "threshold": { + "p99LatencyMs": 1000, + "p99TtfbMs": 1000, + "errorRate": 0 + }, + "passedThresholds": true, + "failures": [] + }, + { + "name": "proposals-list", + "method": "GET", + "path": "/api/proposals", + "expectedStatus": 200, + "requests": 8, + "errors": 0, + "errorRate": 0, + "requestsPerSecond": 4142.3, + "peakRequestsPerSecond": 2000, + "latencyMs": { + "mean": 0.47, + "p50": 0.46, + "p95": 0.55, + "p99": 0.55 + }, + "ttfbMs": { + "mean": 0.44, + "p50": 0.43, + "p95": 0.53, + "p99": 0.53 + }, + "threshold": { + "p99LatencyMs": 1000, + "p99TtfbMs": 1000, + "errorRate": 0 + }, + "passedThresholds": true, + "failures": [] + }, + { + "name": "proposals-create", + "method": "POST", + "path": "/api/proposals", + "expectedStatus": 201, + "requests": 8, + "errors": 0, + "errorRate": 0, + "requestsPerSecond": 3359.05, + "peakRequestsPerSecond": 2000, + "latencyMs": { + "mean": 0.58, + "p50": 0.57, + "p95": 0.7, + "p99": 0.7 + }, + "ttfbMs": { + "mean": 0.55, + "p50": 0.54, + "p95": 0.67, + "p99": 0.67 + }, + "threshold": { + "p99LatencyMs": 1000, + "p99TtfbMs": 1000, + "errorRate": 0 + }, + "passedThresholds": true, + "failures": [] + }, + { + "name": "payments-create", + "method": "POST", + "path": "/api/payments", + "expectedStatus": 201, + "requests": 8, + "errors": 0, + "errorRate": 0, + "requestsPerSecond": 3376.24, + "peakRequestsPerSecond": 2000, + "latencyMs": { + "mean": 0.58, + "p50": 0.57, + "p95": 0.67, + "p99": 0.67 + }, + "ttfbMs": { + "mean": 0.55, + "p50": 0.55, + "p95": 0.64, + "p99": 0.64 + }, + "threshold": { + "p99LatencyMs": 1000, + "p99TtfbMs": 1000, + "errorRate": 0 + }, + "passedThresholds": true, + "failures": [] + }, + { + "name": "reviews-list", + "method": "GET", + "path": "/api/reviews", + "expectedStatus": 200, + "requests": 8, + "errors": 0, + "errorRate": 0, + "requestsPerSecond": 3575.88, + "peakRequestsPerSecond": 2000, + "latencyMs": { + "mean": 0.53, + "p50": 0.46, + "p95": 0.75, + "p99": 0.75 + }, + "ttfbMs": { + "mean": 0.5, + "p50": 0.44, + "p95": 0.72, + "p99": 0.72 + }, + "threshold": { + "p99LatencyMs": 1000, + "p99TtfbMs": 1000, + "errorRate": 0 + }, + "passedThresholds": true, + "failures": [] + }, + { + "name": "reviews-create", + "method": "POST", + "path": "/api/reviews", + "expectedStatus": 201, + "requests": 8, + "errors": 0, + "errorRate": 0, + "requestsPerSecond": 2460.75, + "peakRequestsPerSecond": 2000, + "latencyMs": { + "mean": 0.79, + "p50": 0.59, + "p95": 1.44, + "p99": 1.44 + }, + "ttfbMs": { + "mean": 0.76, + "p50": 0.57, + "p95": 1.35, + "p99": 1.35 + }, + "threshold": { + "p99LatencyMs": 1000, + "p99TtfbMs": 1000, + "errorRate": 0 + }, + "passedThresholds": true, + "failures": [] + }, + { + "name": "messages-list", + "method": "GET", + "path": "/api/messages", + "expectedStatus": 200, + "requests": 8, + "errors": 0, + "errorRate": 0, + "requestsPerSecond": 4278.26, + "peakRequestsPerSecond": 2000, + "latencyMs": { + "mean": 0.45, + "p50": 0.45, + "p95": 0.54, + "p99": 0.54 + }, + "ttfbMs": { + "mean": 0.43, + "p50": 0.42, + "p95": 0.51, + "p99": 0.51 + }, + "threshold": { + "p99LatencyMs": 1000, + "p99TtfbMs": 1000, + "errorRate": 0 + }, + "passedThresholds": true, + "failures": [] + }, + { + "name": "messages-create", + "method": "POST", + "path": "/api/messages", + "expectedStatus": 201, + "requests": 8, + "errors": 0, + "errorRate": 0, + "requestsPerSecond": 1984.31, + "peakRequestsPerSecond": 2000, + "latencyMs": { + "mean": 0.99, + "p50": 0.77, + "p95": 1.73, + "p99": 1.73 + }, + "ttfbMs": { + "mean": 0.96, + "p50": 0.74, + "p95": 1.7, + "p99": 1.7 + }, + "threshold": { + "p99LatencyMs": 1000, + "p99TtfbMs": 1000, + "errorRate": 0 + }, + "passedThresholds": true, + "failures": [] + }, + { + "name": "notifications-list", + "method": "GET", + "path": "/api/notifications", + "expectedStatus": 200, + "requests": 8, + "errors": 0, + "errorRate": 0, + "requestsPerSecond": 4168.84, + "peakRequestsPerSecond": 2000, + "latencyMs": { + "mean": 0.46, + "p50": 0.44, + "p95": 0.57, + "p99": 0.57 + }, + "ttfbMs": { + "mean": 0.44, + "p50": 0.41, + "p95": 0.55, + "p99": 0.55 + }, + "threshold": { + "p99LatencyMs": 1000, + "p99TtfbMs": 1000, + "errorRate": 0 + }, + "passedThresholds": true, + "failures": [] + }, + { + "name": "notifications-create", + "method": "POST", + "path": "/api/notifications", + "expectedStatus": 201, + "requests": 8, + "errors": 0, + "errorRate": 0, + "requestsPerSecond": 3563.67, + "peakRequestsPerSecond": 2000, + "latencyMs": { + "mean": 0.54, + "p50": 0.53, + "p95": 0.66, + "p99": 0.66 + }, + "ttfbMs": { + "mean": 0.52, + "p50": 0.51, + "p95": 0.64, + "p99": 0.64 + }, + "threshold": { + "p99LatencyMs": 1000, + "p99TtfbMs": 1000, + "errorRate": 0 + }, + "passedThresholds": true, + "failures": [] + }, + { + "name": "uploads-create", + "method": "POST", + "path": "/api/uploads", + "expectedStatus": 201, + "requests": 8, + "errors": 0, + "errorRate": 0, + "requestsPerSecond": 939.32, + "peakRequestsPerSecond": 2000, + "latencyMs": { + "mean": 2.05, + "p50": 1.17, + "p95": 4.86, + "p99": 4.86 + }, + "ttfbMs": { + "mean": 2.02, + "p50": 1.15, + "p95": 4.83, + "p99": 4.83 + }, + "threshold": { + "p99LatencyMs": 1500, + "p99TtfbMs": 1500, + "errorRate": 0 + }, + "passedThresholds": true, + "failures": [] + }, + { + "name": "search", + "method": "GET", + "path": "/api/search?q=benchmark", + "expectedStatus": 200, + "requests": 8, + "errors": 0, + "errorRate": 0, + "requestsPerSecond": 3003.28, + "peakRequestsPerSecond": 2000, + "latencyMs": { + "mean": 0.65, + "p50": 0.53, + "p95": 1.05, + "p99": 1.05 + }, + "ttfbMs": { + "mean": 0.61, + "p50": 0.5, + "p95": 1.03, + "p99": 1.03 + }, + "threshold": { + "p99LatencyMs": 1000, + "p99TtfbMs": 1000, + "errorRate": 0 + }, + "passedThresholds": true, + "failures": [] + }, + { + "name": "admin-metrics", + "method": "GET", + "path": "/api/admin/metrics", + "expectedStatus": 200, + "requests": 8, + "errors": 0, + "errorRate": 0, + "requestsPerSecond": 3019.67, + "peakRequestsPerSecond": 2000, + "latencyMs": { + "mean": 0.64, + "p50": 0.52, + "p95": 1.02, + "p99": 1.02 + }, + "ttfbMs": { + "mean": 0.62, + "p50": 0.5, + "p95": 1, + "p99": 1 + }, + "threshold": { + "p99LatencyMs": 1000, + "p99TtfbMs": 1000, + "errorRate": 0 + }, + "passedThresholds": true, + "failures": [] + } + ] +} diff --git a/benchmarks/results/full-latest.md b/benchmarks/results/full-latest.md new file mode 100644 index 000000000..5e61b0e53 --- /dev/null +++ b/benchmarks/results/full-latest.md @@ -0,0 +1,34 @@ +# API Benchmark Report + +- Mode: full +- Target: http://127.0.0.1:51582 +- Generated: 2026-05-20T20:29:40.993Z +- Endpoints: 21 +- Requests: 168 +- Error rate: 0.00% +- Max p99 latency: 13.01 ms +- Max p99 TTFB: 12.74 ms + +| Gate | Method | Path | Requests | Error rate | p50 ms | p95 ms | p99 ms | p99 TTFB ms | RPS | +| --- | --- | --- | ---: | ---: | ---: | ---: | ---: | ---: | ---: | +| pass | GET | /health | 8 | 0.00% | 1.87 | 13.01 | 13.01 | 12.74 | 390.66 | +| pass | POST | /api/auth/register | 8 | 0.00% | 1.37 | 8.73 | 8.73 | 8.55 | 574.19 | +| pass | POST | /api/auth/login | 8 | 0.00% | 1.32 | 1.64 | 1.64 | 1.58 | 1491.7 | +| pass | GET | /api/auth/oauth/benchmark/callback | 8 | 0.00% | 0.53 | 0.9 | 0.9 | 0.86 | 3102.73 | +| pass | POST | /api/auth/refresh | 8 | 0.00% | 0.55 | 0.75 | 0.75 | 0.72 | 3310.46 | +| pass | GET | /api/users | 8 | 0.00% | 0.47 | 0.91 | 0.91 | 0.89 | 3269.87 | +| pass | POST | /api/users | 8 | 0.00% | 0.62 | 1.14 | 1.14 | 1.12 | 2616.98 | +| pass | GET | /api/jobs | 8 | 0.00% | 0.5 | 0.57 | 0.57 | 0.55 | 3787.51 | +| pass | POST | /api/jobs | 8 | 0.00% | 0.78 | 1.57 | 1.57 | 1.54 | 2067.54 | +| pass | GET | /api/proposals | 8 | 0.00% | 0.46 | 0.55 | 0.55 | 0.53 | 4142.3 | +| pass | POST | /api/proposals | 8 | 0.00% | 0.57 | 0.7 | 0.7 | 0.67 | 3359.05 | +| pass | POST | /api/payments | 8 | 0.00% | 0.57 | 0.67 | 0.67 | 0.64 | 3376.24 | +| pass | GET | /api/reviews | 8 | 0.00% | 0.46 | 0.75 | 0.75 | 0.72 | 3575.88 | +| pass | POST | /api/reviews | 8 | 0.00% | 0.59 | 1.44 | 1.44 | 1.35 | 2460.75 | +| pass | GET | /api/messages | 8 | 0.00% | 0.45 | 0.54 | 0.54 | 0.51 | 4278.26 | +| pass | POST | /api/messages | 8 | 0.00% | 0.77 | 1.73 | 1.73 | 1.7 | 1984.31 | +| pass | GET | /api/notifications | 8 | 0.00% | 0.44 | 0.57 | 0.57 | 0.55 | 4168.84 | +| pass | POST | /api/notifications | 8 | 0.00% | 0.53 | 0.66 | 0.66 | 0.64 | 3563.67 | +| pass | POST | /api/uploads | 8 | 0.00% | 1.17 | 4.86 | 4.86 | 4.83 | 939.32 | +| pass | GET | /api/search?q=benchmark | 8 | 0.00% | 0.53 | 1.05 | 1.05 | 1.03 | 3003.28 | +| pass | GET | /api/admin/metrics | 8 | 0.00% | 0.52 | 1.02 | 1.02 | 1 | 3019.67 | diff --git a/benchmarks/results/smoke-latest.json b/benchmarks/results/smoke-latest.json new file mode 100644 index 000000000..f1b88bd52 --- /dev/null +++ b/benchmarks/results/smoke-latest.json @@ -0,0 +1,651 @@ +{ + "mode": "smoke", + "target": "http://127.0.0.1:51574", + "generatedAt": "2026-05-20T20:29:37.611Z", + "configuration": { + "concurrency": 1, + "requestsPerEndpoint": 1, + "thresholdsFile": "benchmarks/thresholds.json" + }, + "summary": { + "endpointCount": 21, + "totalRequests": 21, + "totalErrors": 0, + "errorRate": 0, + "maxP99LatencyMs": 16.6, + "maxP99TtfbMs": 15.2, + "passed": true + }, + "results": [ + { + "name": "health", + "method": "GET", + "path": "/health", + "expectedStatus": 200, + "requests": 1, + "errors": 0, + "errorRate": 0, + "requestsPerSecond": 60.01, + "peakRequestsPerSecond": 60.24, + "latencyMs": { + "mean": 16.6, + "p50": 16.6, + "p95": 16.6, + "p99": 16.6 + }, + "ttfbMs": { + "mean": 15.2, + "p50": 15.2, + "p95": 15.2, + "p99": 15.2 + }, + "threshold": { + "p99LatencyMs": 1000, + "p99TtfbMs": 1000, + "errorRate": 0 + }, + "passedThresholds": true, + "failures": [] + }, + { + "name": "auth-register", + "method": "POST", + "path": "/api/auth/register", + "expectedStatus": 201, + "requests": 1, + "errors": 0, + "errorRate": 0, + "requestsPerSecond": 128.67, + "peakRequestsPerSecond": 129.03, + "latencyMs": { + "mean": 7.75, + "p50": 7.75, + "p95": 7.75, + "p99": 7.75 + }, + "ttfbMs": { + "mean": 7.56, + "p50": 7.56, + "p95": 7.56, + "p99": 7.56 + }, + "threshold": { + "p99LatencyMs": 1000, + "p99TtfbMs": 1000, + "errorRate": 0 + }, + "passedThresholds": true, + "failures": [] + }, + { + "name": "auth-login", + "method": "POST", + "path": "/api/auth/login", + "expectedStatus": 200, + "requests": 1, + "errors": 0, + "errorRate": 0, + "requestsPerSecond": 355.91, + "peakRequestsPerSecond": 357.14, + "latencyMs": { + "mean": 2.8, + "p50": 2.8, + "p95": 2.8, + "p99": 2.8 + }, + "ttfbMs": { + "mean": 2.67, + "p50": 2.67, + "p95": 2.67, + "p99": 2.67 + }, + "threshold": { + "p99LatencyMs": 1000, + "p99TtfbMs": 1000, + "errorRate": 0 + }, + "passedThresholds": true, + "failures": [] + }, + { + "name": "auth-oauth-callback", + "method": "GET", + "path": "/api/auth/oauth/benchmark/callback", + "expectedStatus": 200, + "requests": 1, + "errors": 0, + "errorRate": 0, + "requestsPerSecond": 820.32, + "peakRequestsPerSecond": 826.45, + "latencyMs": { + "mean": 1.21, + "p50": 1.21, + "p95": 1.21, + "p99": 1.21 + }, + "ttfbMs": { + "mean": 1.14, + "p50": 1.14, + "p95": 1.14, + "p99": 1.14 + }, + "threshold": { + "p99LatencyMs": 1000, + "p99TtfbMs": 1000, + "errorRate": 0 + }, + "passedThresholds": true, + "failures": [] + }, + { + "name": "auth-refresh", + "method": "POST", + "path": "/api/auth/refresh", + "expectedStatus": 200, + "requests": 1, + "errors": 0, + "errorRate": 0, + "requestsPerSecond": 465.13, + "peakRequestsPerSecond": 467.29, + "latencyMs": { + "mean": 2.14, + "p50": 2.14, + "p95": 2.14, + "p99": 2.14 + }, + "ttfbMs": { + "mean": 1.77, + "p50": 1.77, + "p95": 1.77, + "p99": 1.77 + }, + "threshold": { + "p99LatencyMs": 1000, + "p99TtfbMs": 1000, + "errorRate": 0 + }, + "passedThresholds": true, + "failures": [] + }, + { + "name": "users-list", + "method": "GET", + "path": "/api/users", + "expectedStatus": 200, + "requests": 1, + "errors": 0, + "errorRate": 0, + "requestsPerSecond": 670.37, + "peakRequestsPerSecond": 675.68, + "latencyMs": { + "mean": 1.48, + "p50": 1.48, + "p95": 1.48, + "p99": 1.48 + }, + "ttfbMs": { + "mean": 1.07, + "p50": 1.07, + "p95": 1.07, + "p99": 1.07 + }, + "threshold": { + "p99LatencyMs": 1000, + "p99TtfbMs": 1000, + "errorRate": 0 + }, + "passedThresholds": true, + "failures": [] + }, + { + "name": "users-create", + "method": "POST", + "path": "/api/users", + "expectedStatus": 201, + "requests": 1, + "errors": 0, + "errorRate": 0, + "requestsPerSecond": 381.49, + "peakRequestsPerSecond": 384.62, + "latencyMs": { + "mean": 2.6, + "p50": 2.6, + "p95": 2.6, + "p99": 2.6 + }, + "ttfbMs": { + "mean": 2.04, + "p50": 2.04, + "p95": 2.04, + "p99": 2.04 + }, + "threshold": { + "p99LatencyMs": 1000, + "p99TtfbMs": 1000, + "errorRate": 0 + }, + "passedThresholds": true, + "failures": [] + }, + { + "name": "jobs-list", + "method": "GET", + "path": "/api/jobs", + "expectedStatus": 200, + "requests": 1, + "errors": 0, + "errorRate": 0, + "requestsPerSecond": 915.61, + "peakRequestsPerSecond": 925.93, + "latencyMs": { + "mean": 1.08, + "p50": 1.08, + "p95": 1.08, + "p99": 1.08 + }, + "ttfbMs": { + "mean": 0.98, + "p50": 0.98, + "p95": 0.98, + "p99": 0.98 + }, + "threshold": { + "p99LatencyMs": 1000, + "p99TtfbMs": 1000, + "errorRate": 0 + }, + "passedThresholds": true, + "failures": [] + }, + { + "name": "jobs-create", + "method": "POST", + "path": "/api/jobs", + "expectedStatus": 201, + "requests": 1, + "errors": 0, + "errorRate": 0, + "requestsPerSecond": 262.83, + "peakRequestsPerSecond": 263.16, + "latencyMs": { + "mean": 3.8, + "p50": 3.8, + "p95": 3.8, + "p99": 3.8 + }, + "ttfbMs": { + "mean": 3.6, + "p50": 3.6, + "p95": 3.6, + "p99": 3.6 + }, + "threshold": { + "p99LatencyMs": 1000, + "p99TtfbMs": 1000, + "errorRate": 0 + }, + "passedThresholds": true, + "failures": [] + }, + { + "name": "proposals-list", + "method": "GET", + "path": "/api/proposals", + "expectedStatus": 200, + "requests": 1, + "errors": 0, + "errorRate": 0, + "requestsPerSecond": 443.45, + "peakRequestsPerSecond": 446.43, + "latencyMs": { + "mean": 2.24, + "p50": 2.24, + "p95": 2.24, + "p99": 2.24 + }, + "ttfbMs": { + "mean": 2.17, + "p50": 2.17, + "p95": 2.17, + "p99": 2.17 + }, + "threshold": { + "p99LatencyMs": 1000, + "p99TtfbMs": 1000, + "errorRate": 0 + }, + "passedThresholds": true, + "failures": [] + }, + { + "name": "proposals-create", + "method": "POST", + "path": "/api/proposals", + "expectedStatus": 201, + "requests": 1, + "errors": 0, + "errorRate": 0, + "requestsPerSecond": 782.98, + "peakRequestsPerSecond": 787.4, + "latencyMs": { + "mean": 1.27, + "p50": 1.27, + "p95": 1.27, + "p99": 1.27 + }, + "ttfbMs": { + "mean": 1.22, + "p50": 1.22, + "p95": 1.22, + "p99": 1.22 + }, + "threshold": { + "p99LatencyMs": 1000, + "p99TtfbMs": 1000, + "errorRate": 0 + }, + "passedThresholds": true, + "failures": [] + }, + { + "name": "payments-create", + "method": "POST", + "path": "/api/payments", + "expectedStatus": 201, + "requests": 1, + "errors": 0, + "errorRate": 0, + "requestsPerSecond": 1000, + "peakRequestsPerSecond": 1000, + "latencyMs": { + "mean": 0.74, + "p50": 0.74, + "p95": 0.74, + "p99": 0.74 + }, + "ttfbMs": { + "mean": 0.69, + "p50": 0.69, + "p95": 0.69, + "p99": 0.69 + }, + "threshold": { + "p99LatencyMs": 1000, + "p99TtfbMs": 1000, + "errorRate": 0 + }, + "passedThresholds": true, + "failures": [] + }, + { + "name": "reviews-list", + "method": "GET", + "path": "/api/reviews", + "expectedStatus": 200, + "requests": 1, + "errors": 0, + "errorRate": 0, + "requestsPerSecond": 1000, + "peakRequestsPerSecond": 1000, + "latencyMs": { + "mean": 0.49, + "p50": 0.49, + "p95": 0.49, + "p99": 0.49 + }, + "ttfbMs": { + "mean": 0.46, + "p50": 0.46, + "p95": 0.46, + "p99": 0.46 + }, + "threshold": { + "p99LatencyMs": 1000, + "p99TtfbMs": 1000, + "errorRate": 0 + }, + "passedThresholds": true, + "failures": [] + }, + { + "name": "reviews-create", + "method": "POST", + "path": "/api/reviews", + "expectedStatus": 201, + "requests": 1, + "errors": 0, + "errorRate": 0, + "requestsPerSecond": 1000, + "peakRequestsPerSecond": 1000, + "latencyMs": { + "mean": 0.62, + "p50": 0.62, + "p95": 0.62, + "p99": 0.62 + }, + "ttfbMs": { + "mean": 0.59, + "p50": 0.59, + "p95": 0.59, + "p99": 0.59 + }, + "threshold": { + "p99LatencyMs": 1000, + "p99TtfbMs": 1000, + "errorRate": 0 + }, + "passedThresholds": true, + "failures": [] + }, + { + "name": "messages-list", + "method": "GET", + "path": "/api/messages", + "expectedStatus": 200, + "requests": 1, + "errors": 0, + "errorRate": 0, + "requestsPerSecond": 1000, + "peakRequestsPerSecond": 1000, + "latencyMs": { + "mean": 0.32, + "p50": 0.32, + "p95": 0.32, + "p99": 0.32 + }, + "ttfbMs": { + "mean": 0.29, + "p50": 0.29, + "p95": 0.29, + "p99": 0.29 + }, + "threshold": { + "p99LatencyMs": 1000, + "p99TtfbMs": 1000, + "errorRate": 0 + }, + "passedThresholds": true, + "failures": [] + }, + { + "name": "messages-create", + "method": "POST", + "path": "/api/messages", + "expectedStatus": 201, + "requests": 1, + "errors": 0, + "errorRate": 0, + "requestsPerSecond": 1000, + "peakRequestsPerSecond": 1000, + "latencyMs": { + "mean": 0.71, + "p50": 0.71, + "p95": 0.71, + "p99": 0.71 + }, + "ttfbMs": { + "mean": 0.67, + "p50": 0.67, + "p95": 0.67, + "p99": 0.67 + }, + "threshold": { + "p99LatencyMs": 1000, + "p99TtfbMs": 1000, + "errorRate": 0 + }, + "passedThresholds": true, + "failures": [] + }, + { + "name": "notifications-list", + "method": "GET", + "path": "/api/notifications", + "expectedStatus": 200, + "requests": 1, + "errors": 0, + "errorRate": 0, + "requestsPerSecond": 1000, + "peakRequestsPerSecond": 1000, + "latencyMs": { + "mean": 0.41, + "p50": 0.41, + "p95": 0.41, + "p99": 0.41 + }, + "ttfbMs": { + "mean": 0.38, + "p50": 0.38, + "p95": 0.38, + "p99": 0.38 + }, + "threshold": { + "p99LatencyMs": 1000, + "p99TtfbMs": 1000, + "errorRate": 0 + }, + "passedThresholds": true, + "failures": [] + }, + { + "name": "notifications-create", + "method": "POST", + "path": "/api/notifications", + "expectedStatus": 201, + "requests": 1, + "errors": 0, + "errorRate": 0, + "requestsPerSecond": 979.19, + "peakRequestsPerSecond": 980.39, + "latencyMs": { + "mean": 1.02, + "p50": 1.02, + "p95": 1.02, + "p99": 1.02 + }, + "ttfbMs": { + "mean": 0.98, + "p50": 0.98, + "p95": 0.98, + "p99": 0.98 + }, + "threshold": { + "p99LatencyMs": 1000, + "p99TtfbMs": 1000, + "errorRate": 0 + }, + "passedThresholds": true, + "failures": [] + }, + { + "name": "uploads-create", + "method": "POST", + "path": "/api/uploads", + "expectedStatus": 201, + "requests": 1, + "errors": 0, + "errorRate": 0, + "requestsPerSecond": 188.74, + "peakRequestsPerSecond": 189.04, + "latencyMs": { + "mean": 5.29, + "p50": 5.29, + "p95": 5.29, + "p99": 5.29 + }, + "ttfbMs": { + "mean": 5.24, + "p50": 5.24, + "p95": 5.24, + "p99": 5.24 + }, + "threshold": { + "p99LatencyMs": 1500, + "p99TtfbMs": 1500, + "errorRate": 0 + }, + "passedThresholds": true, + "failures": [] + }, + { + "name": "search", + "method": "GET", + "path": "/api/search?q=benchmark", + "expectedStatus": 200, + "requests": 1, + "errors": 0, + "errorRate": 0, + "requestsPerSecond": 1000, + "peakRequestsPerSecond": 1000, + "latencyMs": { + "mean": 0.87, + "p50": 0.87, + "p95": 0.87, + "p99": 0.87 + }, + "ttfbMs": { + "mean": 0.83, + "p50": 0.83, + "p95": 0.83, + "p99": 0.83 + }, + "threshold": { + "p99LatencyMs": 1000, + "p99TtfbMs": 1000, + "errorRate": 0 + }, + "passedThresholds": true, + "failures": [] + }, + { + "name": "admin-metrics", + "method": "GET", + "path": "/api/admin/metrics", + "expectedStatus": 200, + "requests": 1, + "errors": 0, + "errorRate": 0, + "requestsPerSecond": 1000, + "peakRequestsPerSecond": 1000, + "latencyMs": { + "mean": 0.87, + "p50": 0.87, + "p95": 0.87, + "p99": 0.87 + }, + "ttfbMs": { + "mean": 0.84, + "p50": 0.84, + "p95": 0.84, + "p99": 0.84 + }, + "threshold": { + "p99LatencyMs": 1000, + "p99TtfbMs": 1000, + "errorRate": 0 + }, + "passedThresholds": true, + "failures": [] + } + ] +} diff --git a/benchmarks/results/smoke-latest.md b/benchmarks/results/smoke-latest.md new file mode 100644 index 000000000..a6d891437 --- /dev/null +++ b/benchmarks/results/smoke-latest.md @@ -0,0 +1,34 @@ +# API Benchmark Report + +- Mode: smoke +- Target: http://127.0.0.1:51574 +- Generated: 2026-05-20T20:29:37.611Z +- Endpoints: 21 +- Requests: 21 +- Error rate: 0.00% +- Max p99 latency: 16.6 ms +- Max p99 TTFB: 15.2 ms + +| Gate | Method | Path | Requests | Error rate | p50 ms | p95 ms | p99 ms | p99 TTFB ms | RPS | +| --- | --- | --- | ---: | ---: | ---: | ---: | ---: | ---: | ---: | +| pass | GET | /health | 1 | 0.00% | 16.6 | 16.6 | 16.6 | 15.2 | 60.01 | +| pass | POST | /api/auth/register | 1 | 0.00% | 7.75 | 7.75 | 7.75 | 7.56 | 128.67 | +| pass | POST | /api/auth/login | 1 | 0.00% | 2.8 | 2.8 | 2.8 | 2.67 | 355.91 | +| pass | GET | /api/auth/oauth/benchmark/callback | 1 | 0.00% | 1.21 | 1.21 | 1.21 | 1.14 | 820.32 | +| pass | POST | /api/auth/refresh | 1 | 0.00% | 2.14 | 2.14 | 2.14 | 1.77 | 465.13 | +| pass | GET | /api/users | 1 | 0.00% | 1.48 | 1.48 | 1.48 | 1.07 | 670.37 | +| pass | POST | /api/users | 1 | 0.00% | 2.6 | 2.6 | 2.6 | 2.04 | 381.49 | +| pass | GET | /api/jobs | 1 | 0.00% | 1.08 | 1.08 | 1.08 | 0.98 | 915.61 | +| pass | POST | /api/jobs | 1 | 0.00% | 3.8 | 3.8 | 3.8 | 3.6 | 262.83 | +| pass | GET | /api/proposals | 1 | 0.00% | 2.24 | 2.24 | 2.24 | 2.17 | 443.45 | +| pass | POST | /api/proposals | 1 | 0.00% | 1.27 | 1.27 | 1.27 | 1.22 | 782.98 | +| pass | POST | /api/payments | 1 | 0.00% | 0.74 | 0.74 | 0.74 | 0.69 | 1000 | +| pass | GET | /api/reviews | 1 | 0.00% | 0.49 | 0.49 | 0.49 | 0.46 | 1000 | +| pass | POST | /api/reviews | 1 | 0.00% | 0.62 | 0.62 | 0.62 | 0.59 | 1000 | +| pass | GET | /api/messages | 1 | 0.00% | 0.32 | 0.32 | 0.32 | 0.29 | 1000 | +| pass | POST | /api/messages | 1 | 0.00% | 0.71 | 0.71 | 0.71 | 0.67 | 1000 | +| pass | GET | /api/notifications | 1 | 0.00% | 0.41 | 0.41 | 0.41 | 0.38 | 1000 | +| pass | POST | /api/notifications | 1 | 0.00% | 1.02 | 1.02 | 1.02 | 0.98 | 979.19 | +| pass | POST | /api/uploads | 1 | 0.00% | 5.29 | 5.29 | 5.29 | 5.24 | 188.74 | +| pass | GET | /api/search?q=benchmark | 1 | 0.00% | 0.87 | 0.87 | 0.87 | 0.83 | 1000 | +| pass | GET | /api/admin/metrics | 1 | 0.00% | 0.87 | 0.87 | 0.87 | 0.84 | 1000 | diff --git a/benchmarks/run-benchmarks.mjs b/benchmarks/run-benchmarks.mjs new file mode 100644 index 000000000..f3d5a428a --- /dev/null +++ b/benchmarks/run-benchmarks.mjs @@ -0,0 +1,264 @@ +import { mkdir, readFile, writeFile } from "node:fs/promises"; +import { createServer } from "node:http"; +import { dirname, resolve } from "node:path"; +import { fileURLToPath } from "node:url"; +import { benchmarkEndpoints } from "./endpoints.mjs"; + +const rootDir = resolve(dirname(fileURLToPath(import.meta.url)), ".."); +const args = new Set(process.argv.slice(2)); +const isSmoke = args.has("--smoke"); +const outputDir = resolve(rootDir, process.env.BENCHMARK_OUTPUT_DIR ?? "benchmarks/results"); +const concurrency = Number(process.env.BENCHMARK_CONCURRENCY ?? (isSmoke ? 1 : 2)); +const requestsPerEndpoint = Number(process.env.BENCHMARK_REQUESTS_PER_ENDPOINT ?? (isSmoke ? 1 : 8)); + +const thresholds = JSON.parse( + await readFile(resolve(rootDir, "benchmarks/thresholds.json"), "utf8") +); + +function percentile(values, rank) { + if (values.length === 0) return 0; + const sorted = [...values].sort((a, b) => a - b); + const index = Math.ceil((rank / 100) * sorted.length) - 1; + return Number(sorted[Math.max(0, Math.min(index, sorted.length - 1))].toFixed(2)); +} + +function mean(values) { + if (values.length === 0) return 0; + return Number((values.reduce((sum, value) => sum + value, 0) / values.length).toFixed(2)); +} + +function routeKey(endpoint) { + return `${endpoint.method} ${endpoint.path.split("?")[0]}`; +} + +function resolveThreshold(endpoint) { + return { + ...thresholds.defaults, + ...(thresholds.routes?.[routeKey(endpoint)] ?? {}) + }; +} + +async function createLocalTarget() { + if (process.env.BENCHMARK_TARGET_URL) { + return { + baseUrl: process.env.BENCHMARK_TARGET_URL.replace(/\/$/, ""), + close: async () => {} + }; + } + + process.env.NODE_ENV ??= "benchmark"; + const [{ createApp }, { signAccessToken }] = await Promise.all([ + import("../apps/api/src/app.js"), + import("../apps/api/src/utils/jwt.js") + ]); + + if (!process.env.BENCHMARK_ADMIN_TOKEN) { + process.env.BENCHMARK_ADMIN_TOKEN = signAccessToken({ + sub: "benchmark-admin", + role: "admin" + }); + } + + const server = createServer(createApp()); + await new Promise((resolveListen) => server.listen(0, "127.0.0.1", resolveListen)); + const { port } = server.address(); + + return { + baseUrl: `http://127.0.0.1:${port}`, + close: () => new Promise((resolveClose, rejectClose) => { + server.close((error) => error ? rejectClose(error) : resolveClose()); + }) + }; +} + +function buildRequest(endpoint, iteration) { + const headers = {}; + const request = { + method: endpoint.method, + headers + }; + + if (endpoint.auth) { + headers.authorization = `Bearer ${process.env.BENCHMARK_ADMIN_TOKEN ?? ""}`; + } + + if (endpoint.json) { + headers["content-type"] = "application/json"; + const payload = typeof endpoint.json === "function" + ? endpoint.json({ iteration }) + : endpoint.json; + request.body = JSON.stringify(payload); + } + + if (endpoint.multipart) { + const form = new FormData(); + form.append( + endpoint.multipart.fieldName, + new Blob([endpoint.multipart.body], { type: endpoint.multipart.type }), + endpoint.multipart.filename + ); + request.body = form; + } + + return request; +} + +async function runOne(baseUrl, endpoint, iteration) { + const startedAt = performance.now(); + let headersAt = startedAt; + let status = 0; + let ok = false; + let error = null; + + try { + const response = await fetch(`${baseUrl}${endpoint.path}`, buildRequest(endpoint, iteration)); + headersAt = performance.now(); + status = response.status; + await response.arrayBuffer(); + ok = status === endpoint.expectedStatus; + } catch (requestError) { + error = requestError instanceof Error ? requestError.message : String(requestError); + } + + const endedAt = performance.now(); + return { + ok, + status, + error, + latencyMs: Number((endedAt - startedAt).toFixed(2)), + ttfbMs: Number((headersAt - startedAt).toFixed(2)) + }; +} + +async function runEndpoint(baseUrl, endpoint) { + const measurements = []; + let nextIteration = 0; + const startedAt = performance.now(); + + async function worker() { + while (nextIteration < requestsPerEndpoint) { + const iteration = nextIteration; + nextIteration += 1; + measurements.push(await runOne(baseUrl, endpoint, iteration)); + } + } + + await Promise.all(Array.from({ length: Math.min(concurrency, requestsPerEndpoint) }, worker)); + + const durationSeconds = Math.max((performance.now() - startedAt) / 1000, 0.001); + const latencies = measurements.map((sample) => sample.latencyMs); + const ttfbs = measurements.map((sample) => sample.ttfbMs); + const errors = measurements.filter((sample) => !sample.ok); + const threshold = resolveThreshold(endpoint); + + const result = { + name: endpoint.name, + method: endpoint.method, + path: endpoint.path, + expectedStatus: endpoint.expectedStatus, + requests: measurements.length, + errors: errors.length, + errorRate: Number((errors.length / measurements.length).toFixed(4)), + requestsPerSecond: Number((measurements.length / durationSeconds).toFixed(2)), + peakRequestsPerSecond: Number((concurrency / Math.max(Math.min(...latencies) / 1000, 0.001)).toFixed(2)), + latencyMs: { + mean: mean(latencies), + p50: percentile(latencies, 50), + p95: percentile(latencies, 95), + p99: percentile(latencies, 99) + }, + ttfbMs: { + mean: mean(ttfbs), + p50: percentile(ttfbs, 50), + p95: percentile(ttfbs, 95), + p99: percentile(ttfbs, 99) + }, + threshold, + passedThresholds: + errors.length / measurements.length <= threshold.errorRate && + percentile(latencies, 99) <= threshold.p99LatencyMs && + percentile(ttfbs, 99) <= threshold.p99TtfbMs, + failures: errors.slice(0, 3) + }; + + return result; +} + +function markdownReport(report) { + const rows = report.results.map((result) => [ + result.passedThresholds ? "pass" : "fail", + result.method, + result.path, + result.requests, + `${(result.errorRate * 100).toFixed(2)}%`, + result.latencyMs.p50, + result.latencyMs.p95, + result.latencyMs.p99, + result.ttfbMs.p99, + result.requestsPerSecond + ]); + + return [ + "# API Benchmark Report", + "", + `- Mode: ${report.mode}`, + `- Target: ${report.target}`, + `- Generated: ${report.generatedAt}`, + `- Endpoints: ${report.summary.endpointCount}`, + `- Requests: ${report.summary.totalRequests}`, + `- Error rate: ${(report.summary.errorRate * 100).toFixed(2)}%`, + `- Max p99 latency: ${report.summary.maxP99LatencyMs} ms`, + `- Max p99 TTFB: ${report.summary.maxP99TtfbMs} ms`, + "", + "| Gate | Method | Path | Requests | Error rate | p50 ms | p95 ms | p99 ms | p99 TTFB ms | RPS |", + "| --- | --- | --- | ---: | ---: | ---: | ---: | ---: | ---: | ---: |", + ...rows.map((row) => `| ${row.join(" | ")} |`), + "" + ].join("\n"); +} + +const target = await createLocalTarget(); + +try { + const results = []; + for (const endpoint of benchmarkEndpoints) { + results.push(await runEndpoint(target.baseUrl, endpoint)); + } + + const totalRequests = results.reduce((sum, result) => sum + result.requests, 0); + const totalErrors = results.reduce((sum, result) => sum + result.errors, 0); + const report = { + mode: isSmoke ? "smoke" : "full", + target: target.baseUrl, + generatedAt: new Date().toISOString(), + configuration: { + concurrency, + requestsPerEndpoint, + thresholdsFile: "benchmarks/thresholds.json" + }, + summary: { + endpointCount: results.length, + totalRequests, + totalErrors, + errorRate: Number((totalErrors / totalRequests).toFixed(4)), + maxP99LatencyMs: Math.max(...results.map((result) => result.latencyMs.p99)), + maxP99TtfbMs: Math.max(...results.map((result) => result.ttfbMs.p99)), + passed: results.every((result) => result.passedThresholds) + }, + results + }; + + await mkdir(outputDir, { recursive: true }); + const suffix = isSmoke ? "smoke" : "full"; + const jsonPath = resolve(outputDir, `${suffix}-latest.json`); + const markdownPath = resolve(outputDir, `${suffix}-latest.md`); + await writeFile(jsonPath, `${JSON.stringify(report, null, 2)}\n`); + await writeFile(markdownPath, markdownReport(report)); + + console.log(markdownReport(report)); + if (!report.summary.passed) { + process.exitCode = 1; + } +} finally { + await target.close(); +} diff --git a/benchmarks/thresholds.json b/benchmarks/thresholds.json new file mode 100644 index 000000000..846d68f4c --- /dev/null +++ b/benchmarks/thresholds.json @@ -0,0 +1,14 @@ +{ + "defaults": { + "p99LatencyMs": 1000, + "p99TtfbMs": 1000, + "errorRate": 0 + }, + "routes": { + "POST /api/uploads": { + "p99LatencyMs": 1500, + "p99TtfbMs": 1500, + "errorRate": 0 + } + } +} diff --git a/benchmarks/verify-endpoint-coverage.mjs b/benchmarks/verify-endpoint-coverage.mjs new file mode 100644 index 000000000..ecafa457e --- /dev/null +++ b/benchmarks/verify-endpoint-coverage.mjs @@ -0,0 +1,33 @@ +import assert from "node:assert/strict"; +import { readFile } from "node:fs/promises"; +import { resolve } from "node:path"; +import { benchmarkEndpoints } from "./endpoints.mjs"; + +const rootDir = resolve(import.meta.dirname, ".."); +const appSource = await readFile(resolve(rootDir, "apps/api/src/app.js"), "utf8"); + +const mountedApiPrefixes = [...appSource.matchAll(/app\.use\("([^"]+)"/g)] + .map((match) => match[1]) + .filter((path) => path.startsWith("/api/")); + +const endpointPaths = benchmarkEndpoints.map((endpoint) => endpoint.path.split("?")[0]); + +for (const prefix of mountedApiPrefixes) { + assert( + endpointPaths.some((path) => path === prefix || path.startsWith(`${prefix}/`)), + `Missing benchmark endpoint for mounted API prefix ${prefix}` + ); +} + +assert( + endpointPaths.includes("/health"), + "Missing benchmark endpoint for /health" +); + +const duplicateKeys = benchmarkEndpoints + .map((endpoint) => `${endpoint.method} ${endpoint.path}`) + .filter((key, index, all) => all.indexOf(key) !== index); + +assert.deepEqual(duplicateKeys, [], "Benchmark endpoint list contains duplicate method/path entries"); + +console.log(`Verified benchmark coverage for ${mountedApiPrefixes.length} API route prefixes and /health.`); diff --git a/demos/api-benchmark-demo.mp4 b/demos/api-benchmark-demo.mp4 new file mode 100644 index 000000000..04c3de3c1 Binary files /dev/null and b/demos/api-benchmark-demo.mp4 differ diff --git a/package.json b/package.json index 675e6e69d..ea26655ab 100644 --- a/package.json +++ b/package.json @@ -7,6 +7,9 @@ ], "scripts": { "build": "echo \"Run package-specific builds (e.g. npm run build -w apps/web)\"", + "benchmark": "node benchmarks/run-benchmarks.mjs", + "benchmark:coverage": "node benchmarks/verify-endpoint-coverage.mjs", + "benchmark:smoke": "node benchmarks/run-benchmarks.mjs --smoke", "lint": "echo \"No root lint configured\"", "test": "npm run test -w apps/api" }