diff --git a/.github/workflows/benchmark-smoke.yml b/.github/workflows/benchmark-smoke.yml new file mode 100644 index 000000000..0458da938 --- /dev/null +++ b/.github/workflows/benchmark-smoke.yml @@ -0,0 +1,26 @@ +name: API Benchmark Smoke + +on: + pull_request: + paths: + - "apps/api/**" + - "benchmarks/**" + - "package*.json" + - ".github/workflows/benchmark-smoke.yml" + +jobs: + api-benchmark-smoke: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-node@v4 + with: + node-version: 22 + cache: npm + - run: npm ci + - run: npm run benchmark:smoke + - uses: actions/upload-artifact@v4 + if: always() + with: + name: api-benchmark-results + path: benchmarks/results/ diff --git a/.gitignore b/.gitignore index 1e3ce10dd..9bdf6b263 100644 --- a/.gitignore +++ b/.gitignore @@ -3,5 +3,6 @@ node_modules dist .env .env.* +!benchmarks/.env.benchmark.example coverage *.log diff --git a/apps/api/package.json b/apps/api/package.json index 25fa0e90e..17c3c1a6e 100644 --- a/apps/api/package.json +++ b/apps/api/package.json @@ -5,7 +5,7 @@ "scripts": { "dev": "node src/server.js", "start": "node src/server.js", - "test": "node --test src/tests" + "test": "node --test \"src/tests/**/*.test.js\"" }, "dependencies": { "cors": "^2.8.5", diff --git a/benchmarks/.env.benchmark.example b/benchmarks/.env.benchmark.example new file mode 100644 index 000000000..6e12bef79 --- /dev/null +++ b/benchmarks/.env.benchmark.example @@ -0,0 +1,10 @@ +# Optional. When unset, the benchmark runner starts the local Express app. +BENCHMARK_BASE_URL=http://127.0.0.1:4000 + +# Optional for protected routes. When unset for the local app, a benchmark JWT +# is generated with the development JWT secret. +BENCHMARK_AUTH_TOKEN= + +# Tunables for local runs. +BENCHMARK_REQUESTS=5 +BENCHMARK_CONCURRENCY=2 diff --git a/benchmarks/README.md b/benchmarks/README.md new file mode 100644 index 000000000..9c195c42e --- /dev/null +++ b/benchmarks/README.md @@ -0,0 +1,47 @@ +# API Benchmarks + +This directory contains a reproducible benchmark suite for the platform API. + +## Commands + +```console +npm run benchmark +npm run benchmark:smoke +``` + +By default, the runner starts the local Express app on a random port and benchmarks every `/api/` endpoint. To run against an already running local or staging server, set `BENCHMARK_BASE_URL`. + +## Reviewer Demo + +Use `benchmark:smoke` for a short end-to-end review pass: + +```console +npm ci +npm test +npm run benchmark:smoke +sed -n '1,80p' benchmarks/results/api-benchmark-latest.md +``` + +The smoke run starts the local API automatically, exercises all configured routes once, checks the thresholds in `thresholds.json`, and writes the JSON and Markdown reports under `benchmarks/results/`. See `demo-walkthrough.md` for a recording-ready walkthrough. + +## Environment + +Copy `.env.benchmark.example` into your local environment or CI secret store and set: + +- `BENCHMARK_BASE_URL`: target server URL. If omitted, the local app is started automatically. +- `BENCHMARK_AUTH_TOKEN`: bearer token for protected routes. If omitted for a local run, a benchmark JWT is generated. +- `BENCHMARK_REQUESTS`: requests per endpoint. Defaults to `5`, or `1` in smoke mode. +- `BENCHMARK_CONCURRENCY`: concurrent requests per endpoint. Defaults to `2`, or `1` in smoke mode. + +## Output + +Each run writes: + +- `benchmarks/results/api-benchmark-latest.json` +- `benchmarks/results/api-benchmark-latest.md` + +The JSON result is machine-readable for regression tracking. The Markdown summary is designed to be pasted into PR descriptions. + +## Thresholds + +`thresholds.json` stores reviewable smoke-test gates. The runner fails when an endpoint exceeds its p99 latency threshold or error-rate threshold. diff --git a/benchmarks/demo-walkthrough.md b/benchmarks/demo-walkthrough.md new file mode 100644 index 000000000..646232f1b --- /dev/null +++ b/benchmarks/demo-walkthrough.md @@ -0,0 +1,36 @@ +# API Benchmark Demo Walkthrough + +This is a short, recording-ready walkthrough for reviewers to verify the benchmark suite added for bounty issue #30. + +## Goal + +Show that the benchmark runner can start the local API, exercise the platform endpoints, enforce latency/error thresholds, and publish reviewable benchmark output. + +## Demo Steps + +```console +npm ci +npm test +npm run benchmark:smoke +sed -n '1,80p' benchmarks/results/api-benchmark-latest.md +``` + +## Expected Result + +The smoke run should complete successfully and write: + +- `benchmarks/results/api-benchmark-latest.json` +- `benchmarks/results/api-benchmark-latest.md` + +The committed smoke output covers 20 API routes with 1 request per endpoint, 1 concurrent request per endpoint, and a passing threshold result. + +## What To Check + +- `benchmarks/routes.mjs` lists the route inventory and request payloads. +- `benchmarks/thresholds.json` defines reviewable p99 latency and error-rate gates. +- `benchmarks/run-api-benchmarks.mjs` starts a local app when `BENCHMARK_BASE_URL` is not set, or benchmarks an external target when it is set. +- `.github/workflows/benchmark-smoke.yml` runs the smoke benchmark on relevant pull requests and uploads `benchmarks/results/` as an artifact. + +## CI Artifact + +The `API Benchmark Smoke` workflow uploads the generated benchmark report as `api-benchmark-results`, so maintainers can inspect the exact JSON and Markdown generated by the pull request run. diff --git a/benchmarks/results/.gitkeep b/benchmarks/results/.gitkeep new file mode 100644 index 000000000..8b1378917 --- /dev/null +++ b/benchmarks/results/.gitkeep @@ -0,0 +1 @@ + diff --git a/benchmarks/results/api-benchmark-latest.json b/benchmarks/results/api-benchmark-latest.json new file mode 100644 index 000000000..e1c9470e0 --- /dev/null +++ b/benchmarks/results/api-benchmark-latest.json @@ -0,0 +1,552 @@ +{ + "generatedAt": "2026-05-25T09:15:48.370Z", + "mode": "smoke", + "target": "local", + "config": { + "requestCount": 1, + "concurrency": 1 + }, + "environment": { + "node": "v26.0.0", + "os": "Darwin 25.4.0 arm64", + "cpu": "Apple M5", + "cpuCount": 10, + "totalMemoryMb": 16384, + "freeMemoryMb": 143 + }, + "thresholds": { + "defaults": { + "p99Ms": 1000, + "errorRatePercent": 0 + }, + "routes": { + "uploads.create": { + "p99Ms": 1500, + "errorRatePercent": 0 + } + } + }, + "thresholdFailures": [], + "results": [ + { + "name": "auth.register", + "method": "POST", + "path": "/api/auth/register", + "requests": 1, + "successful": 1, + "errors": 0, + "errorRatePercent": 0, + "latencyMs": { + "p50": 15.51, + "p95": 15.51, + "p99": 15.51 + }, + "ttfbMs": { + "p50": 15.1, + "p95": 15.1, + "p99": 15.1 + }, + "rps": { + "sustained": 64.29, + "peak": 64.46 + }, + "statuses": { + "201": 1 + } + }, + { + "name": "auth.login", + "method": "POST", + "path": "/api/auth/login", + "requests": 1, + "successful": 1, + "errors": 0, + "errorRatePercent": 0, + "latencyMs": { + "p50": 1.59, + "p95": 1.59, + "p99": 1.59 + }, + "ttfbMs": { + "p50": 1.53, + "p95": 1.53, + "p99": 1.53 + }, + "rps": { + "sustained": 628.29, + "peak": 630.82 + }, + "statuses": { + "200": 1 + } + }, + { + "name": "auth.oauthCallback", + "method": "GET", + "path": "/api/auth/oauth/github/callback", + "requests": 1, + "successful": 1, + "errors": 0, + "errorRatePercent": 0, + "latencyMs": { + "p50": 0.82, + "p95": 0.82, + "p99": 0.82 + }, + "ttfbMs": { + "p50": 0.76, + "p95": 0.76, + "p99": 0.76 + }, + "rps": { + "sustained": 1213.9, + "peak": 1220.88 + }, + "statuses": { + "200": 1 + } + }, + { + "name": "auth.refresh", + "method": "POST", + "path": "/api/auth/refresh", + "requests": 1, + "successful": 1, + "errors": 0, + "errorRatePercent": 0, + "latencyMs": { + "p50": 0.57, + "p95": 0.57, + "p99": 0.57 + }, + "ttfbMs": { + "p50": 0.53, + "p95": 0.53, + "p99": 0.53 + }, + "rps": { + "sustained": 1744.56, + "peak": 1754.39 + }, + "statuses": { + "200": 1 + } + }, + { + "name": "users.list", + "method": "GET", + "path": "/api/users", + "requests": 1, + "successful": 1, + "errors": 0, + "errorRatePercent": 0, + "latencyMs": { + "p50": 0.46, + "p95": 0.46, + "p99": 0.46 + }, + "ttfbMs": { + "p50": 0.41, + "p95": 0.41, + "p99": 0.41 + }, + "rps": { + "sustained": 2156.72, + "peak": 2171.75 + }, + "statuses": { + "200": 1 + } + }, + { + "name": "users.create", + "method": "POST", + "path": "/api/users", + "requests": 1, + "successful": 1, + "errors": 0, + "errorRatePercent": 0, + "latencyMs": { + "p50": 0.59, + "p95": 0.59, + "p99": 0.59 + }, + "ttfbMs": { + "p50": 0.55, + "p95": 0.55, + "p99": 0.55 + }, + "rps": { + "sustained": 1684.45, + "peak": 1692.88 + }, + "statuses": { + "201": 1 + } + }, + { + "name": "jobs.list", + "method": "GET", + "path": "/api/jobs", + "requests": 1, + "successful": 1, + "errors": 0, + "errorRatePercent": 0, + "latencyMs": { + "p50": 0.41, + "p95": 0.41, + "p99": 0.41 + }, + "ttfbMs": { + "p50": 0.36, + "p95": 0.36, + "p99": 0.36 + }, + "rps": { + "sustained": 2434.32, + "peak": 2457 + }, + "statuses": { + "200": 1 + } + }, + { + "name": "jobs.create", + "method": "POST", + "path": "/api/jobs", + "requests": 1, + "successful": 1, + "errors": 0, + "errorRatePercent": 0, + "latencyMs": { + "p50": 0.63, + "p95": 0.63, + "p99": 0.63 + }, + "ttfbMs": { + "p50": 0.59, + "p95": 0.59, + "p99": 0.59 + }, + "rps": { + "sustained": 1593.31, + "peak": 1599.9 + }, + "statuses": { + "201": 1 + } + }, + { + "name": "proposals.list", + "method": "GET", + "path": "/api/proposals", + "requests": 1, + "successful": 1, + "errors": 0, + "errorRatePercent": 0, + "latencyMs": { + "p50": 0.41, + "p95": 0.41, + "p99": 0.41 + }, + "ttfbMs": { + "p50": 0.37, + "p95": 0.37, + "p99": 0.37 + }, + "rps": { + "sustained": 2422.04, + "peak": 2437.29 + }, + "statuses": { + "200": 1 + } + }, + { + "name": "proposals.create", + "method": "POST", + "path": "/api/proposals", + "requests": 1, + "successful": 1, + "errors": 0, + "errorRatePercent": 0, + "latencyMs": { + "p50": 0.76, + "p95": 0.76, + "p99": 0.76 + }, + "ttfbMs": { + "p50": 0.71, + "p95": 0.71, + "p99": 0.71 + }, + "rps": { + "sustained": 1316.44, + "peak": 1323.19 + }, + "statuses": { + "201": 1 + } + }, + { + "name": "payments.create", + "method": "POST", + "path": "/api/payments", + "requests": 1, + "successful": 1, + "errors": 0, + "errorRatePercent": 0, + "latencyMs": { + "p50": 0.57, + "p95": 0.57, + "p99": 0.57 + }, + "ttfbMs": { + "p50": 0.54, + "p95": 0.54, + "p99": 0.54 + }, + "rps": { + "sustained": 1749.78, + "peak": 1759.4 + }, + "statuses": { + "201": 1 + } + }, + { + "name": "reviews.list", + "method": "GET", + "path": "/api/reviews", + "requests": 1, + "successful": 1, + "errors": 0, + "errorRatePercent": 0, + "latencyMs": { + "p50": 0.36, + "p95": 0.36, + "p99": 0.36 + }, + "ttfbMs": { + "p50": 0.33, + "p95": 0.33, + "p99": 0.33 + }, + "rps": { + "sustained": 2763.06, + "peak": 2799.48 + }, + "statuses": { + "200": 1 + } + }, + { + "name": "reviews.create", + "method": "POST", + "path": "/api/reviews", + "requests": 1, + "successful": 1, + "errors": 0, + "errorRatePercent": 0, + "latencyMs": { + "p50": 0.43, + "p95": 0.43, + "p99": 0.43 + }, + "ttfbMs": { + "p50": 0.4, + "p95": 0.4, + "p99": 0.4 + }, + "rps": { + "sustained": 2312.36, + "peak": 2333.72 + }, + "statuses": { + "201": 1 + } + }, + { + "name": "messages.list", + "method": "GET", + "path": "/api/messages", + "requests": 1, + "successful": 1, + "errors": 0, + "errorRatePercent": 0, + "latencyMs": { + "p50": 0.25, + "p95": 0.25, + "p99": 0.25 + }, + "ttfbMs": { + "p50": 0.22, + "p95": 0.22, + "p99": 0.22 + }, + "rps": { + "sustained": 4006.01, + "peak": 4039.03 + }, + "statuses": { + "200": 1 + } + }, + { + "name": "messages.create", + "method": "POST", + "path": "/api/messages", + "requests": 1, + "successful": 1, + "errors": 0, + "errorRatePercent": 0, + "latencyMs": { + "p50": 0.37, + "p95": 0.37, + "p99": 0.37 + }, + "ttfbMs": { + "p50": 0.34, + "p95": 0.34, + "p99": 0.34 + }, + "rps": { + "sustained": 2726.34, + "peak": 2736.61 + }, + "statuses": { + "201": 1 + } + }, + { + "name": "notifications.list", + "method": "GET", + "path": "/api/notifications", + "requests": 1, + "successful": 1, + "errors": 0, + "errorRatePercent": 0, + "latencyMs": { + "p50": 0.24, + "p95": 0.24, + "p99": 0.24 + }, + "ttfbMs": { + "p50": 0.22, + "p95": 0.22, + "p99": 0.22 + }, + "rps": { + "sustained": 4079.55, + "peak": 4097.66 + }, + "statuses": { + "200": 1 + } + }, + { + "name": "notifications.create", + "method": "POST", + "path": "/api/notifications", + "requests": 1, + "successful": 1, + "errors": 0, + "errorRatePercent": 0, + "latencyMs": { + "p50": 0.37, + "p95": 0.37, + "p99": 0.37 + }, + "ttfbMs": { + "p50": 0.35, + "p95": 0.35, + "p99": 0.35 + }, + "rps": { + "sustained": 2693.6, + "peak": 2708.81 + }, + "statuses": { + "201": 1 + } + }, + { + "name": "uploads.create", + "method": "POST", + "path": "/api/uploads", + "requests": 1, + "successful": 1, + "errors": 0, + "errorRatePercent": 0, + "latencyMs": { + "p50": 2.72, + "p95": 2.72, + "p99": 2.72 + }, + "ttfbMs": { + "p50": 2.7, + "p95": 2.7, + "p99": 2.7 + }, + "rps": { + "sustained": 367.08, + "peak": 367.29 + }, + "statuses": { + "201": 1 + } + }, + { + "name": "search.query", + "method": "GET", + "path": "/api/search?q=benchmark", + "requests": 1, + "successful": 1, + "errors": 0, + "errorRatePercent": 0, + "latencyMs": { + "p50": 0.61, + "p95": 0.61, + "p99": 0.61 + }, + "ttfbMs": { + "p50": 0.58, + "p95": 0.58, + "p99": 0.58 + }, + "rps": { + "sustained": 1642.6, + "peak": 1645.75 + }, + "statuses": { + "200": 1 + } + }, + { + "name": "admin.metrics", + "method": "GET", + "path": "/api/admin/metrics", + "requests": 1, + "successful": 1, + "errors": 0, + "errorRatePercent": 0, + "latencyMs": { + "p50": 0.63, + "p95": 0.63, + "p99": 0.63 + }, + "ttfbMs": { + "p50": 0.61, + "p95": 0.61, + "p99": 0.61 + }, + "rps": { + "sustained": 1572.22, + "peak": 1575.32 + }, + "statuses": { + "200": 1 + } + } + ] +} diff --git a/benchmarks/results/api-benchmark-latest.md b/benchmarks/results/api-benchmark-latest.md new file mode 100644 index 000000000..6c9b64214 --- /dev/null +++ b/benchmarks/results/api-benchmark-latest.md @@ -0,0 +1,35 @@ +# API Benchmark Summary + +- Mode: smoke +- Target: local +- Routes covered: 20 +- Requests per endpoint: 1 +- Concurrency per endpoint: 1 +- Runtime: v26.0.0 +- OS: Darwin 25.4.0 arm64 +- Threshold result: passed + +| Route | Endpoint | Requests | p50 ms | p95 ms | p99 ms | p95 TTFB ms | Sustained RPS | Peak RPS | Error % | +| --- | --- | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | +| auth.register | POST /api/auth/register | 1 | 15.51 | 15.51 | 15.51 | 15.1 | 64.29 | 64.46 | 0 | +| auth.login | POST /api/auth/login | 1 | 1.59 | 1.59 | 1.59 | 1.53 | 628.29 | 630.82 | 0 | +| auth.oauthCallback | GET /api/auth/oauth/github/callback | 1 | 0.82 | 0.82 | 0.82 | 0.76 | 1213.9 | 1220.88 | 0 | +| auth.refresh | POST /api/auth/refresh | 1 | 0.57 | 0.57 | 0.57 | 0.53 | 1744.56 | 1754.39 | 0 | +| users.list | GET /api/users | 1 | 0.46 | 0.46 | 0.46 | 0.41 | 2156.72 | 2171.75 | 0 | +| users.create | POST /api/users | 1 | 0.59 | 0.59 | 0.59 | 0.55 | 1684.45 | 1692.88 | 0 | +| jobs.list | GET /api/jobs | 1 | 0.41 | 0.41 | 0.41 | 0.36 | 2434.32 | 2457 | 0 | +| jobs.create | POST /api/jobs | 1 | 0.63 | 0.63 | 0.63 | 0.59 | 1593.31 | 1599.9 | 0 | +| proposals.list | GET /api/proposals | 1 | 0.41 | 0.41 | 0.41 | 0.37 | 2422.04 | 2437.29 | 0 | +| proposals.create | POST /api/proposals | 1 | 0.76 | 0.76 | 0.76 | 0.71 | 1316.44 | 1323.19 | 0 | +| payments.create | POST /api/payments | 1 | 0.57 | 0.57 | 0.57 | 0.54 | 1749.78 | 1759.4 | 0 | +| reviews.list | GET /api/reviews | 1 | 0.36 | 0.36 | 0.36 | 0.33 | 2763.06 | 2799.48 | 0 | +| reviews.create | POST /api/reviews | 1 | 0.43 | 0.43 | 0.43 | 0.4 | 2312.36 | 2333.72 | 0 | +| messages.list | GET /api/messages | 1 | 0.25 | 0.25 | 0.25 | 0.22 | 4006.01 | 4039.03 | 0 | +| messages.create | POST /api/messages | 1 | 0.37 | 0.37 | 0.37 | 0.34 | 2726.34 | 2736.61 | 0 | +| notifications.list | GET /api/notifications | 1 | 0.24 | 0.24 | 0.24 | 0.22 | 4079.55 | 4097.66 | 0 | +| notifications.create | POST /api/notifications | 1 | 0.37 | 0.37 | 0.37 | 0.35 | 2693.6 | 2708.81 | 0 | +| uploads.create | POST /api/uploads | 1 | 2.72 | 2.72 | 2.72 | 2.7 | 367.08 | 367.29 | 0 | +| search.query | GET /api/search?q=benchmark | 1 | 0.61 | 0.61 | 0.61 | 0.58 | 1642.6 | 1645.75 | 0 | +| admin.metrics | GET /api/admin/metrics | 1 | 0.63 | 0.63 | 0.63 | 0.61 | 1572.22 | 1575.32 | 0 | + + diff --git a/benchmarks/routes.mjs b/benchmarks/routes.mjs new file mode 100644 index 000000000..4e8641b79 --- /dev/null +++ b/benchmarks/routes.mjs @@ -0,0 +1,167 @@ +export const benchmarkRoutes = [ + { + name: "auth.register", + method: "POST", + path: "/api/auth/register", + payload: () => ({ + email: `benchmark-${Date.now()}-${Math.random().toString(16).slice(2)}@example.com`, + password: "benchmark-password", + role: "client" + }) + }, + { + name: "auth.login", + method: "POST", + path: "/api/auth/login", + payload: () => ({ + email: "benchmark@example.com", + password: "benchmark-password" + }) + }, + { + name: "auth.oauthCallback", + method: "GET", + path: "/api/auth/oauth/github/callback" + }, + { + name: "auth.refresh", + method: "POST", + path: "/api/auth/refresh" + }, + { + name: "users.list", + method: "GET", + path: "/api/users" + }, + { + name: "users.create", + method: "POST", + path: "/api/users", + payload: () => ({ + email: `freelancer-${Date.now()}@example.com`, + name: "Benchmark Freelancer", + role: "freelancer", + skills: ["node", "react", "api-performance"], + hourlyRate: 75 + }) + }, + { + name: "jobs.list", + method: "GET", + path: "/api/jobs" + }, + { + name: "jobs.create", + method: "POST", + path: "/api/jobs", + payload: () => ({ + title: "Benchmark API Suite", + description: "Create and maintain reproducible API performance benchmarks.", + budgetMin: 500, + budgetMax: 1500, + categoryId: "performance", + skills: ["node", "express", "benchmarking"] + }) + }, + { + name: "proposals.list", + method: "GET", + path: "/api/proposals" + }, + { + name: "proposals.create", + method: "POST", + path: "/api/proposals", + payload: () => ({ + jobId: "job_benchmark", + freelancerId: "usr_benchmark", + coverLetter: "I can deliver a documented API benchmark suite.", + proposedRate: 900, + estimatedDays: 3 + }) + }, + { + name: "payments.create", + method: "POST", + path: "/api/payments", + payload: () => ({ + amount: 750, + currency: "usd", + jobId: "job_benchmark", + clientId: "usr_client", + freelancerId: "usr_freelancer" + }) + }, + { + name: "reviews.list", + method: "GET", + path: "/api/reviews" + }, + { + name: "reviews.create", + method: "POST", + path: "/api/reviews", + payload: () => ({ + jobId: "job_benchmark", + reviewerId: "usr_client", + revieweeId: "usr_freelancer", + rating: 5, + comment: "Fast delivery and clear communication." + }) + }, + { + name: "messages.list", + method: "GET", + path: "/api/messages" + }, + { + name: "messages.create", + method: "POST", + path: "/api/messages", + payload: () => ({ + senderId: "usr_client", + recipientId: "usr_freelancer", + body: "Can you share the benchmark report?" + }) + }, + { + name: "notifications.list", + method: "GET", + path: "/api/notifications" + }, + { + name: "notifications.create", + method: "POST", + path: "/api/notifications", + payload: () => ({ + userId: "usr_client", + type: "benchmark.completed", + message: "API benchmark suite finished successfully." + }) + }, + { + name: "uploads.create", + method: "POST", + path: "/api/uploads", + multipart: () => { + const form = new FormData(); + form.set( + "file", + new Blob(["benchmark upload payload"], { type: "text/plain" }), + "benchmark.txt" + ); + return form; + } + }, + { + name: "search.query", + method: "GET", + path: "/api/search?q=benchmark" + }, + { + name: "admin.metrics", + method: "GET", + path: "/api/admin/metrics", + protected: true + } +]; diff --git a/benchmarks/run-api-benchmarks.mjs b/benchmarks/run-api-benchmarks.mjs new file mode 100644 index 000000000..226e7d6e0 --- /dev/null +++ b/benchmarks/run-api-benchmarks.mjs @@ -0,0 +1,285 @@ +import fs from "node:fs/promises"; +import http from "node:http"; +import os from "node:os"; +import path from "node:path"; +import { performance } from "node:perf_hooks"; +import { fileURLToPath } from "node:url"; + +import { createApp } from "../apps/api/src/app.js"; +import { signAccessToken } from "../apps/api/src/utils/jwt.js"; +import { benchmarkRoutes } from "./routes.mjs"; + +const __dirname = path.dirname(fileURLToPath(import.meta.url)); +const resultsDir = path.join(__dirname, "results"); +const thresholdsPath = path.join(__dirname, "thresholds.json"); + +const isSmoke = process.argv.includes("--smoke"); +const requestCount = Number(process.env.BENCHMARK_REQUESTS ?? (isSmoke ? 1 : 5)); +const concurrency = Number(process.env.BENCHMARK_CONCURRENCY ?? (isSmoke ? 1 : 2)); + +function percentile(values, rank) { + if (values.length === 0) { + return 0; + } + + const sorted = [...values].sort((a, b) => a - b); + const index = Math.ceil((rank / 100) * sorted.length) - 1; + return sorted[Math.min(Math.max(index, 0), sorted.length - 1)]; +} + +function round(value) { + return Number(value.toFixed(2)); +} + +function buildHeaders(route, authToken) { + const headers = { + "user-agent": "freelance-platform-api-benchmark/1.0" + }; + + if (route.protected || authToken) { + headers.authorization = `Bearer ${authToken}`; + } + + if (!route.multipart && route.payload) { + headers["content-type"] = "application/json"; + } + + return headers; +} + +function buildRequest(route, authToken) { + const request = { + method: route.method, + headers: buildHeaders(route, authToken) + }; + + if (route.multipart) { + request.body = route.multipart(); + return request; + } + + if (route.payload) { + request.body = JSON.stringify(route.payload()); + } + + return request; +} + +async function startLocalApp() { + const server = http.createServer(createApp()); + await new Promise((resolve, reject) => { + server.once("listening", resolve); + server.once("error", reject); + server.listen(0, "127.0.0.1"); + }); + + const { port } = server.address(); + return { + baseUrl: `http://127.0.0.1:${port}`, + close: () => new Promise((resolve, reject) => { + server.close((error) => (error ? reject(error) : resolve())); + }) + }; +} + +async function timeRequest(baseUrl, route, authToken) { + const startedAt = performance.now(); + let response; + + try { + response = await fetch(new URL(route.path, baseUrl), buildRequest(route, authToken)); + const headersAt = performance.now(); + await response.arrayBuffer(); + const completedAt = performance.now(); + + return { + ok: response.ok, + status: response.status, + ttfbMs: headersAt - startedAt, + totalMs: completedAt - startedAt + }; + } catch (error) { + const completedAt = performance.now(); + return { + ok: false, + status: 0, + error: error instanceof Error ? error.message : String(error), + ttfbMs: completedAt - startedAt, + totalMs: completedAt - startedAt + }; + } +} + +async function runRoute(baseUrl, route, authToken) { + const samples = []; + const routeStartedAt = performance.now(); + + for (let completed = 0; completed < requestCount; completed += concurrency) { + const batchSize = Math.min(concurrency, requestCount - completed); + const batch = Array.from({ length: batchSize }, () => timeRequest(baseUrl, route, authToken)); + samples.push(...await Promise.all(batch)); + } + + const wallMs = performance.now() - routeStartedAt; + const totalLatencies = samples.map((sample) => sample.totalMs); + const ttfbLatencies = samples.map((sample) => sample.ttfbMs); + const errors = samples.filter((sample) => !sample.ok).length; + const successful = samples.length - errors; + const fastestMs = Math.min(...totalLatencies); + + return { + name: route.name, + method: route.method, + path: route.path, + requests: samples.length, + successful, + errors, + errorRatePercent: round((errors / samples.length) * 100), + latencyMs: { + p50: round(percentile(totalLatencies, 50)), + p95: round(percentile(totalLatencies, 95)), + p99: round(percentile(totalLatencies, 99)) + }, + ttfbMs: { + p50: round(percentile(ttfbLatencies, 50)), + p95: round(percentile(ttfbLatencies, 95)), + p99: round(percentile(ttfbLatencies, 99)) + }, + rps: { + sustained: round(samples.length / (wallMs / 1000)), + peak: round(1000 / fastestMs) + }, + statuses: samples.reduce((acc, sample) => { + const key = String(sample.status); + acc[key] = (acc[key] ?? 0) + 1; + return acc; + }, {}) + }; +} + +async function loadThresholds() { + const raw = await fs.readFile(thresholdsPath, "utf8"); + return JSON.parse(raw); +} + +function evaluateThresholds(results, thresholds) { + const failures = []; + + for (const result of results) { + const routeThreshold = thresholds.routes?.[result.name] ?? {}; + const p99Ms = routeThreshold.p99Ms ?? thresholds.defaults.p99Ms; + const errorRatePercent = routeThreshold.errorRatePercent ?? thresholds.defaults.errorRatePercent; + + if (result.latencyMs.p99 > p99Ms) { + failures.push(`${result.name} p99 ${result.latencyMs.p99}ms exceeded ${p99Ms}ms`); + } + + if (result.errorRatePercent > errorRatePercent) { + failures.push(`${result.name} error rate ${result.errorRatePercent}% exceeded ${errorRatePercent}%`); + } + } + + return failures; +} + +function formatMarkdown(report) { + const rows = report.results.map((result) => [ + result.name, + `${result.method} ${result.path}`, + result.requests, + result.latencyMs.p50, + result.latencyMs.p95, + result.latencyMs.p99, + result.ttfbMs.p95, + result.rps.sustained, + result.rps.peak, + result.errorRatePercent + ]); + + return `# API Benchmark Summary + +- Mode: ${report.mode} +- Target: ${report.target} +- Routes covered: ${report.results.length} +- Requests per endpoint: ${report.config.requestCount} +- Concurrency per endpoint: ${report.config.concurrency} +- Runtime: ${report.environment.node} +- OS: ${report.environment.os} +- Threshold result: ${report.thresholdFailures.length === 0 ? "passed" : "failed"} + +| Route | Endpoint | Requests | p50 ms | p95 ms | p99 ms | p95 TTFB ms | Sustained RPS | Peak RPS | Error % | +| --- | --- | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | +${rows.map((row) => `| ${row.join(" | ")} |`).join("\n")} + +${report.thresholdFailures.length === 0 ? "" : `## Threshold Failures\n\n${report.thresholdFailures.map((failure) => `- ${failure}`).join("\n")}\n`} +`; +} + +async function writeReports(report) { + await fs.mkdir(resultsDir, { recursive: true }); + const jsonPath = path.join(resultsDir, "api-benchmark-latest.json"); + const markdownPath = path.join(resultsDir, "api-benchmark-latest.md"); + + await fs.writeFile(jsonPath, `${JSON.stringify(report, null, 2)}\n`); + await fs.writeFile(markdownPath, formatMarkdown(report)); + + return { jsonPath, markdownPath }; +} + +async function main() { + let localApp; + const configuredBaseUrl = process.env.BENCHMARK_BASE_URL; + const authToken = process.env.BENCHMARK_AUTH_TOKEN + ?? signAccessToken({ sub: "usr_benchmark", role: "admin", scope: "benchmark" }); + + if (!configuredBaseUrl) { + localApp = await startLocalApp(); + } + + const baseUrl = configuredBaseUrl ?? localApp.baseUrl; + const results = []; + + try { + for (const route of benchmarkRoutes) { + results.push(await runRoute(baseUrl, route, authToken)); + } + } finally { + if (localApp) { + await localApp.close(); + } + } + + const thresholds = await loadThresholds(); + const thresholdFailures = evaluateThresholds(results, thresholds); + const report = { + generatedAt: new Date().toISOString(), + mode: isSmoke ? "smoke" : "full", + target: configuredBaseUrl ? "configured" : "local", + config: { + requestCount, + concurrency + }, + environment: { + node: process.version, + os: `${os.type()} ${os.release()} ${os.arch()}`, + cpu: os.cpus()[0]?.model ?? "unknown", + cpuCount: os.cpus().length, + totalMemoryMb: Math.round(os.totalmem() / 1024 / 1024), + freeMemoryMb: Math.round(os.freemem() / 1024 / 1024) + }, + thresholds, + thresholdFailures, + results + }; + + const { jsonPath, markdownPath } = await writeReports(report); + console.log(`Wrote ${jsonPath}`); + console.log(`Wrote ${markdownPath}`); + + if (thresholdFailures.length > 0) { + console.error(thresholdFailures.join("\n")); + process.exitCode = 1; + } +} + +await main(); diff --git a/benchmarks/thresholds.json b/benchmarks/thresholds.json new file mode 100644 index 000000000..6cec361a0 --- /dev/null +++ b/benchmarks/thresholds.json @@ -0,0 +1,12 @@ +{ + "defaults": { + "p99Ms": 1000, + "errorRatePercent": 0 + }, + "routes": { + "uploads.create": { + "p99Ms": 1500, + "errorRatePercent": 0 + } + } +} diff --git a/package.json b/package.json index 675e6e69d..6fb41d794 100644 --- a/package.json +++ b/package.json @@ -8,6 +8,8 @@ "scripts": { "build": "echo \"Run package-specific builds (e.g. npm run build -w apps/web)\"", "lint": "echo \"No root lint configured\"", - "test": "npm run test -w apps/api" + "test": "npm run test -w apps/api", + "benchmark": "node benchmarks/run-api-benchmarks.mjs", + "benchmark:smoke": "node benchmarks/run-api-benchmarks.mjs --smoke" } }