diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml index 8aebd34..a3dcbf8 100644 --- a/.github/workflows/benchmark.yml +++ b/.github/workflows/benchmark.yml @@ -2,9 +2,20 @@ name: Benchmark on: workflow_dispatch: + inputs: + engines: + description: 'Which engine benchmarks to run' + type: choice + default: all + options: + - all + - java + - ruby + - node jobs: benchmark-java: + if: ${{ inputs.engines == 'all' || inputs.engines == 'java' }} runs-on: ubuntu-latest strategy: @@ -85,6 +96,7 @@ jobs: retention-days: 30 benchmark-ruby: + if: ${{ inputs.engines == 'all' || inputs.engines == 'ruby' }} runs-on: ubuntu-latest strategy: @@ -152,8 +164,86 @@ jobs: path: ${{ steps.names.outputs.result_file }} retention-days: 30 + benchmark-node: + if: ${{ inputs.engines == 'all' || inputs.engines == 'node' }} + runs-on: ubuntu-latest + + strategy: + fail-fast: false + matrix: + driver: + - configs/drivers/default/valkey-glide-node.json + - configs/drivers/default/ioredis.json + - configs/drivers/default/iovalkey.json + workload: + - configs/workloads/reference/basic-standalone-single-client-1M-reqs.json + + steps: + - uses: actions/checkout@v4 + + # Node 22 LTS: ioredis 6 and node-redis 6 both require >= 20, and 18 is EOL. + - name: Set up Node.js + uses: actions/setup-node@v4 + with: + node-version: '22' + cache: npm + cache-dependency-path: node/package-lock.json + + - name: Install build dependencies + run: | + sudo apt-get update + sudo apt-get install -y build-essential + + - name: Start Valkey server + run: | + # Use Makefile target which builds from source and configures with persistence disabled + make server-standalone-start + # Wait for server to be ready + sleep 2 + # Verify server is up and persistence is disabled + work/valkey/bin/valkey-cli ping + work/valkey/bin/valkey-cli CONFIG GET save + + - name: Build Node.js benchmark + run: cd node && npm ci && npm run build + + - name: Extract names for result file + id: names + run: | + DRIVER_NAME=$(basename ${{ matrix.driver }} .json) + WORKLOAD_NAME=$(basename ${{ matrix.workload }} .json) + echo "driver_name=$DRIVER_NAME" >> $GITHUB_OUTPUT + echo "workload_name=$WORKLOAD_NAME" >> $GITHUB_OUTPUT + echo "result_file=results/github-runner/reference/${DRIVER_NAME}-${WORKLOAD_NAME}.ndjson" >> $GITHUB_OUTPUT + + - name: Run benchmark + run: | + mkdir -p results/github-runner/reference + node node/dist/src/cli.js \ + --server localhost:6379 \ + --driver ${{ matrix.driver }} \ + --workload ${{ matrix.workload }} \ + --metrics ${{ steps.names.outputs.result_file }} \ + --commit-id ${{ github.sha }} + + - name: Stop Valkey server + if: always() + run: | + make server-standalone-stop || true + + - name: Upload results + uses: actions/upload-artifact@v4 + with: + name: benchmark-node-${{ steps.names.outputs.driver_name }}-${{ steps.names.outputs.workload_name }} + path: ${{ steps.names.outputs.result_file }} + retention-days: 30 + generate-graphs: - needs: [benchmark-java, benchmark-ruby] + needs: [benchmark-java, benchmark-ruby, benchmark-node] + # A skipped `needs` job would skip this one by default, so an engine-scoped + # run would produce no graphs at all. Individual graph steps are gated on the + # same input instead. + if: ${{ !cancelled() }} runs-on: ubuntu-latest permissions: contents: write @@ -185,6 +275,7 @@ jobs: # TODO - rework with matrix orchistrator - name: Generate Java graphs - 1 Client + if: ${{ inputs.engines == 'all' || inputs.engines == 'java' }} run: | python scripts/generate_graphs.py \ --results "results/github-runner/reference/*-basic-standalone-single-client-1M-reqs.ndjson" \ @@ -195,6 +286,7 @@ jobs: --commit-id ${{ github.sha }} - name: Generate Java graphs - 10 Clients + if: ${{ inputs.engines == 'all' || inputs.engines == 'java' }} run: | python scripts/generate_graphs.py \ --results "results/github-runner/reference/*-basic-standalone-10-clients.ndjson" \ @@ -205,6 +297,7 @@ jobs: --commit-id ${{ github.sha }} - name: Generate Java graphs - 100 Clients + if: ${{ inputs.engines == 'all' || inputs.engines == 'java' }} run: | python scripts/generate_graphs.py \ --results "results/github-runner/reference/*-basic-standalone-100-clients.ndjson" \ @@ -216,6 +309,7 @@ jobs: # Ruby graphs — per concurrency level - name: Generate Ruby graphs - 1 Client + if: ${{ inputs.engines == 'all' || inputs.engines == 'ruby' }} run: | python scripts/generate_graphs.py \ --results "results/github-runner/reference/*-basic-standalone-single-client-1M-reqs.ndjson" \ @@ -226,6 +320,7 @@ jobs: --commit-id ${{ github.sha }} - name: Generate Ruby graphs - 10 Clients + if: ${{ inputs.engines == 'all' || inputs.engines == 'ruby' }} run: | python scripts/generate_graphs.py \ --results "results/github-runner/reference/*-basic-standalone-10-clients.ndjson" \ @@ -236,6 +331,7 @@ jobs: --commit-id ${{ github.sha }} - name: Generate Ruby graphs - 100 Clients + if: ${{ inputs.engines == 'all' || inputs.engines == 'ruby' }} run: | python scripts/generate_graphs.py \ --results "results/github-runner/reference/*-basic-standalone-100-clients.ndjson" \ @@ -245,6 +341,17 @@ jobs: --workload "Ruby - 100 Clients" \ --commit-id ${{ github.sha }} + - name: Generate Node.js graphs - 1 Client + if: ${{ inputs.engines == 'all' || inputs.engines == 'node' }} + run: | + python scripts/generate_graphs.py \ + --results "results/github-runner/reference/*-basic-standalone-single-client-1M-reqs.ndjson" \ + --output graphs/node/1-client/ \ + --phase STEADY \ + --language node \ + --workload "Node.js - 1 Client" \ + --commit-id ${{ github.sha }} + - name: Upload graphs uses: actions/upload-artifact@v4 with: diff --git a/Makefile b/Makefile index db7d5d2..c83bb95 100644 --- a/Makefile +++ b/Makefile @@ -34,6 +34,8 @@ WORK_DIR=$(shell pwd)/work python-build python-test python-run python-clean \ ruby-build ruby-test ruby-run ruby-clean ruby-info \ csharp-build csharp-test csharp-run csharp-clean csharp-info \ + node-build node-deps node-test node-unit-test node-integration-test \ + node-run node-clean node-info \ config-editor-build config-editor-dev # ============================================================================ @@ -74,6 +76,13 @@ help: @echo " make csharp-clean Clean C# build artifacts" @echo " make csharp-info Show supported C# drivers and commands" @echo "" + @echo "Node.js Engine:" + @echo " make node-build Install deps and compile the Node.js engine" + @echo " make node-test Run Node.js tests (unit + integration)" + @echo " make node-run Run Node.js benchmark (requires DRIVER and WORKLOAD)" + @echo " make node-clean Clean Node.js build artifacts" + @echo " make node-info Show supported Node.js drivers and commands" + @echo "" @echo "Config Editor:" @echo " make config-editor-build Build config editor UI" @echo " make config-editor-dev Run config editor in development mode" @@ -387,6 +396,56 @@ csharp-clean: csharp-info: csharp-build dotnet run --project $(CSHARP_PROJECT) -c Release -- --info +# ============================================================================ +# Node.js Engine +# ============================================================================ + +# tsc emits into node/dist mirroring the source tree, so src/cli.ts -> dist/src/cli.js +NODE_CLI=node/dist/src/cli.js + +# Stamp file so `npm ci` runs only when the manifests actually change. The matrix +# runner invokes `make node-run` once per cell, and `npm ci` deletes and +# reinstalls node_modules every time it runs — on a 75-cell sweep that is ~15 +# minutes of pure reinstall plus 75 chances for a network blip mid-sweep. The +# stamp lives inside node_modules, so wiping that directory correctly forces a +# reinstall. `npm run build` stays on every invocation: tsc is incremental and +# no-ops in well under a second. +NODE_DEPS_STAMP=node/node_modules/.resp-bench-deps-stamp + +$(NODE_DEPS_STAMP): node/package.json node/package-lock.json + cd node && npm ci + @touch $@ + +node-deps: $(NODE_DEPS_STAMP) + +node-build: node-deps + cd node && npm run build + +node-test: node-unit-test node-integration-test + +node-unit-test: node-build + cd node && node --test dist/test/unit/ + +# The live-server tests skip themselves unless VALKEY_HOST is set, so the server +# has to be up before this runs. +node-integration-test: node-build server-standalone-start + sleep 1 + cd node && VALKEY_HOST=localhost VALKEY_PORT=6379 node --test dist/test/integration/ + $(MAKE) server-standalone-stop + +node-run: node-build + node $(NODE_CLI) \ + --server $(SERVER) \ + --driver $(DRIVER) \ + --workload $(WORKLOAD) \ + --metrics $(METRICS_OUTPUT) + +node-info: node-build + node $(NODE_CLI) --info + +node-clean: + cd node && rm -rf dist node_modules coverage + # ============================================================================ # Config Editor # ============================================================================ @@ -447,8 +506,8 @@ test-scripts-all: java-build # All Languages # ============================================================================ -build-all: java-build ruby-build csharp-build python-build +build-all: java-build ruby-build csharp-build node-build python-build -test-all: java-test ruby-test csharp-test python-test +test-all: java-test ruby-test csharp-test node-test python-test -clean-all: java-clean ruby-clean csharp-clean python-clean clean +clean-all: java-clean ruby-clean csharp-clean node-clean python-clean clean diff --git a/README.md b/README.md index 5f55ed1..f14aa08 100644 --- a/README.md +++ b/README.md @@ -13,6 +13,7 @@ A multi-language benchmark suite for RESP protocol (Redis/Valkey) compatible dat ### Prerequisites - Python 3.8+, Java 21+, Maven +- Node.js 20+ (for the Node.js engine) - Make - A server CLI (`valkey-cli`) for the matrix runner's readiness probe and per-cell FLUSHALL — the Makefile's `server-*` targets build one into @@ -103,6 +104,7 @@ Thread-based system metrics collector that runs alongside benchmarks, collecting | Java | ✅ Ready | Jedis, Lettuce, Valkey-Glide, Redisson, Spring Data Valkey/Redis | | Ruby | ✅ Ready | redis-rb, valkey-glide-ruby | | C# | ✅ Ready | valkey-glide-csharp, StackExchange.Redis | +| Node.js | ✅ Ready | valkey-glide-node, ioredis, iovalkey | | Python | 🚧 Planned | redis-py, aioredis, valkey-glide | ## Project Structure @@ -133,6 +135,7 @@ resp-bench/ ├── java/ # Java benchmark engine ├── ruby/ # Ruby benchmark engine ├── csharp/ # C# (.NET 10) benchmark engine +├── node/ # Node.js (TypeScript) benchmark engine ├── docs/ │ ├── ARCHITECTURE.md # System architecture │ ├── BENCHMARK_MATRIX.md # Matrix orchestrator docs @@ -140,7 +143,8 @@ resp-bench/ │ ├── CONFIG_SPECIFICATION.md # Configuration format spec │ ├── BENCHMARKS_JAVA.md # Java benchmark details │ ├── BENCHMARKS_CSHARP.md # C# benchmark details -│ └── BENCHMARKS_RUBY.md # Ruby benchmark details +│ ├── BENCHMARKS_RUBY.md # Ruby benchmark details +│ └── BENCHMARKS_NODE.md # Node.js benchmark details └── graphs/interactive/ # Generated HTML graphs ``` @@ -197,6 +201,7 @@ See [docs/CONFIG_SPECIFICATION.md](docs/CONFIG_SPECIFICATION.md) for full detail | `make java-test` | Run Java unit tests | | `make ruby-test` | Run Ruby tests | | `make csharp-test` | Run C# tests | +| `make node-test` | Run Node.js tests (unit + integration) | ### Engines @@ -205,8 +210,10 @@ See [docs/CONFIG_SPECIFICATION.md](docs/CONFIG_SPECIFICATION.md) for full detail | `make java-run` | Run Java engine (DRIVER, WORKLOAD, SERVER) | | `make ruby-run` | Run Ruby engine (DRIVER, WORKLOAD, SERVER) | | `make csharp-run` | Run C# engine (DRIVER, WORKLOAD, SERVER) | +| `make node-run` | Run Node.js engine (DRIVER, WORKLOAD, SERVER) | | `make java-build` | Build Java JAR | | `make csharp-build` | Build C# executable | +| `make node-build` | Install deps and compile the Node.js engine | ### Server Management diff --git a/config-editor/src/App.tsx b/config-editor/src/App.tsx index 7b8e89f..492df38 100644 --- a/config-editor/src/App.tsx +++ b/config-editor/src/App.tsx @@ -35,7 +35,12 @@ interface CommandConfig { data_size_bytes?: number } -const DRIVERS = ['jedis', 'lettuce', 'valkey-glide', 'redisson', 'spring-data-valkey', 'spring-data-redis'] +const DRIVERS = [ + // Java + 'jedis', 'lettuce', 'valkey-glide', 'redisson', 'spring-data-valkey', 'spring-data-redis', + // Node.js + 'valkey-glide-node', 'ioredis', 'iovalkey', +] const COMMANDS = ['set', 'get', 'ping'] const ALGORITHMS = ['sequential_int', 'uniform_rand'] diff --git a/configs/drivers/default/ioredis.json b/configs/drivers/default/ioredis.json new file mode 100644 index 0000000..8261f9d --- /dev/null +++ b/configs/drivers/default/ioredis.json @@ -0,0 +1,7 @@ +{ + "schema_version": "1.0", + "description": "ioredis client - default configuration", + "driver_id": "ioredis", + "mode": "standalone", + "specific_driver_config": {} +} diff --git a/configs/drivers/default/iovalkey.json b/configs/drivers/default/iovalkey.json new file mode 100644 index 0000000..e4793ec --- /dev/null +++ b/configs/drivers/default/iovalkey.json @@ -0,0 +1,7 @@ +{ + "schema_version": "1.0", + "description": "iovalkey client - default configuration", + "driver_id": "iovalkey", + "mode": "standalone", + "specific_driver_config": {} +} diff --git a/configs/drivers/default/valkey-glide-node.json b/configs/drivers/default/valkey-glide-node.json new file mode 100644 index 0000000..bc25ae1 --- /dev/null +++ b/configs/drivers/default/valkey-glide-node.json @@ -0,0 +1,7 @@ +{ + "schema_version": "1.0", + "description": "Valkey GLIDE for Node.js client - default configuration", + "driver_id": "valkey-glide-node", + "mode": "standalone", + "specific_driver_config": {} +} diff --git a/configs/drivers/example-ioredis-standalone.json b/configs/drivers/example-ioredis-standalone.json new file mode 100644 index 0000000..9b594f5 --- /dev/null +++ b/configs/drivers/example-ioredis-standalone.json @@ -0,0 +1,7 @@ +{ + "schema_version": "1.0", + "description": "ioredis client, standalone mode", + "driver_id": "ioredis", + "mode": "standalone", + "specific_driver_config": {} +} diff --git a/configs/drivers/example-iovalkey-standalone.json b/configs/drivers/example-iovalkey-standalone.json new file mode 100644 index 0000000..978fa32 --- /dev/null +++ b/configs/drivers/example-iovalkey-standalone.json @@ -0,0 +1,7 @@ +{ + "schema_version": "1.0", + "description": "iovalkey client, standalone mode", + "driver_id": "iovalkey", + "mode": "standalone", + "specific_driver_config": {} +} diff --git a/configs/drivers/example-valkey-glide-node-standalone.json b/configs/drivers/example-valkey-glide-node-standalone.json new file mode 100644 index 0000000..5c5bb3d --- /dev/null +++ b/configs/drivers/example-valkey-glide-node-standalone.json @@ -0,0 +1,7 @@ +{ + "schema_version": "1.0", + "description": "Valkey GLIDE for Node.js client, standalone mode", + "driver_id": "valkey-glide-node", + "mode": "standalone", + "specific_driver_config": {} +} diff --git a/configs/drivers/high-throughput/ioredis.json b/configs/drivers/high-throughput/ioredis.json new file mode 100644 index 0000000..d214e79 --- /dev/null +++ b/configs/drivers/high-throughput/ioredis.json @@ -0,0 +1,8 @@ +{ + "schema_version": "1.0", + "description": "ioredis client - high-throughput configuration", + "driver_id": "ioredis", + "mode": "standalone", + "specific_driver_config": {}, + "command_timeout_ms": 10000 +} diff --git a/configs/drivers/high-throughput/iovalkey.json b/configs/drivers/high-throughput/iovalkey.json new file mode 100644 index 0000000..6a1d347 --- /dev/null +++ b/configs/drivers/high-throughput/iovalkey.json @@ -0,0 +1,8 @@ +{ + "schema_version": "1.0", + "description": "iovalkey client - high-throughput configuration", + "driver_id": "iovalkey", + "mode": "standalone", + "specific_driver_config": {}, + "command_timeout_ms": 10000 +} diff --git a/configs/drivers/high-throughput/valkey-glide-node.json b/configs/drivers/high-throughput/valkey-glide-node.json new file mode 100644 index 0000000..abc7a6a --- /dev/null +++ b/configs/drivers/high-throughput/valkey-glide-node.json @@ -0,0 +1,8 @@ +{ + "schema_version": "1.0", + "description": "Valkey GLIDE for Node.js client - high-throughput configuration", + "driver_id": "valkey-glide-node", + "mode": "standalone", + "specific_driver_config": {}, + "command_timeout_ms": 10000 +} diff --git a/configs/matrices/node-driver-comparison.json b/configs/matrices/node-driver-comparison.json new file mode 100644 index 0000000..b9eefff --- /dev/null +++ b/configs/matrices/node-driver-comparison.json @@ -0,0 +1,20 @@ +{ + "description": "Compare Node.js drivers (GLIDE vs ioredis vs iovalkey) across client counts", + "x_axis": "connections", + "workload_template": "configs/workloads/reference/basic-standalone-single-client-10-secs.json", + "iterations": 5, + "dimensions": { + "connections": [ + 1, + 2, + 4, + 8, + 16 + ], + "driver_config": [ + "configs/drivers/default/valkey-glide-node.json", + "configs/drivers/default/ioredis.json", + "configs/drivers/default/iovalkey.json" + ] + } +} diff --git a/configs/schemas/driver-config.schema.json b/configs/schemas/driver-config.schema.json index e06cf4e..c20cccc 100644 --- a/configs/schemas/driver-config.schema.json +++ b/configs/schemas/driver-config.schema.json @@ -18,7 +18,7 @@ "driver_id": { "type": "string", "description": "Identifier for the client driver implementation", - "examples": ["jedis", "lettuce", "valkey-glide", "redisson", "stackexchange-redis", "valkey-glide-csharp", "redis-py", "aioredis"] + "examples": ["jedis", "lettuce", "valkey-glide", "redisson", "stackexchange-redis", "valkey-glide-csharp", "valkey-glide-node", "ioredis", "iovalkey", "redis-py", "aioredis"] }, "mode": { "type": "string", diff --git a/docs/ADDING_LANGUAGE.md b/docs/ADDING_LANGUAGE.md index c8adab4..5ddb8c8 100644 --- a/docs/ADDING_LANGUAGE.md +++ b/docs/ADDING_LANGUAGE.md @@ -7,7 +7,10 @@ This guide explains how to add support for a new programming language to resp-be Before adding a new language, ensure you understand: - [Architecture](ARCHITECTURE.md) - Overall system design - [Configuration Specification](CONFIG_SPECIFICATION.md) - Config format details -- Existing implementations (Java is the reference implementation) +- Existing implementations: **Java is the reference implementation**; Ruby, C# and + Node.js follow it. For an async/event-loop language, `node/` is the closest model + and its README documents the parity traps worth knowing up front (RNG width, HDR + encoding, percentile-vs-raw min/max, shared request budget). ## Step-by-Step Guide @@ -178,9 +181,10 @@ class MetricsCollector: def record(self, command: str, latency_us: int, success: bool) -> None: if command not in self.command_metrics: - # 1µs to 1 hour, 3 significant figures + # 1µs to 600s, 3 significant figures (must match the other engines: + # Java/C#/Ruby/Node all use a max of 600_000_000µs, not 1 hour) self.command_metrics[command] = CommandMetrics( - histogram=HdrHistogram(1, 3600000000, 3) + histogram=HdrHistogram(1, 600000000, 3) ) metrics = self.command_metrics[command] @@ -389,9 +393,28 @@ Before submitting a new language engine: - [ ] Config parsing handles all schema fields - [ ] Key generator produces identical sequences (test with seed=12345) +- [ ] Keys are zero-padded to `key_size_bytes` (Java: `"%0" + max(1, key_size_bytes - prefix.length) + "d"`) +- [ ] The `uniform_rand` PRNG matches `java.util.Random` **including** the int32 + overflow check in `nextInt`'s rejection branch. Watch the arithmetic width: + the 48-bit LCG multiply exceeds what a double-based number type holds exactly + (this bites JavaScript, where `BigInt` is required) +- [ ] `sequential_int` uses a counter **shared** across all workers; `uniform_rand` + uses a per-worker PRNG seeded `base_seed + worker_index` +- [ ] The request budget is **shared** across workers and claimed one request at a + time, not pre-divided per worker (Java: one `AtomicLong` per phase) +- [ ] PING does **not** consume a generated key (Java's `PingCommand` ignores the + key generator, so consuming one shifts every subsequent key) - [ ] Rate limiter achieves target rates within 5% tolerance - [ ] Metrics output matches NDJSON schema exactly -- [ ] HdrHistogram produces compatible base64 payloads +- [ ] HdrHistogram produces compatible base64 payloads. If your library's encode + already returns base64, use it **directly** — encoding it again yields a + payload Java and Ruby cannot decode +- [ ] `summary.min`/`max` match Java's `getMinValue()`/`getMaxValue()`, which return + the *bucket's* equivalent bounds. Many ports expose a raw min/max property + instead; those diverge above ~1000µs at 3 significant figures. Prefer + `getValueAtPercentile(0)` / `getValueAtPercentile(100)` +- [ ] `warmup_requests`, `cps_limit`, `rps_limit`, `pipeline_depth`, + `command_timeout_ms`, `tls` and `auth` are all actually honoured, not just parsed - [ ] All unit tests pass - [ ] Integration tests pass against live server - [ ] Documentation complete diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md index 6463d7e..f1ac564 100644 --- a/docs/ARCHITECTURE.md +++ b/docs/ARCHITECTURE.md @@ -156,6 +156,18 @@ class BenchmarkClient(ABC): async def close(self): ... ``` +**Node.js (TypeScript):** +```typescript +interface BenchmarkClient { + connect(host: string, port: number, config: DriverConfig): Promise; + ping(): Promise>; + get(key: string): Promise>; + set(key: string, value: Buffer): Promise>; + close(): Promise; + driverVersion(): string; +} +``` + ## Metrics Output Format All engines produce identical NDJSON output: @@ -236,10 +248,15 @@ Different languages use appropriate concurrency primitives: | C# | Task-per-client with async/await (.NET 8+) | | Python | asyncio with async/await | | Go | goroutines and channels | -| Node.js | Promise/async-await | +| Node.js | One event loop, worker-per-connection (Promise/async-await) | The key requirement is that N connections can operate concurrently, each potentially with pipeline_depth in-flight requests. +**Node.js caveat:** the engine is single-threaded, so one CPU core bounds the whole +run. Past that point measurements reflect the engine rather than the client. See +[BENCHMARKS_NODE.md](BENCHMARKS_NODE.md) § "The Single-Core Ceiling" for the +measured plateau and how to tell when you have hit it. + ## Parallel Command Issuers (Java) At high connection counts (128+), a single command-issuing thread becomes a CPU bottleneck — saturating one core on semaphore contention, key generation (`String.format()`), and round-robin scanning. To address this, the Java engine supports **parallel command issuer threads** that partition client connections across multiple threads. diff --git a/docs/BENCHMARKS_NODE.md b/docs/BENCHMARKS_NODE.md new file mode 100644 index 0000000..00e2a4b --- /dev/null +++ b/docs/BENCHMARKS_NODE.md @@ -0,0 +1,147 @@ +# Node.js Benchmarks + +Details specific to the Node.js engine (`node/`). See [ARCHITECTURE.md](ARCHITECTURE.md) +for the cross-engine design and [../node/README.md](../node/README.md) for build and +usage. + +## Drivers + +| `driver_id` | Package | Notes | +|---|---|---| +| `valkey-glide-node` | `@valkey/valkey-glide` | Valkey GLIDE for Node.js. Prebuilt native binaries per platform. | +| `ioredis` | `ioredis` | The most widely used Node.js Redis client. | +| `iovalkey` | `iovalkey` | The Valkey-maintained fork of ioredis; API-identical. | +| `recording` | — | Synthetic-latency client for server-free testing. | + +The GLIDE id is `valkey-glide-node`, **not** `valkey-glide` — the latter belongs to +the Java engine in the global `DRIVER_ENGINE_MAP`. + +## Concurrency Model + +One event loop, one client per connection, one worker per connection. An awaited +command parks its worker, not the loop, so other connections keep progressing — +the analogue of Java's virtual-thread-per-client design. + +`pipeline_depth > 1` gives each connection that many independent +issue/await/record slots, so a settled request is replaced immediately rather than +waiting for a batch. + +The per-phase request budget is shared across workers and claimed one request at a +time (matching Java's `AtomicLong`), so a slow connection cannot cap the run. + +## The Single-Core Ceiling + +**Node runs the whole engine on one thread, so one CPU core is the hard ceiling.** +This is the single most important caveat when comparing Node numbers to Java or C#, +which spread issuing across threads. + +Measured locally (Apple M-series laptop, Valkey 8 co-located on the same host, +100% GET, 512B values, 10k keys, 5s phases). **Absolute numbers here are not +publishable results** — client and server contend for the same cores. The *shape* +is the point: + +Throughput vs connections (`pipeline_depth=1`): + +| Connections | glide RPS | ioredis RPS | iovalkey RPS | +|---|---|---|---| +| 1 | 1,139 | 1,584 | 1,646 | +| 10 | 8,424 | 8,467 | 10,360 | +| 50 | 25,065 | 23,071 | 26,268 | +| 100 | 33,639 | 34,898 | 35,087 | +| 200 | 41,812 | 42,748 | 41,144 | + +Doubling 100 → 200 connections buys only ~20% more throughput while p50 latency +roughly doubles (2.6ms → 4.5ms). That plateau is the event loop saturating, not the +clients. + +Throughput vs `pipeline_depth` at 10 connections (ioredis), with process CPU: + +| `pipeline_depth` | RPS | p50 | p99 | CPU (of one core) | +|---|---|---|---|---| +| 1 | 10,124 | 929µs | 2,055µs | 29% | +| 4 | 32,194 | 1,157µs | 2,731µs | 52% | +| 16 | 72,631 | 1,955µs | 4,089µs | 99% | + +CPU rises in lockstep with throughput and pins at 99% of a single core, where +throughput stops scaling. **The engine, not the client, is the limit past that +point.** + +Practical guidance: + +- Prefer raising `pipeline_depth` over raising `connections` to reach high + throughput on Node — it is far cheaper per unit of RPS. +- When comparing Node against Java/C#, check whether the Node process is CPU-bound. + If it is at ~100% of a core, you are measuring the engine, not the driver. +- Java addresses the same ceiling with parallel command-issuer threads (see + [ARCHITECTURE.md](ARCHITECTURE.md) § "Parallel Command Issuers"). The Node + equivalent would be `worker_threads` with a client partition per worker. That is + deliberately **not** implemented — it is a follow-up, to be justified by + measurements rather than assumed. + +## Fairness Controls + +Node-specific hazards with no analogue in the other engines, each handled +explicitly: + +- **Auto-pipelining is forced off** (`enableAutoPipelining: false`). ioredis and + iovalkey can transparently batch commands issued in the same event-loop tick, + which would inflate throughput against every other engine while looking like a + driver win. +- **Reconnects are disabled** (`retryStrategy: () => null`). ioredis' default + retries forever, so a wrong host would hang a run rather than fail it, and a + mid-phase reconnect would fold connection setup into request latency. +- **Response decoding is uniform.** `get` returns a `string` in all three drivers, + so none is charged for a different amount of decoding. GLIDE returns strings by + default; we do not opt one driver into bytes. +- **SET payloads are allocated once** per command object, so GC churn from payload + construction is not attributed to the driver. +- **Sub-millisecond rate limits work.** `setTimeout` clamps to ~1ms, so the limiter + yields via `setImmediate` below that. A 100k rps limit is a 10µs interval; a + timer-based wait would undershoot by ~100×. +- **Memory is not comparable to the JVM.** The system monitor's RSS samples include + V8 heap growth, which grows and collects on a different schedule from the JVM's. + +## Cross-Engine Parity + +Verified against the Java reference rather than assumed: + +- **Key sequences are byte-identical to Java.** 79,000 keys diffed against Java's + real `KeyGenerator` across `sequential_int` and `uniform_rand`, 1–16 workers, + prime `keys_count`, and prefix-width edge cases. `javaRandom.ts` ports + `java.util.Random` with `BigInt` (the 48-bit LCG multiply reaches ~2^83, past + what a JS `number` holds exactly) and is anchored to + `new Random(0).nextInt() == -1155484576`. +- **HDR payloads decode in Java.** `payload_b64` is + `encodeIntoCompressedBase64()` used directly — it is already base64, so encoding + it again would produce something Java cannot read. Verified by decoding a + Node-produced payload with `org.HdrHistogram.Histogram`: identical count and + percentiles. Range `(1, 600_000_000, 3)`, as in every engine. +- **`summary.min`/`max` match Java's quantization.** Java's `getMinValue()`/ + `getMaxValue()` return the bucket's equivalent bounds, while hdr-histogram-js' + `minNonZeroValue`/`maxValue` return the raw sample — they diverge above ~1000µs + (50000µs recorded reads back as 50015 in Java). The engine uses + `getValueAtPercentile(0)` and `getValueAtPercentile(100)`, which match Java exactly. +- **PING does not consume a key**, matching Java's `PingCommand`, so mixing PING + into a workload does not shift the key sequence other engines would produce. + +## Reproducing the Numbers Above + +```bash +make server-standalone-start + +# Populate the keyspace, then sweep connections. +make node-run \ + DRIVER=configs/drivers/default/ioredis.json \ + WORKLOAD=configs/workloads/reference/basic-standalone-single-client-1M-reqs.json \ + METRICS_OUTPUT=output/node-ioredis.ndjson + +make server-standalone-stop +``` + +For a full matrix across drivers and connection counts, use the orchestrator — +Node drivers are registered in `DRIVER_ENGINE_MAP`, so it dispatches to +`make node-run` automatically: + +```bash +python scripts/run_benchmark_matrix.py --matrix configs/matrices/.json +``` diff --git a/docs/CONFIG_SPECIFICATION.md b/docs/CONFIG_SPECIFICATION.md index 6a4c0b7..153ea73 100644 --- a/docs/CONFIG_SPECIFICATION.md +++ b/docs/CONFIG_SPECIFICATION.md @@ -54,6 +54,11 @@ resp-bench uses two JSON configuration files: - `stackexchange-redis` - StackExchange.Redis client - `valkey-glide-csharp` - Valkey GLIDE C# client +**Node.js:** +- `valkey-glide-node` - Valkey GLIDE Node.js client (not `valkey-glide`, which is Java's) +- `ioredis` - ioredis client +- `iovalkey` - iovalkey client (the Valkey-maintained ioredis fork) + **Python (planned):** - `redis-py` - redis-py synchronous client - `redis-py-async` - redis-py async client diff --git a/infra/provision.sh b/infra/provision.sh index 1c3699f..1de54b7 100755 --- a/infra/provision.sh +++ b/infra/provision.sh @@ -213,6 +213,35 @@ env_add "export DOTNET_CLI_TELEMETRY_OPTOUT=1" env_add "export DOTNET_NOLOGO=1" "${DOTNET_ROOT}/dotnet" --info 2>&1 | head -n3 | sed 's/^/[provision] /' || true +# ═════════════════════════════════════════════════════════════════════════════ +# LANGUAGE: Node.js (20+) — engines: valkey-glide-node, ioredis, iovalkey. +# node/package.json declares "engines": {"node": ">=20"}; distro feeds often +# ship 18 (EOL) or older, so install from NodeSource, which is version-pinned +# and works on both dnf and apt. valkey-glide ships prebuilt native binaries +# per platform, so no compiler is needed beyond build-essential (already +# installed above for the server build). +# ═════════════════════════════════════════════════════════════════════════════ + +NODE_MAJOR="${NODE_MAJOR:-22}" +log "LANGUAGE: Node.js — installing Node ${NODE_MAJOR} + npm" +node_major_installed() { + command -v node >/dev/null 2>&1 && + [ "$(node -p 'process.versions.node.split(".")[0]' 2>/dev/null || echo 0)" -ge 20 ] +} +if ! node_major_installed; then + if [ "${PKG}" = "dnf" ]; then + curl -fsSL "https://rpm.nodesource.com/setup_${NODE_MAJOR}.x" | ${SUDO} bash - + pkg_install nodejs -- nodejs + else + curl -fsSL "https://deb.nodesource.com/setup_${NODE_MAJOR}.x" | ${SUDO} bash - + pkg_install nodejs -- nodejs + fi +else + log "Node.js $(node -v) already present (>= 20)" +fi +node -v 2>&1 | sed 's/^/[provision] /' || true +npm -v 2>&1 | sed 's/^/[provision] /' || true + # ═════════════════════════════════════════════════════════════════════════════ # LANGUAGE: Python (3.9+ + pip) — used by the matrix orchestrator and graph # generator (scripts/*.py). Engine deps are installed from the repo's pinned @@ -241,7 +270,6 @@ python --version 2>&1 | sed 's/^/[provision] /' || true # ───────────────────────────────────────────────────────────────────────────── # FUTURE ENGINES (leave room — see resp-bench plan §5.5): -# Node.js (#13): pkg_install nodejs npm -- nodejs npm + `npm ci` warm-up # Go (#14): install the Go toolchain + `go mod download` # PHP (#15): pkg_install php php-cli composer -- php-cli composer # Add each as its own "LANGUAGE:" block above, mirroring the pattern. @@ -295,6 +323,8 @@ if [ "${SKIP_WARM_CACHES:-0}" != "1" ]; then make -C "${REPO_DIR}" ruby-build || log "WARNING: Ruby warm-up build failed (see above)" log "warming C# build cache (dotnet build)" make -C "${REPO_DIR}" csharp-build || log "WARNING: C# warm-up build failed (see above)" + log "warming Node.js deps (npm ci + tsc)" + make -C "${REPO_DIR}" node-build || log "WARNING: Node.js warm-up build failed (see above)" else log "SKIP_WARM_CACHES=1 — skipping engine cache warm-up" fi diff --git a/node/.gitignore b/node/.gitignore new file mode 100644 index 0000000..c19bb02 --- /dev/null +++ b/node/.gitignore @@ -0,0 +1,4 @@ +node_modules/ +dist/ +coverage/ +*.tsbuildinfo diff --git a/node/README.md b/node/README.md new file mode 100644 index 0000000..af7834f --- /dev/null +++ b/node/README.md @@ -0,0 +1,141 @@ +# resp-bench Node.js Engine + +Node.js implementation of the resp-bench benchmark suite, written in TypeScript. + +## Supported Drivers + +| `driver_id` | Package | Notes | +|---|---|---| +| `valkey-glide-node` | [`@valkey/valkey-glide`](https://www.npmjs.com/package/@valkey/valkey-glide) | Valkey GLIDE for Node.js. Ships prebuilt native binaries per platform. | +| `ioredis` | [`ioredis`](https://www.npmjs.com/package/ioredis) | The most widely used Node.js Redis client. | +| `iovalkey` | [`iovalkey`](https://www.npmjs.com/package/iovalkey) | The Valkey-maintained fork of ioredis; API-identical. | +| `recording` | — | In-memory synthetic-latency client for server-free tests. | + +> **Note:** the GLIDE driver id is `valkey-glide-node`, not `valkey-glide`. The +> latter is already the Java engine's id in the global `DRIVER_ENGINE_MAP` in +> `scripts/run_benchmark_matrix.py`, and reusing it would reroute Java's runs here. + +## Requirements + +- Node.js **20 or newer** (`package.json` declares `engines.node >= 20`). CI uses 22 LTS. + +## Build + +```bash +npm ci && npm run build # or: make node-build +``` + +TypeScript compiles to `dist/`, mirroring the source tree, so the entry point is +`dist/src/cli.js`. + +## Usage + +```bash +node dist/src/cli.js \ + --server localhost:6379 \ + --driver ../configs/drivers/default/ioredis.json \ + --workload ../configs/workloads/reference/basic-standalone-single-client-1M-reqs.json \ + --metrics ../output/node.ndjson +``` + +Or through the Makefile, from the repository root: + +```bash +make node-run \ + DRIVER=configs/drivers/default/valkey-glide-node.json \ + WORKLOAD=configs/workloads/reference/basic-standalone-single-client-1M-reqs.json \ + METRICS_OUTPUT=output/node.ndjson + +make node-info # list supported drivers and commands +``` + +## Tests + +```bash +make node-test # unit + integration (starts/stops a server) +cd node && npm run test:unit # unit only, no server needed +``` + +The live-server tests skip themselves unless `VALKEY_HOST` is set: + +```bash +cd node && VALKEY_HOST=localhost VALKEY_PORT=6379 npm run test:integration +``` + +The `recording` driver lets the full engine — phases, warmup, budget, pipelining, +rate limiting, NDJSON — be exercised with no server at all. + +## Concurrency Model + +A single event loop with **one client per connection** and **one worker per +connection**, all started together. Node is single-threaded with async I/O, so an +awaited command parks that worker rather than the loop, and the other connections +keep making progress. This is the analogue of Java's +virtual-thread-per-client design. + +`pipeline_depth > 1` is supported: each connection runs that many independent +issue/await/record slots, so a settled request is replaced immediately. + +The per-phase request budget is **shared across all workers** and claimed one +request at a time, matching the Java reference's `AtomicLong`. It is deliberately +not pre-divided per worker — a shared budget lets fast connections absorb a slow +one's slack, so wall-clock is not bounded by the slowest connection. + +## Cross-Engine Parity + +Verified against the Java reference engine, not assumed: + +- **Key sequences are byte-identical.** 79,000 keys were diffed against Java's + actual `KeyGenerator` across both algorithms, 1–16 workers, prime `keys_count`, + and tight padding. `src/engine/javaRandom.ts` is a `java.util.Random` port + anchored to `new Random(0).nextInt() == -1155484576`; it uses `BigInt` because + the 48-bit LCG multiply overflows a JS `number`. +- **HDR payloads are mutually decodable.** `payload_b64` is + `encodeIntoCompressedBase64()` used directly (it is *already* base64 — encoding + it again would produce something Java cannot read). Java's + `Histogram.decodeFromCompressedByteBuffer` reads our payloads with matching + count and percentiles. Range is `(1, 600_000_000, 3)`, as in every engine. +- **`summary.min`/`max` use `getValueAtPercentile(0/100)`**, not + `minNonZeroValue`/`maxValue`. Java reports the bucket's equivalent bounds, and + hdr-histogram-js' properties return the raw sample — they diverge above ~1000µs + (recording 50000µs yields 50000 in JS but 50015 in Java). +- **PING does not consume a key**, matching Java's `PingCommand`, so mixing PING + into a workload does not shift the key sequence. + +## Node-Specific Fairness Notes + +Read these before comparing Node numbers to another engine: + +- **Auto-pipelining is explicitly disabled.** ioredis and iovalkey can + transparently batch commands issued in the same event-loop tick, which would + inflate throughput against every other engine. `enableAutoPipelining: false`. +- **Reconnects are disabled** (`retryStrategy: () => null`). ioredis otherwise + retries forever, so a wrong host would hang a run instead of failing it, and a + mid-phase reconnect would fold connection setup into request latency. +- **All drivers decode responses the same way.** `get` returns a `string` in all + three, so none is charged for a different amount of decoding. +- **SET payloads are built once** per command object, not per request, so the + driver is not charged for the engine's own allocation churn. +- **Sub-millisecond rate limits work.** `setTimeout` clamps to ~1 ms, so the + limiter yields via `setImmediate` for shorter intervals; a 100k rps limit is a + 10 µs interval and a timer-based wait would undershoot it by ~100×. +- **RSS is not comparable to the JVM's.** The system monitor's memory samples + include V8 heap growth and GC timing, which behave differently from the JVM's. +- **One event loop is one core.** At high connection counts the engine itself, not + the client, may become the ceiling. See `docs/BENCHMARKS_NODE.md`. + +## Layout + +``` +node/ +├── src/ +│ ├── cli.ts # arg parsing, --info, error boundary +│ ├── client/ # driver interface, factory, per-driver impls +│ ├── command/ # GET / SET / PING +│ ├── config/ # JSON config parsing + validation +│ ├── engine/ # benchmark loops, key gen, RNG, rate limiter +│ └── metrics/ # HDR histogram, collector, NDJSON writer +└── test/ + ├── unit/ # no server, no optional deps + └── integration/ # recording driver + live-server tests +``` diff --git a/node/package-lock.json b/node/package-lock.json new file mode 100644 index 0000000..068c475 --- /dev/null +++ b/node/package-lock.json @@ -0,0 +1,420 @@ +{ + "name": "resp-bench-node", + "version": "1.0.0", + "lockfileVersion": 3, + "requires": true, + "packages": { + "": { + "name": "resp-bench-node", + "version": "1.0.0", + "license": "Apache-2.0", + "dependencies": { + "@valkey/valkey-glide": "^2.5.2", + "hdr-histogram-js": "^3.0.1", + "ioredis": "^5.11.1", + "iovalkey": "^0.4.0" + }, + "devDependencies": { + "@types/node": "^20.19.0", + "typescript": "^5.9.2" + }, + "engines": { + "node": ">=20" + } + }, + "node_modules/@assemblyscript/loader": { + "version": "0.19.23", + "resolved": "https://registry.npmjs.org/@assemblyscript/loader/-/loader-0.19.23.tgz", + "integrity": "sha512-ulkCYfFbYj01ie1MDOyxv2F6SpRN1TOj7fQxbP07D6HmeR+gr2JLSmINKjga2emB+b1L2KGrFKBTc+e00p54nw==", + "license": "Apache-2.0" + }, + "node_modules/@ioredis/commands": { + "version": "1.10.0", + "resolved": "https://registry.npmjs.org/@ioredis/commands/-/commands-1.10.0.tgz", + "integrity": "sha512-UmeW7z4LfctwoQ5wkhVzgq8tXkreED2xZGpX+Bg+zA+WJFZCT6c062AfCK/Dfk81xZnnwdhJCUMkitihRaoC2Q==", + "license": "MIT" + }, + "node_modules/@iovalkey/commands": { + "version": "0.1.0", + "resolved": "https://registry.npmjs.org/@iovalkey/commands/-/commands-0.1.0.tgz", + "integrity": "sha512-/B9W4qKSSITDii5nkBCHyPkIkAi+ealUtr1oqBJsLxjSRLka4pxun2VvMNSmcwgAMxgXtQfl0qRv7TE+udPJzg==", + "license": "MIT" + }, + "node_modules/@protobufjs/aspromise": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/@protobufjs/aspromise/-/aspromise-1.1.2.tgz", + "integrity": "sha512-j+gKExEuLmKwvz3OgROXtrJ2UG2x8Ch2YZUxahh+s1F2HZ+wAceUNLkvy6zKCPVRkU++ZWQrdxsUeQXmcg4uoQ==", + "license": "BSD-3-Clause" + }, + "node_modules/@protobufjs/base64": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/@protobufjs/base64/-/base64-1.1.2.tgz", + "integrity": "sha512-AZkcAA5vnN/v4PDqKyMR5lx7hZttPDgClv83E//FMNhR2TMcLUhfRUBHCmSl0oi9zMgDDqRUJkSxO3wm85+XLg==", + "license": "BSD-3-Clause" + }, + "node_modules/@protobufjs/codegen": { + "version": "2.0.5", + "resolved": "https://registry.npmjs.org/@protobufjs/codegen/-/codegen-2.0.5.tgz", + "integrity": "sha512-zgXFLzW3Ap33e6d0Wlj4MGIm6Ce8O89n/apUaGNB/jx+hw+ruWEp7EwGUshdLKVRCxZW12fp9r40E1mQrf/34g==", + "license": "BSD-3-Clause" + }, + "node_modules/@protobufjs/eventemitter": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/@protobufjs/eventemitter/-/eventemitter-1.1.1.tgz", + "integrity": "sha512-vW1GmwMZNnL+gMRaovlh9yZX74kc+TTU3FObkkurpMaRtBfLP3ldjS9KQWlwZgraRE0+dheEEoAxdzcJQ8eXZg==", + "license": "BSD-3-Clause" + }, + "node_modules/@protobufjs/fetch": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/@protobufjs/fetch/-/fetch-1.1.1.tgz", + "integrity": "sha512-GpptLrs57adMSuHi3VNj0mAF8dwh36LMaYF6XyJ6JMWlVsc+t42tm1HSEDmOs3A8fC9yyeisgLhsTVQokOZ0zw==", + "license": "BSD-3-Clause", + "dependencies": { + "@protobufjs/aspromise": "^1.1.1" + } + }, + "node_modules/@protobufjs/float": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/@protobufjs/float/-/float-1.0.2.tgz", + "integrity": "sha512-Ddb+kVXlXst9d+R9PfTIxh1EdNkgoRe5tOX6t01f1lYWOvJnSPDBlG241QLzcyPdoNTsblLUdujGSE4RzrTZGQ==", + "license": "BSD-3-Clause" + }, + "node_modules/@protobufjs/path": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/@protobufjs/path/-/path-1.1.2.tgz", + "integrity": "sha512-6JOcJ5Tm08dOHAbdR3GrvP+yUUfkjG5ePsHYczMFLq3ZmMkAD98cDgcT2iA1lJ9NVwFd4tH/iSSoe44YWkltEA==", + "license": "BSD-3-Clause" + }, + "node_modules/@protobufjs/pool": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/@protobufjs/pool/-/pool-1.1.0.tgz", + "integrity": "sha512-0kELaGSIDBKvcgS4zkjz1PeddatrjYcmMWOlAuAPwAeccUrPHdUqo/J6LiymHHEiJT5NrF1UVwxY14f+fy4WQw==", + "license": "BSD-3-Clause" + }, + "node_modules/@protobufjs/utf8": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/@protobufjs/utf8/-/utf8-1.1.2.tgz", + "integrity": "sha512-b1UQwcEZ4yCnMCD8DAL1VlbvBJE9/IX4FTIp7BG1xYpf29SLazLSrqUkj4w7Y5y7cCVP6E5tcqqcI0xemPkHug==", + "license": "BSD-3-Clause" + }, + "node_modules/@types/node": { + "version": "20.19.43", + "resolved": "https://registry.npmjs.org/@types/node/-/node-20.19.43.tgz", + "integrity": "sha512-6oYBAi5ikg4Pl+kGsoYtawUMBT2zZMCvPNF7pVLnHZfd1zf38DRiWn/gT01RYCdUqkv7Fhr+C9ot4/tb+2sVvA==", + "license": "MIT", + "dependencies": { + "undici-types": "~6.21.0" + } + }, + "node_modules/@valkey/valkey-glide": { + "version": "2.5.2", + "resolved": "https://registry.npmjs.org/@valkey/valkey-glide/-/valkey-glide-2.5.2.tgz", + "integrity": "sha512-8d3m2NyveQ2Ws4OUd7Zz3zsPP835Nqi4Rm3ZMoLdUjTfnEJejIMPJGR8b09foiPEG4GXV9j+QWyDRuM9VHOuVw==", + "license": "Apache-2.0", + "dependencies": { + "long": "5", + "protobufjs": "^7.6.3" + }, + "engines": { + "node": ">=16" + }, + "optionalDependencies": { + "@valkey/valkey-glide-darwin-arm64": "2.5.2", + "@valkey/valkey-glide-darwin-x64": "2.5.2", + "@valkey/valkey-glide-linux-arm64-gnu": "2.5.2", + "@valkey/valkey-glide-linux-arm64-musl": "2.5.2", + "@valkey/valkey-glide-linux-x64-gnu": "2.5.2", + "@valkey/valkey-glide-linux-x64-musl": "2.5.2" + } + }, + "node_modules/@valkey/valkey-glide-darwin-arm64": { + "version": "2.5.2", + "resolved": "https://registry.npmjs.org/@valkey/valkey-glide-darwin-arm64/-/valkey-glide-darwin-arm64-2.5.2.tgz", + "integrity": "sha512-6kmFsyBrMj8kRF72ZiKpLsT0WZrRQdM3P2p9vJskRaXbRiMGmLVoqj8vv1tYcu/xRr7bI6aEONOlwKbHojRLkQ==", + "cpu": [ + "arm64" + ], + "license": "Apache-2.0", + "optional": true, + "os": [ + "darwin" + ] + }, + "node_modules/@valkey/valkey-glide-darwin-x64": { + "version": "2.5.2", + "resolved": "https://registry.npmjs.org/@valkey/valkey-glide-darwin-x64/-/valkey-glide-darwin-x64-2.5.2.tgz", + "integrity": "sha512-2RgllAJlYvoKirMKDIseYfX7LNFOF06UohqRB4LM2U3Y9mjE/ssNrPy+netFYdqz0aWcgYajzCMr2jXeckmoEg==", + "cpu": [ + "x64" + ], + "license": "Apache-2.0", + "optional": true, + "os": [ + "darwin" + ] + }, + "node_modules/@valkey/valkey-glide-linux-arm64-gnu": { + "version": "2.5.2", + "resolved": "https://registry.npmjs.org/@valkey/valkey-glide-linux-arm64-gnu/-/valkey-glide-linux-arm64-gnu-2.5.2.tgz", + "integrity": "sha512-EVWT7VT9porB670AxjQ+b/g/Dx5Nm/GaMy9irCzdDdnnTOwPyHvK1kPFzMATNuyDgXd5L504i99u8iHL1GpDEw==", + "cpu": [ + "arm64" + ], + "license": "Apache-2.0", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@valkey/valkey-glide-linux-arm64-musl": { + "version": "2.5.2", + "resolved": "https://registry.npmjs.org/@valkey/valkey-glide-linux-arm64-musl/-/valkey-glide-linux-arm64-musl-2.5.2.tgz", + "integrity": "sha512-CFTrYPWIrh+k66eqH3/FvUlMZ4S7p328iFTK+qsHxyYH3CnhAmKMaOHcptFKz5n7ypT8L/XuOBl+NIQgUrJGQA==", + "cpu": [ + "arm64" + ], + "license": "Apache-2.0", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@valkey/valkey-glide-linux-x64-gnu": { + "version": "2.5.2", + "resolved": "https://registry.npmjs.org/@valkey/valkey-glide-linux-x64-gnu/-/valkey-glide-linux-x64-gnu-2.5.2.tgz", + "integrity": "sha512-osWsmJ+cg2iZjjNZ5IHQUMg/Xb/TQVyBvDxKA11vpLuEFMulQYfUfw+LWo0PwnG2mGJGKHhDnZ8KZAJpd03LWQ==", + "cpu": [ + "x64" + ], + "license": "Apache-2.0", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@valkey/valkey-glide-linux-x64-musl": { + "version": "2.5.2", + "resolved": "https://registry.npmjs.org/@valkey/valkey-glide-linux-x64-musl/-/valkey-glide-linux-x64-musl-2.5.2.tgz", + "integrity": "sha512-gJCma4s3j3AUQk3k2Kcxe3ImWpla4e9jr41f1w9VwZd6CkWuZQdk1PsLpdROvtaBxUiRiq+RZvvLKvGf0md1Kg==", + "cpu": [ + "x64" + ], + "license": "Apache-2.0", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/base64-js": { + "version": "1.5.1", + "resolved": "https://registry.npmjs.org/base64-js/-/base64-js-1.5.1.tgz", + "integrity": "sha512-AKpaYlHn8t4SVbOHCy+b5+KKgvR4vrsD8vbvrbiQJps7fKDTkjkDry6ji0rUJjC0kzbNePLwzxq8iypo41qeWA==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ], + "license": "MIT" + }, + "node_modules/cluster-key-slot": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/cluster-key-slot/-/cluster-key-slot-1.1.1.tgz", + "integrity": "sha512-rwHwUfXL40Chm1r08yrhU3qpUvdVlgkKNeyeGPOxnW8/SyVDvgRaed/Uz54AqWNaTCAThlj6QAs3TZcKI0xDEw==", + "license": "Apache-2.0", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/debug": { + "version": "4.4.3", + "resolved": "https://registry.npmjs.org/debug/-/debug-4.4.3.tgz", + "integrity": "sha512-RGwwWnwQvkVfavKVt22FGLw+xYSdzARwm0ru6DhTVA3umU5hZc28V3kO4stgYryrTlLpuvgI9GiijltAjNbcqA==", + "license": "MIT", + "dependencies": { + "ms": "^2.1.3" + }, + "engines": { + "node": ">=6.0" + }, + "peerDependenciesMeta": { + "supports-color": { + "optional": true + } + } + }, + "node_modules/denque": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/denque/-/denque-2.1.0.tgz", + "integrity": "sha512-HVQE3AAb/pxF8fQAoiqpvg9i3evqug3hoiwakOyZAwJm+6vZehbkYXZ0l4JxS+I3QxM97v5aaRNhj8v5oBhekw==", + "license": "Apache-2.0", + "engines": { + "node": ">=0.10" + } + }, + "node_modules/hdr-histogram-js": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/hdr-histogram-js/-/hdr-histogram-js-3.0.1.tgz", + "integrity": "sha512-l3GSdZL1Jr1C0kyb461tUjEdrRPZr8Qry7jByltf5JGrA0xvqOSrxRBfcrJqqV/AMEtqqhHhC6w8HW0gn76tRQ==", + "license": "BSD-2-Clause", + "dependencies": { + "@assemblyscript/loader": "^0.19.21", + "base64-js": "^1.2.0", + "pako": "^1.0.3" + }, + "engines": { + "node": ">=14" + } + }, + "node_modules/ioredis": { + "version": "5.11.1", + "resolved": "https://registry.npmjs.org/ioredis/-/ioredis-5.11.1.tgz", + "integrity": "sha512-ehuGcf94bQXhfagULNXrJdfnWO38v070jxSx/qE87Kjzmu2fU7ro5EFAb+OPituLqgfyuQaym5DlrNydW2sJ9A==", + "license": "MIT", + "dependencies": { + "@ioredis/commands": "1.10.0", + "cluster-key-slot": "1.1.1", + "debug": "4.4.3", + "denque": "2.1.0", + "redis-errors": "1.2.0", + "redis-parser": "3.0.0", + "standard-as-callback": "2.1.0" + }, + "engines": { + "node": ">=12.22.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/ioredis" + } + }, + "node_modules/iovalkey": { + "version": "0.4.0", + "resolved": "https://registry.npmjs.org/iovalkey/-/iovalkey-0.4.0.tgz", + "integrity": "sha512-OSUKxJ+s44CLdUTaicX4+pVrBN/zHKIYwr2oKhRDuczr19FwcSCXEZHzcMUYWZNh1mNSCV9bNJHW/T6cHVm9Aw==", + "license": "MIT", + "dependencies": { + "@iovalkey/commands": "^0.1.0", + "cluster-key-slot": "^1.1.0", + "debug": "^4.3.4", + "denque": "^2.1.0", + "lodash.defaults": "^4.2.0", + "lodash.isarguments": "^3.1.0", + "redis-errors": "^1.2.0", + "redis-parser": "^3.0.0", + "standard-as-callback": "^2.1.0" + }, + "engines": { + "node": ">=18.12.0" + } + }, + "node_modules/lodash.defaults": { + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/lodash.defaults/-/lodash.defaults-4.2.0.tgz", + "integrity": "sha512-qjxPLHd3r5DnsdGacqOMU6pb/avJzdh9tFX2ymgoZE27BmjXrNy/y4LoaiTeAb+O3gL8AfpJGtqfX/ae2leYYQ==", + "license": "MIT" + }, + "node_modules/lodash.isarguments": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/lodash.isarguments/-/lodash.isarguments-3.1.0.tgz", + "integrity": "sha512-chi4NHZlZqZD18a0imDHnZPrDeBbTtVN7GXMwuGdRH9qotxAjYs3aVLKc7zNOG9eddR5Ksd8rvFEBc9SsggPpg==", + "license": "MIT" + }, + "node_modules/long": { + "version": "5.3.2", + "resolved": "https://registry.npmjs.org/long/-/long-5.3.2.tgz", + "integrity": "sha512-mNAgZ1GmyNhD7AuqnTG3/VQ26o760+ZYBPKjPvugO8+nLbYfX6TVpJPseBvopbdY+qpZ/lKUnmEc1LeZYS3QAA==", + "license": "Apache-2.0" + }, + "node_modules/ms": { + "version": "2.1.3", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", + "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==", + "license": "MIT" + }, + "node_modules/pako": { + "version": "1.0.11", + "resolved": "https://registry.npmjs.org/pako/-/pako-1.0.11.tgz", + "integrity": "sha512-4hLB8Py4zZce5s4yd9XzopqwVv/yGNhV1Bl8NTmCq1763HeK2+EwVTv+leGeL13Dnh2wfbqowVPXCIO0z4taYw==", + "license": "(MIT AND Zlib)" + }, + "node_modules/protobufjs": { + "version": "7.6.6", + "resolved": "https://registry.npmjs.org/protobufjs/-/protobufjs-7.6.6.tgz", + "integrity": "sha512-dYDWdjSl5RNb7SgPxGQcRU+GtvP7s2fpkrY0r432PcOIaZ0/rBcxEZnQN67iJhFuQiVw754JDoPruPCNdGsbjg==", + "hasInstallScript": true, + "license": "BSD-3-Clause", + "dependencies": { + "@protobufjs/aspromise": "^1.1.2", + "@protobufjs/base64": "^1.1.2", + "@protobufjs/codegen": "^2.0.5", + "@protobufjs/eventemitter": "^1.1.1", + "@protobufjs/fetch": "^1.1.1", + "@protobufjs/float": "^1.0.2", + "@protobufjs/path": "^1.1.2", + "@protobufjs/pool": "^1.1.0", + "@protobufjs/utf8": "^1.1.1", + "@types/node": ">=13.7.0", + "long": "^5.3.2" + }, + "engines": { + "node": ">=12.0.0" + } + }, + "node_modules/redis-errors": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/redis-errors/-/redis-errors-1.2.0.tgz", + "integrity": "sha512-1qny3OExCf0UvUV/5wpYKf2YwPcOqXzkwKKSmKHiE6ZMQs5heeE/c8eXK+PNllPvmjgAbfnsbpkGZWy8cBpn9w==", + "license": "MIT", + "engines": { + "node": ">=4" + } + }, + "node_modules/redis-parser": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/redis-parser/-/redis-parser-3.0.0.tgz", + "integrity": "sha512-DJnGAeenTdpMEH6uAJRK/uiyEIH9WVsUmoLwzudwGJUwZPp80PDBWPHXSAGNPwNvIXAbe7MSUB1zQFugFml66A==", + "license": "MIT", + "dependencies": { + "redis-errors": "^1.0.0" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/standard-as-callback": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/standard-as-callback/-/standard-as-callback-2.1.0.tgz", + "integrity": "sha512-qoRRSyROncaz1z0mvYqIE4lCd9p2R90i6GxW3uZv5ucSu8tU7B5HXUP1gG8pVZsYNVaXjk8ClXHPttLyxAL48A==", + "license": "MIT" + }, + "node_modules/typescript": { + "version": "5.9.3", + "resolved": "https://registry.npmjs.org/typescript/-/typescript-5.9.3.tgz", + "integrity": "sha512-jl1vZzPDinLr9eUt3J/t7V6FgNEw9QjvBPdysz9KfQDD41fQrC2Y4vKQdiaUpFT4bXlb1RHhLpp8wtm6M5TgSw==", + "dev": true, + "license": "Apache-2.0", + "bin": { + "tsc": "bin/tsc", + "tsserver": "bin/tsserver" + }, + "engines": { + "node": ">=14.17" + } + }, + "node_modules/undici-types": { + "version": "6.21.0", + "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-6.21.0.tgz", + "integrity": "sha512-iwDZqg0QAGrg9Rav5H4n0M64c3mkR59cJ6wQp+7C4nI0gsmExaedaYLNO44eT4AtBBwjbTiGPMlt2Md0T9H9JQ==", + "license": "MIT" + } + } +} diff --git a/node/package.json b/node/package.json new file mode 100644 index 0000000..e2b8b39 --- /dev/null +++ b/node/package.json @@ -0,0 +1,30 @@ +{ + "name": "resp-bench-node", + "version": "1.0.0", + "description": "resp-bench Node.js benchmark engine", + "license": "Apache-2.0", + "private": true, + "type": "module", + "engines": { + "node": ">=20" + }, + "scripts": { + "build": "tsc", + "clean": "rm -rf dist", + "typecheck": "tsc --noEmit", + "test": "npm run build && node --test dist/test/", + "test:unit": "npm run build && node --test dist/test/unit/", + "test:integration": "npm run build && node --test dist/test/integration/", + "start": "node dist/src/cli.js" + }, + "dependencies": { + "@valkey/valkey-glide": "2.5.2", + "hdr-histogram-js": "3.0.1", + "ioredis": "5.11.1", + "iovalkey": "0.4.0" + }, + "devDependencies": { + "@types/node": "20.19.43", + "typescript": "5.9.3" + } +} diff --git a/node/src/cli.ts b/node/src/cli.ts new file mode 100644 index 0000000..9c6fef3 --- /dev/null +++ b/node/src/cli.ts @@ -0,0 +1,142 @@ +/** + * Command-line interface for the resp-bench Node.js engine. + * + * Implements the shared cross-engine CLI contract: `--server`, `--driver`, + * `--workload`, `--metrics`, plus `--info`, `--commit-id` (used by CI), and + * `--version`. Deliberately no `--concurrency` flag: the engine has a single + * execution model, and pipelining comes from the workload's `pipeline_depth`. + */ + +import { existsSync } from 'node:fs'; +import { parseArgs } from 'node:util'; + +import { BenchmarkClientFactory } from './client/factory.js'; +import { CommandFactory } from './command/factory.js'; +import { ConfigLoader } from './config/loader.js'; +import { BenchmarkEngine } from './engine/benchmark.js'; +import { VERSION } from './version.js'; + +const DEFAULT_SERVER = 'localhost:6379'; +const DEFAULT_PORT = 6379; + +const USAGE = `resp-bench Node.js engine v${VERSION} + +Usage: + node dist/src/cli.js --driver --workload --metrics [options] + +Options: + --server Server address (default: ${DEFAULT_SERVER}) + --driver Driver configuration JSON (required) + --workload Workload configuration JSON (required) + --metrics Metrics NDJSON output path (required) + --commit-id Git commit ID recorded in the metrics metadata + --info Show supported drivers and commands + --version Show the engine version + --help Show this message +`; + +function parseServer(server: string): { host: string; port: number } { + const separator = server.lastIndexOf(':'); + if (separator === -1) return { host: server || 'localhost', port: DEFAULT_PORT }; + const host = server.slice(0, separator) || 'localhost'; + const port = Number(server.slice(separator + 1)); + if (!Number.isInteger(port) || port <= 0) { + throw new Error(`invalid --server "${server}": port must be a positive integer`); + } + return { host, port }; +} + +function printInfo(): void { + const lines = [ + `resp-bench Node.js Engine v${VERSION}`, + '', + 'Supported Drivers:', + ...BenchmarkClientFactory.describe().map( + ({ driverId, description }) => ` - ${driverId.padEnd(20)} : ${description}`, + ), + '', + 'Supported Commands:', + ...CommandFactory.describe().map(({ name, description }) => ` - ${name.padEnd(10)} : ${description}`), + '', + 'Supported Key Generation Algorithms:', + ' - sequential_int : Sequential integers (0 to keys_count), shared across connections', + ' - uniform_rand : Uniform random, java.util.Random-compatible per connection', + '', + 'Supported Completion Types:', + ' - duration : Run for the specified seconds', + ' - requests : Run until the shared request budget is exhausted', + '', + 'Concurrency: event-loop task-per-connection (one client per connection)', + ]; + console.log(lines.join('\n')); +} + +export async function main(argv: string[] = process.argv.slice(2)): Promise { + let options; + try { + ({ values: options } = parseArgs({ + args: argv, + options: { + server: { type: 'string', default: DEFAULT_SERVER }, + driver: { type: 'string' }, + workload: { type: 'string' }, + metrics: { type: 'string' }, + 'commit-id': { type: 'string' }, + info: { type: 'boolean', default: false }, + version: { type: 'boolean', default: false }, + help: { type: 'boolean', default: false }, + }, + strict: true, + })); + } catch (error) { + console.error(`Error: ${(error as Error).message}`); + console.error(USAGE); + return 1; + } + + if (options.help) { + console.log(USAGE); + return 0; + } + if (options.version) { + console.log(`resp-bench Node.js Engine v${VERSION}`); + return 0; + } + if (options.info) { + printInfo(); + return 0; + } + + try { + const missing = (['driver', 'workload', 'metrics'] as const).filter((flag) => !options[flag]); + if (missing.length > 0) { + throw new Error(`missing required options: ${missing.map((f) => `--${f}`).join(', ')}`); + } + for (const flag of ['driver', 'workload'] as const) { + if (!existsSync(options[flag]!)) { + throw new Error(`${flag} config not found: ${options[flag]}`); + } + } + + const { host, port } = parseServer(options.server!); + const engine = new BenchmarkEngine({ + host, + port, + driverConfig: ConfigLoader.loadDriverConfig(options.driver!), + workloadConfig: ConfigLoader.loadWorkloadConfig(options.workload!), + metricsPath: options.metrics!, + commitId: options['commit-id'] ?? null, + }); + await engine.run(); + return 0; + } catch (error) { + console.error(`Error: ${(error as Error).message}`); + if (process.env['DEBUG']) console.error((error as Error).stack); + return 1; + } +} + +// `import.meta.url` check keeps the module importable by tests without running. +if (process.argv[1] && import.meta.url === `file://${process.argv[1]}`) { + process.exitCode = await main(); +} diff --git a/node/src/client/benchmarkClient.ts b/node/src/client/benchmarkClient.ts new file mode 100644 index 0000000..214e655 --- /dev/null +++ b/node/src/client/benchmarkClient.ts @@ -0,0 +1,59 @@ +/** + * The interface every driver implements. + * + * One client instance maps to exactly one transport connection (the + * `client == connection` invariant shared by all engines); the engine never + * shares a client across workers. + * + * `measure()` is the single place latency is captured, so every driver reports it + * identically: `process.hrtime.bigint()` around the awaited command, truncated to + * whole microseconds, recorded even when the command throws. + */ + +import type { DriverConfig } from '../config/driverConfig.js'; +import type { TimedResult } from './timedResult.js'; + +export interface BenchmarkClient { + connect(host: string, port: number, config: DriverConfig): Promise; + ping(): Promise>; + get(key: string): Promise>; + set(key: string, value: Buffer): Promise>; + close(): Promise; + driverVersion(): string; + /** Secondary driver version, for composite drivers only. */ + secondaryDriverVersion?(): string | null; + /** + * Mark the client as warming up, mirroring Java's `setWarmupMode` + * (BenchmarkEngine.java:212-237). + * + * Real drivers ignore this. The recording driver uses it to suppress simulated + * errors so that an `error_rate` workload does not abort in warmup — the + * warmup fail-fast is meant to catch an unreachable server, not injected + * errors the phase is deliberately measuring. + */ + setWarmupMode?(warmup: boolean): void; +} + +const NANOS_PER_MICRO = 1000n; + +/** + * Await `operation` and record its latency in microseconds. + * + * Errors are captured, not thrown: the engine records a failed request and keeps + * going, matching the other engines. Latency is measured on the error path too. + */ +export async function measure(operation: () => Promise): Promise> { + const start = process.hrtime.bigint(); + try { + const value = await operation(); + const latencyMicros = Number((process.hrtime.bigint() - start) / NANOS_PER_MICRO); + return { value, latencyMicros }; + } catch (error) { + const latencyMicros = Number((process.hrtime.bigint() - start) / NANOS_PER_MICRO); + return { + value: null, + latencyMicros, + error: error instanceof Error ? error : new Error(String(error)), + }; + } +} diff --git a/node/src/client/driverVersion.ts b/node/src/client/driverVersion.ts new file mode 100644 index 0000000..bf7c933 --- /dev/null +++ b/node/src/client/driverVersion.ts @@ -0,0 +1,45 @@ +/** + * Reads an installed package's version for the NDJSON `metadata` block. + * + * `require('/package.json')` is NOT usable here: `@valkey/valkey-glide` + * declares an `exports` map with no `./package.json` entry, so the subpath import + * fails with ERR_PACKAGE_PATH_NOT_EXPORTED. Resolving the module's entry point + * and walking up to the nearest package.json with `fs` sidesteps `exports` + * entirely (it only governs specifier resolution, not file reads) and works for + * every driver. + */ + +import { readFileSync } from 'node:fs'; +import { createRequire } from 'node:module'; +import { dirname, join } from 'node:path'; + +const require = createRequire(import.meta.url); + +/** Depth to walk up from the resolved entry point looking for package.json. */ +const MAX_WALK_UP = 6; + +export function packageVersion(name: string): string { + let dir: string; + try { + dir = dirname(require.resolve(name)); + } catch { + return 'unknown'; + } + + for (let i = 0; i < MAX_WALK_UP; i++) { + try { + const manifest = JSON.parse(readFileSync(join(dir, 'package.json'), 'utf8')) as { + name?: string; + version?: string; + }; + // Guard against picking up a nested manifest of a different package. + if (manifest.name === name && typeof manifest.version === 'string') return manifest.version; + } catch { + // Not here (or unreadable) -- keep walking up. + } + const parent = dirname(dir); + if (parent === dir) break; + dir = parent; + } + return 'unknown'; +} diff --git a/node/src/client/factory.ts b/node/src/client/factory.ts new file mode 100644 index 0000000..a5dac3d --- /dev/null +++ b/node/src/client/factory.ts @@ -0,0 +1,81 @@ +/** + * Driver registry: maps `driver_id` to a client implementation. + * + * Implementations are loaded with a dynamic `import()` so `--info` and the unit + * tests work even if a driver's native bits are missing or broken on this + * platform -- only the driver actually requested gets loaded. + * + * The ids matter beyond this file. `DRIVER_ENGINE_MAP` in + * scripts/run_benchmark_matrix.py is a single global map shared by every engine, + * and `valkey-glide` there already means *Java*. Hence `valkey-glide-node`: a + * bare `valkey-glide` here would silently reroute Java's glide runs to Node. + */ + +import type { DriverConfig } from '../config/driverConfig.js'; +import type { BenchmarkClient } from './benchmarkClient.js'; + +interface DriverEntry { + description: string; + load: () => Promise; +} + +const DRIVERS = new Map([ + [ + 'valkey-glide-node', + { + description: 'Valkey GLIDE for Node.js (@valkey/valkey-glide)', + load: async () => new (await import('./impl/glideClient.js')).GlideBenchmarkClient(), + }, + ], + [ + 'ioredis', + { + description: 'ioredis — the most widely used Node.js Redis client', + load: async () => new (await import('./impl/ioredisClient.js')).IoredisBenchmarkClient(), + }, + ], + [ + 'iovalkey', + { + description: 'iovalkey — the Valkey-maintained fork of ioredis', + load: async () => new (await import('./impl/iovalkeyClient.js')).IovalkeyBenchmarkClient(), + }, + ], + [ + 'recording', + { + description: 'In-memory synthetic-latency client (no server required)', + load: async () => new (await import('./impl/recordingClient.js')).RecordingClient(), + }, + ], +]); + +export class BenchmarkClientFactory { + static supportedDrivers(): string[] { + return [...DRIVERS.keys()]; + } + + static describe(): Array<{ driverId: string; description: string }> { + return [...DRIVERS.entries()].map(([driverId, { description }]) => ({ driverId, description })); + } + + static async create(driverId: string): Promise { + const entry = DRIVERS.get((driverId ?? '').toLowerCase()); + if (entry === undefined) { + throw new Error( + `Unknown driver: ${driverId}. Supported: ${BenchmarkClientFactory.supportedDrivers().join(', ')}`, + ); + } + return entry.load(); + } + + static async createAndConnect( + host: string, + port: number, + config: DriverConfig, + ): Promise { + const client = await BenchmarkClientFactory.create(config.driverId); + await client.connect(host, port, config); + return client; + } +} diff --git a/node/src/client/impl/glideClient.ts b/node/src/client/impl/glideClient.ts new file mode 100644 index 0000000..74e3d2d --- /dev/null +++ b/node/src/client/impl/glideClient.ts @@ -0,0 +1,88 @@ +/** + * valkey-glide driver (`@valkey/valkey-glide`). + * + * One GlideClient per connection, honouring the `client == connection` invariant + * shared across engines. + * + * Two glide-specific shapes to note: + * - `close()` is **synchronous** (returns void, not a Promise), unlike every + * other driver here. + * - `get()` returns a `string` by default. That is deliberate and matches the + * Python engine and the ioredis/iovalkey clients here, so all drivers are + * charged for the same UTF-8 decode. Do not switch one driver to bytes. + */ + +import type { + GlideClient, + GlideClientConfiguration, + GlideClusterClient, + GlideClusterClientConfiguration, + ServerCredentials, +} from '@valkey/valkey-glide'; + +import type { DriverConfig } from '../../config/driverConfig.js'; +import { measure, type BenchmarkClient } from '../benchmarkClient.js'; +import { packageVersion } from '../driverVersion.js'; +import type { TimedResult } from '../timedResult.js'; + +const PACKAGE = '@valkey/valkey-glide'; + +export class GlideBenchmarkClient implements BenchmarkClient { + private client: GlideClient | GlideClusterClient | null = null; + + async connect(host: string, port: number, config: DriverConfig): Promise { + const glide = await import('@valkey/valkey-glide'); + + const addresses = [{ host, port }]; + const credentials: ServerCredentials | undefined = config.hasAuth() + ? ({ + password: config.auth?.password ?? '', + ...(config.auth?.username ? { username: config.auth.username } : {}), + } as ServerCredentials) + : undefined; + + const shared = { + addresses, + useTLS: config.tlsEnabled(), + ...(credentials ? { credentials } : {}), + ...(config.commandTimeoutMs ? { requestTimeout: config.commandTimeoutMs } : {}), + }; + + this.client = config.isCluster() + ? await glide.GlideClusterClient.createClient(shared as GlideClusterClientConfiguration) + : await glide.GlideClient.createClient(shared as GlideClientConfiguration); + } + + private requireClient(): GlideClient | GlideClusterClient { + if (this.client === null) throw new Error('glide client is not connected'); + return this.client; + } + + async ping(): Promise> { + const client = this.requireClient(); + return measure(async () => String(await client.ping())); + } + + async get(key: string): Promise> { + const client = this.requireClient(); + return measure(async () => { + const value = await client.get(key); + return value === null ? null : String(value); + }) as Promise>; + } + + async set(key: string, value: Buffer): Promise> { + const client = this.requireClient(); + return measure(async () => String(await client.set(key, value))); + } + + async close(): Promise { + // Synchronous in glide -- there is nothing to await. + this.client?.close(); + this.client = null; + } + + driverVersion(): string { + return packageVersion(PACKAGE); + } +} diff --git a/node/src/client/impl/ioredisClient.ts b/node/src/client/impl/ioredisClient.ts new file mode 100644 index 0000000..85b609b --- /dev/null +++ b/node/src/client/impl/ioredisClient.ts @@ -0,0 +1,16 @@ +/** ioredis driver — the most widely used Node.js Redis client. */ + +import { IoredisFamilyClient, type RedisModuleLike } from './ioredisFamilyClient.js'; + +export class IoredisBenchmarkClient extends IoredisFamilyClient { + protected override packageName(): string { + return 'ioredis'; + } + + protected override async loadModule(): Promise { + const module = await import('ioredis'); + // ioredis' constructors carry overloads that RedisModuleLike narrows to the + // one form this engine calls; the shapes are compatible at runtime. + return { Redis: module.Redis, Cluster: module.Cluster } as unknown as RedisModuleLike; + } +} diff --git a/node/src/client/impl/ioredisFamilyClient.ts b/node/src/client/impl/ioredisFamilyClient.ts new file mode 100644 index 0000000..2959874 --- /dev/null +++ b/node/src/client/impl/ioredisFamilyClient.ts @@ -0,0 +1,131 @@ +/** + * Shared implementation for the ioredis-family drivers. + * + * `ioredis` and `iovalkey` (the Valkey-maintained fork) expose the same + * constructor options and the same `Redis`/`Cluster` exports, so both drivers + * differ only in which module they load and which package they report a version + * for. They are described structurally here rather than against either package's + * types, so neither becomes a compile-time dependency of the other's driver. + * + * Fairness note: `enableAutoPipelining` is forced **off**. Left on (it is off by + * default, but that default has changed before) ioredis transparently batches + * commands issued in the same event-loop tick, which would inflate throughput + * against every other engine and silently make the comparison meaningless. + */ + +import { readFileSync } from 'node:fs'; + +import type { DriverConfig } from '../../config/driverConfig.js'; +import { measure, type BenchmarkClient } from '../benchmarkClient.js'; +import { packageVersion } from '../driverVersion.js'; +import type { TimedResult } from '../timedResult.js'; + +/** The slice of the ioredis surface this engine uses. */ +interface RedisLike { + connect(): Promise; + ping(): Promise; + get(key: string): Promise; + set(key: string, value: Buffer): Promise; + quit(): Promise; + disconnect(): void; +} + +export interface RedisModuleLike { + Redis: new (options: Record) => RedisLike; + Cluster: new ( + nodes: Array<{ host: string; port: number }>, + options: Record, + ) => RedisLike; +} + +function buildTlsOptions(config: DriverConfig): Record | undefined { + if (!config.tlsEnabled()) return undefined; + const tls: Record = {}; + if (config.tls?.ca_path) tls['ca'] = readFileSync(config.tls.ca_path); + if (config.tls?.cert_path) tls['cert'] = readFileSync(config.tls.cert_path); + if (config.tls?.key_path) tls['key'] = readFileSync(config.tls.key_path); + if (config.tls?.verify_hostname === false) tls['rejectUnauthorized'] = false; + return tls; +} + +export abstract class IoredisFamilyClient implements BenchmarkClient { + private client: RedisLike | null = null; + + /** npm package name, used for both loading and version reporting. */ + protected abstract packageName(): string; + + protected abstract loadModule(): Promise; + + async connect(host: string, port: number, config: DriverConfig): Promise { + const module = await this.loadModule(); + const tls = buildTlsOptions(config); + + const options: Record = { + // Batching commands issued in one tick would not be comparable to the + // other engines -- keep every request a distinct round trip. + enableAutoPipelining: false, + // Connect explicitly below so a connection failure surfaces here rather + // than as a first-command error, and so cps_limit really gates setup. + lazyConnect: true, + // Fail a stuck request instead of retrying it under a different latency. + maxRetriesPerRequest: 0, + // Never silently reconnect. ioredis' default retryStrategy retries + // forever, so a wrong host would hang the run instead of failing it, and a + // mid-phase reconnect would fold connection setup into request latency. + retryStrategy: () => null, + // Bound the initial connect so an unreachable host fails fast. + connectTimeout: config.commandTimeoutMs ?? 10_000, + ...(config.hasAuth() && config.auth?.username ? { username: config.auth.username } : {}), + ...(config.hasAuth() && config.auth?.password ? { password: config.auth.password } : {}), + ...(config.commandTimeoutMs ? { commandTimeout: config.commandTimeoutMs } : {}), + ...(tls ? { tls } : {}), + }; + + if (config.isCluster()) { + this.client = new module.Cluster([{ host, port }], { + lazyConnect: true, + redisOptions: options, + }); + } else { + this.client = new module.Redis({ ...options, host, port }); + } + await this.client.connect(); + } + + private requireClient(): RedisLike { + if (this.client === null) throw new Error(`${this.packageName()} client is not connected`); + return this.client; + } + + async ping(): Promise> { + const client = this.requireClient(); + return measure(() => client.ping()); + } + + async get(key: string): Promise> { + const client = this.requireClient(); + return measure(() => client.get(key)) as Promise>; + } + + async set(key: string, value: Buffer): Promise> { + const client = this.requireClient(); + return measure(() => client.set(key, value)) as Promise>; + } + + async close(): Promise { + if (this.client === null) return; + const client = this.client; + this.client = null; + try { + await client.quit(); + } catch { + // A server that already dropped the connection makes QUIT reject; the + // socket still has to go, or the process will not exit. + client.disconnect(); + } + } + + driverVersion(): string { + return packageVersion(this.packageName()); + } +} diff --git a/node/src/client/impl/iovalkeyClient.ts b/node/src/client/impl/iovalkeyClient.ts new file mode 100644 index 0000000..0340d77 --- /dev/null +++ b/node/src/client/impl/iovalkeyClient.ts @@ -0,0 +1,19 @@ +/** + * iovalkey driver — the Valkey-maintained fork of ioredis. + * + * API-identical to ioredis, so all behaviour lives in the shared base class. + */ + +import { IoredisFamilyClient, type RedisModuleLike } from './ioredisFamilyClient.js'; + +export class IovalkeyBenchmarkClient extends IoredisFamilyClient { + protected override packageName(): string { + return 'iovalkey'; + } + + protected override async loadModule(): Promise { + const module = await import('iovalkey'); + // Same overload-narrowing as the ioredis driver; iovalkey mirrors its API. + return { Redis: module.Redis, Cluster: module.Cluster } as unknown as RedisModuleLike; + } +} diff --git a/node/src/client/impl/recordingClient.ts b/node/src/client/impl/recordingClient.ts new file mode 100644 index 0000000..2b60618 --- /dev/null +++ b/node/src/client/impl/recordingClient.ts @@ -0,0 +1,131 @@ +/** + * In-memory recording client for server-free testing. + * + * Records operations and supports simulated latency and error injection via + * `specific_driver_config` (`operation_delay_micros`, `delay_variation_micros`, + * `error_rate`, `error_message`). This lets the integration tests exercise the + * whole engine without a live server, mirroring the Ruby/Python recording + * drivers. + */ + +import type { DriverConfig } from '../../config/driverConfig.js'; +import type { BenchmarkClient } from '../benchmarkClient.js'; +import type { TimedResult } from '../timedResult.js'; + +export interface RecordedOperation { + command: string; + key: string | null; + value: Buffer | null; + success: boolean; + errorMessage: string | null; +} + +const NANOS_PER_MICRO = 1000n; + +function readNumber(source: Record, key: string, fallback: number): number { + const value = source[key]; + return typeof value === 'number' && Number.isFinite(value) ? value : fallback; +} + +/** Sleep for whole microseconds, tolerating setTimeout's ~1ms floor. */ +function sleepMicros(micros: number): Promise { + return new Promise((resolve) => setTimeout(resolve, micros / 1000)); +} + +export class RecordingClient implements BenchmarkClient { + readonly operations: RecordedOperation[] = []; + private readonly storedData = new Map(); + private operationDelayMicros = 0; + private delayVariationMicros = 0; + private errorRate = 0; + private errorMessage = 'Simulated error'; + private warmupMode = false; + + async connect(_host: string, _port: number, config: DriverConfig): Promise { + const cfg = config.specificDriverConfig; + this.operationDelayMicros = readNumber(cfg, 'operation_delay_micros', 0); + this.delayVariationMicros = readNumber(cfg, 'delay_variation_micros', 0); + this.errorRate = readNumber(cfg, 'error_rate', 0); + const message = cfg['error_message']; + if (typeof message === 'string') this.errorMessage = message; + this.record('CONNECT', null, null, true, null); + } + + async ping(): Promise> { + const { latencyMicros, success } = await this.simulate(); + this.record('PING', null, null, success, success ? null : this.errorMessage); + return success + ? { value: 'PONG', latencyMicros } + : { value: null, latencyMicros, error: new Error(this.errorMessage) }; + } + + async get(key: string): Promise> { + const { latencyMicros, success } = await this.simulate(); + this.record('GET', key, null, success, success ? null : this.errorMessage); + if (!success) return { value: null, latencyMicros, error: new Error(this.errorMessage) }; + const stored = this.storedData.get(key); + return { value: stored === undefined ? null : stored.toString('latin1'), latencyMicros }; + } + + async set(key: string, value: Buffer): Promise> { + const { latencyMicros, success } = await this.simulate(); + if (success) this.storedData.set(key, value); + this.record('SET', key, value, success, success ? null : this.errorMessage); + return success + ? { value: 'OK', latencyMicros } + : { value: null, latencyMicros, error: new Error(this.errorMessage) }; + } + + async close(): Promise { + this.record('CLOSE', null, null, true, null); + } + + driverVersion(): string { + return '1.0.0'; + } + + /** + * Suppress simulated errors during warmup, matching Java's recording client. + * The engine's warmup fail-fast exists to catch an unreachable server, so + * injected errors must not trip it. + */ + setWarmupMode(warmup: boolean): void { + this.warmupMode = warmup; + } + + private record( + command: string, + key: string | null, + value: Buffer | null, + success: boolean, + errorMessage: string | null, + ): void { + this.operations.push({ command, key, value, success, errorMessage }); + } + + private async simulate(): Promise<{ latencyMicros: number; success: boolean }> { + const start = process.hrtime.bigint(); + const delayMicros = this.calculateDelayMicros(); + if (delayMicros > 0) await sleepMicros(delayMicros); + const latencyMicros = Number((process.hrtime.bigint() - start) / NANOS_PER_MICRO); + return { latencyMicros, success: !this.shouldSimulateError() }; + } + + private calculateDelayMicros(): number { + if (this.operationDelayMicros <= 0) return 0; + let delay = this.operationDelayMicros; + if (this.delayVariationMicros > 0) { + const spread = 2 * this.delayVariationMicros + 1; + const variation = Math.floor(Math.random() * spread) - this.delayVariationMicros; + delay = Math.max(0, delay + variation); + } + return delay; + } + + private shouldSimulateError(): boolean { + if (this.warmupMode) return false; + if (this.errorRate <= 0) return false; + if (this.errorRate >= 1) return true; + return Math.random() < this.errorRate; + } +} diff --git a/node/src/client/timedResult.ts b/node/src/client/timedResult.ts new file mode 100644 index 0000000..3d36c5a --- /dev/null +++ b/node/src/client/timedResult.ts @@ -0,0 +1,12 @@ +/** + * The outcome of a single measured command. + * + * `error === undefined` means success. Latency is populated on both paths so a + * failure still contributes a measured duration, matching the other engines. + */ +export interface TimedResult { + readonly value: T | null; + /** Command latency in whole microseconds. Recorded even on error. */ + readonly latencyMicros: number; + readonly error?: Error; +} diff --git a/node/src/command/command.ts b/node/src/command/command.ts new file mode 100644 index 0000000..8c74699 --- /dev/null +++ b/node/src/command/command.ts @@ -0,0 +1,19 @@ +/** A benchmark command: a weighted operation the engine can issue. */ + +import type { BenchmarkClient } from '../client/benchmarkClient.js'; + +export interface CommandResult { + readonly commandName: string; + readonly latencyMicros: number; + readonly success: boolean; + readonly errorMessage?: string; +} + +export interface Command { + /** Upper-case command name, used as the NDJSON metrics key (GET/SET/PING). */ + readonly name: string; + readonly weight: number; + /** Whether this command consumes a generated key (PING does not). */ + readonly usesKey: boolean; + execute(client: BenchmarkClient, key: string): Promise; +} diff --git a/node/src/command/factory.ts b/node/src/command/factory.ts new file mode 100644 index 0000000..635a85a --- /dev/null +++ b/node/src/command/factory.ts @@ -0,0 +1,39 @@ +/** Maps a workload's `command` strings to Command implementations. */ + +import type { CommandConfig } from '../config/commandConfig.js'; +import type { Command } from './command.js'; +import { GetCommand } from './impl/getCommand.js'; +import { PingCommand } from './impl/pingCommand.js'; +import { SetCommand } from './impl/setCommand.js'; + +type CommandBuilder = (config: CommandConfig) => Command; + +const BUILDERS = new Map([ + ['get', { build: (c) => new GetCommand(c), description: 'GET key' }], + ['set', { build: (c) => new SetCommand(c), description: 'SET key value' }], + ['ping', { build: (c) => new PingCommand(c), description: 'PING' }], +]); + +export class CommandFactory { + static supportedCommands(): string[] { + return [...BUILDERS.keys()]; + } + + static describe(): Array<{ name: string; description: string }> { + return [...BUILDERS.entries()].map(([name, { description }]) => ({ name, description })); + } + + static create(config: CommandConfig): Command { + const entry = BUILDERS.get(config.command.toLowerCase()); + if (entry === undefined) { + throw new Error( + `Unknown command: ${config.command}. Supported: ${CommandFactory.supportedCommands().join(', ')}`, + ); + } + return entry.build(config); + } + + static createAll(configs: CommandConfig[]): Command[] { + return configs.map((config) => CommandFactory.create(config)); + } +} diff --git a/node/src/command/impl/getCommand.ts b/node/src/command/impl/getCommand.ts new file mode 100644 index 0000000..723e512 --- /dev/null +++ b/node/src/command/impl/getCommand.ts @@ -0,0 +1,23 @@ +import type { BenchmarkClient } from '../../client/benchmarkClient.js'; +import type { CommandConfig } from '../../config/commandConfig.js'; +import type { Command, CommandResult } from '../command.js'; + +export class GetCommand implements Command { + readonly name = 'GET'; + readonly weight: number; + readonly usesKey = true; + + constructor(config: CommandConfig) { + this.weight = config.weight; + } + + async execute(client: BenchmarkClient, key: string): Promise { + const result = await client.get(key); + return { + commandName: this.name, + latencyMicros: result.latencyMicros, + success: result.error === undefined, + ...(result.error ? { errorMessage: result.error.message } : {}), + }; + } +} diff --git a/node/src/command/impl/pingCommand.ts b/node/src/command/impl/pingCommand.ts new file mode 100644 index 0000000..86ac70f --- /dev/null +++ b/node/src/command/impl/pingCommand.ts @@ -0,0 +1,23 @@ +import type { BenchmarkClient } from '../../client/benchmarkClient.js'; +import type { CommandConfig } from '../../config/commandConfig.js'; +import type { Command, CommandResult } from '../command.js'; + +export class PingCommand implements Command { + readonly name = 'PING'; + readonly weight: number; + readonly usesKey = false; + + constructor(config: CommandConfig) { + this.weight = config.weight; + } + + async execute(client: BenchmarkClient): Promise { + const result = await client.ping(); + return { + commandName: this.name, + latencyMicros: result.latencyMicros, + success: result.error === undefined, + ...(result.error ? { errorMessage: result.error.message } : {}), + }; + } +} diff --git a/node/src/command/impl/setCommand.ts b/node/src/command/impl/setCommand.ts new file mode 100644 index 0000000..eae42ff --- /dev/null +++ b/node/src/command/impl/setCommand.ts @@ -0,0 +1,41 @@ +import type { BenchmarkClient } from '../../client/benchmarkClient.js'; +import type { CommandConfig } from '../../config/commandConfig.js'; +import type { Command, CommandResult } from '../command.js'; + +/** + * Deterministic filler pattern, matching the Ruby and Python engines. Java uses + * random bytes instead, but only the payload *length* affects RESP framing and + * server work, and a fixed pattern makes runs reproducible. + */ +const PATTERN = '0123456789ABCDEF'; + +export class SetCommand implements Command { + readonly name = 'SET'; + readonly weight: number; + readonly usesKey = true; + /** + * Built once at construction, not per request: allocating a fresh payload in + * the hot loop would charge the driver for the engine's own GC churn. + */ + private readonly value: Buffer; + + constructor(config: CommandConfig) { + this.weight = config.weight; + this.value = SetCommand.generateValue(config.dataSizeBytes); + } + + async execute(client: BenchmarkClient, key: string): Promise { + const result = await client.set(key, this.value); + return { + commandName: this.name, + latencyMicros: result.latencyMicros, + success: result.error === undefined, + ...(result.error ? { errorMessage: result.error.message } : {}), + }; + } + + static generateValue(size: number): Buffer { + const repeats = Math.floor(size / PATTERN.length) + 1; + return Buffer.from(PATTERN.repeat(repeats).slice(0, size), 'latin1'); + } +} diff --git a/node/src/config/commandConfig.ts b/node/src/config/commandConfig.ts new file mode 100644 index 0000000..eb85f02 --- /dev/null +++ b/node/src/config/commandConfig.ts @@ -0,0 +1,21 @@ +/** A single weighted command entry within a phase. */ + +/** + * Default SET payload size. 256 is the cross-engine default: Java + * (SetCommand.java `getDataSizeBytesOrDefault(256)`), C# (SetCommand.cs), + * Ruby and Python all use it. Do not change without changing them too. + */ +export const DEFAULT_DATA_SIZE_BYTES = 256; + +export class CommandConfig { + readonly command: string; + readonly weight: number; + readonly dataSizeBytes: number; + + constructor(init: { command: string; weight?: number | null; dataSizeBytes?: number | null }) { + this.command = init.command.toLowerCase(); + // A missing weight defaults to 1.0 (matching Java), rather than NaN. + this.weight = init.weight ?? 1.0; + this.dataSizeBytes = init.dataSizeBytes ?? DEFAULT_DATA_SIZE_BYTES; + } +} diff --git a/node/src/config/completionConfig.ts b/node/src/config/completionConfig.ts new file mode 100644 index 0000000..1dd8770 --- /dev/null +++ b/node/src/config/completionConfig.ts @@ -0,0 +1,29 @@ +/** Phase completion criteria: run for a duration, or until a request count. */ + +export class CompletionConfig { + readonly type: string; + readonly seconds: number | null; + readonly requests: number | null; + + constructor(init: { type: string; seconds?: number | null; requests?: number | null }) { + this.type = init.type; + this.seconds = init.seconds ?? null; + this.requests = init.requests ?? null; + } + + isDurationBased(): boolean { + return this.type === 'duration'; + } + + isRequestBased(): boolean { + return this.type === 'requests'; + } + + durationSeconds(): number { + return this.seconds ?? 0; + } + + totalRequests(): number { + return this.requests ?? 0; + } +} diff --git a/node/src/config/driverConfig.ts b/node/src/config/driverConfig.ts new file mode 100644 index 0000000..c432107 --- /dev/null +++ b/node/src/config/driverConfig.ts @@ -0,0 +1,80 @@ +/** + * Driver (client library) configuration. + * + * Maps to configs/schemas/driver-config.schema.json. Field names and defaults + * mirror the Java/Ruby/Python engines so the same JSON files work unchanged + * across every engine. + */ + +export interface TlsConfig { + enabled?: boolean; + cert_path?: string; + key_path?: string; + ca_path?: string; + verify_hostname?: boolean; +} + +export interface AuthConfig { + username?: string; + password?: string; +} + +export const DEFAULT_MODE = 'standalone'; + +export class DriverConfig { + readonly schemaVersion: string; + readonly description: string | null; + readonly driverId: string; + readonly mode: string; + readonly commandTimeoutMs: number | null; + readonly tls: TlsConfig | null; + readonly auth: AuthConfig | null; + readonly specificDriverConfig: Record; + + constructor(init: { + schemaVersion?: string; + description?: string | null; + driverId: string; + mode?: string | null; + commandTimeoutMs?: number | null; + tls?: TlsConfig | null; + auth?: AuthConfig | null; + specificDriverConfig?: Record | null; + }) { + this.schemaVersion = init.schemaVersion ?? '1.0'; + this.description = init.description ?? null; + this.driverId = init.driverId; + this.mode = init.mode ?? DEFAULT_MODE; + this.commandTimeoutMs = init.commandTimeoutMs ?? null; + this.tls = init.tls ?? null; + this.auth = init.auth ?? null; + this.specificDriverConfig = init.specificDriverConfig ?? {}; + } + + /** Secondary driver id for composite drivers (e.g. Java's spring-data-*). */ + secondaryDriverId(): string | null { + const value = this.specificDriverConfig['secondary_driver_id']; + return typeof value === 'string' ? value : null; + } + + isStandalone(): boolean { + return this.mode === 'standalone'; + } + + isCluster(): boolean { + return this.mode === 'cluster'; + } + + isSentinel(): boolean { + return this.mode === 'sentinel'; + } + + tlsEnabled(): boolean { + return this.tls?.enabled === true; + } + + /** Username/password only count as auth when at least one is non-empty. */ + hasAuth(): boolean { + return Boolean(this.auth && (this.auth.password || this.auth.username)); + } +} diff --git a/node/src/config/keyspaceConfig.ts b/node/src/config/keyspaceConfig.ts new file mode 100644 index 0000000..b20b4f9 --- /dev/null +++ b/node/src/config/keyspaceConfig.ts @@ -0,0 +1,41 @@ +/** Key-generation configuration for a benchmark phase. */ + +export const DEFAULT_KEY_SIZE_BYTES = 16; +export const DEFAULT_KEY_PREFIX = 'bench:'; +export const DEFAULT_GENERATION_ALG = 'sequential_int'; + +export class KeyspaceConfig { + readonly keysCount: number; + readonly keySizeBytes: number; + readonly keyPrefix: string; + readonly generationAlg: string; + readonly seed: number | null; + + constructor(init: { + keysCount: number; + keySizeBytes?: number | null; + keyPrefix?: string | null; + generationAlg?: string | null; + seed?: number | null; + }) { + this.keysCount = init.keysCount; + // A null/undefined value falls back to the default rather than staying + // null, matching the Ruby/Python engines. + this.keySizeBytes = init.keySizeBytes ?? DEFAULT_KEY_SIZE_BYTES; + this.keyPrefix = init.keyPrefix ?? DEFAULT_KEY_PREFIX; + this.generationAlg = init.generationAlg ?? DEFAULT_GENERATION_ALG; + this.seed = init.seed ?? null; + } + + isSequentialInt(): boolean { + return this.generationAlg === 'sequential_int'; + } + + isUniformRand(): boolean { + return this.generationAlg === 'uniform_rand'; + } + + seedValue(): number { + return this.seed ?? 0; + } +} diff --git a/node/src/config/loader.ts b/node/src/config/loader.ts new file mode 100644 index 0000000..000a914 --- /dev/null +++ b/node/src/config/loader.ts @@ -0,0 +1,198 @@ +/** + * Loads driver and workload configuration from the shared configs/ JSON files. + * + * Field names and defaults mirror the Java/Ruby/Python engines exactly, so the + * same JSON is consumed identically by every engine. Unlike those engines this + * one also validates the required fields up front: a typo'd config should fail + * with a clear message rather than surface later as a null-shaped error deep in + * a worker loop. + */ + +import { readFileSync } from 'node:fs'; + +import { CommandConfig } from './commandConfig.js'; +import { CompletionConfig } from './completionConfig.js'; +import { DriverConfig } from './driverConfig.js'; +import { KeyspaceConfig } from './keyspaceConfig.js'; +import { PhaseConfig } from './phaseConfig.js'; +import { WorkloadConfig } from './workloadConfig.js'; + +type Json = Record; + +export class ConfigError extends Error {} + +function asRecord(value: unknown, what: string): Json { + if (value === null || typeof value !== 'object' || Array.isArray(value)) { + throw new ConfigError(`${what} must be a JSON object`); + } + return value as Json; +} + +function optString(data: Json, key: string): string | null { + const value = data[key]; + if (value === undefined || value === null) return null; + if (typeof value !== 'string') throw new ConfigError(`"${key}" must be a string`); + return value; +} + +function reqString(data: Json, key: string, what: string): string { + const value = optString(data, key); + if (value === null || value === '') throw new ConfigError(`${what} is missing required "${key}"`); + return value; +} + +function optNumber(data: Json, key: string): number | null { + const value = data[key]; + if (value === undefined || value === null) return null; + if (typeof value !== 'number' || !Number.isFinite(value)) { + throw new ConfigError(`"${key}" must be a finite number`); + } + return value; +} + +function reqInt(data: Json, key: string, what: string): number { + const value = optNumber(data, key); + if (value === null) throw new ConfigError(`${what} is missing required "${key}"`); + if (!Number.isInteger(value)) throw new ConfigError(`"${key}" must be an integer`); + return value; +} + +function readJson(path: string, what: string): Json { + let text: string; + try { + text = readFileSync(path, 'utf8'); + } catch (error) { + throw new ConfigError(`cannot read ${what} "${path}": ${(error as Error).message}`); + } + try { + return asRecord(JSON.parse(text), what); + } catch (error) { + if (error instanceof ConfigError) throw error; + throw new ConfigError(`${what} "${path}" is not valid JSON: ${(error as Error).message}`); + } +} + +export class ConfigLoader { + static loadDriverConfig(path: string): DriverConfig { + return ConfigLoader.parseDriverConfig(readJson(path, 'driver config')); + } + + static loadWorkloadConfig(path: string): WorkloadConfig { + return ConfigLoader.parseWorkloadConfig(readJson(path, 'workload config')); + } + + static parseDriverConfig(data: Json): DriverConfig { + const mode = optString(data, 'mode') ?? 'standalone'; + if (!['standalone', 'cluster', 'sentinel'].includes(mode)) { + throw new ConfigError( + `driver config "mode" must be standalone, cluster or sentinel (got "${mode}")`, + ); + } + return new DriverConfig({ + schemaVersion: optString(data, 'schema_version') ?? '1.0', + description: optString(data, 'description'), + driverId: reqString(data, 'driver_id', 'driver config'), + mode, + commandTimeoutMs: optNumber(data, 'command_timeout_ms'), + tls: (data['tls'] ?? null) as DriverConfig['tls'], + auth: (data['auth'] ?? null) as DriverConfig['auth'], + specificDriverConfig: (data['specific_driver_config'] ?? {}) as Record, + }); + } + + static parseWorkloadConfig(data: Json): WorkloadConfig { + const rawPhases = data['phases']; + if (!Array.isArray(rawPhases) || rawPhases.length === 0) { + throw new ConfigError('workload config must have a non-empty "phases" array'); + } + return new WorkloadConfig({ + schemaVersion: optString(data, 'schema_version') ?? '1.0', + benchmarkProfile: (data['benchmark_profile'] ?? {}) as WorkloadConfig['benchmarkProfile'], + phases: rawPhases.map((phase, index) => + ConfigLoader.parsePhase(asRecord(phase, `phases[${index}]`), index), + ), + }); + } + + static parsePhase(data: Json, index = 0): PhaseConfig { + const what = `phases[${index}]`; + const rawCommands = data['commands']; + if (!Array.isArray(rawCommands) || rawCommands.length === 0) { + throw new ConfigError(`${what} must have a non-empty "commands" array`); + } + const connections = reqInt(data, 'connections', what); + if (connections <= 0) throw new ConfigError(`${what} "connections" must be positive`); + + return new PhaseConfig({ + id: reqString(data, 'id', what), + description: optString(data, 'description'), + connections, + cpsLimit: optNumber(data, 'cps_limit'), + rpsLimit: optNumber(data, 'rps_limit'), + pipelineDepth: optNumber(data, 'pipeline_depth'), + warmupRequests: optNumber(data, 'warmup_requests'), + completion: ConfigLoader.parseCompletion( + asRecord(data['completion'] ?? {}, `${what}.completion`), + what, + ), + keyspace: ConfigLoader.parseKeyspace( + asRecord(data['keyspace'] ?? {}, `${what}.keyspace`), + what, + ), + commands: rawCommands.map((command, i) => + ConfigLoader.parseCommand(asRecord(command, `${what}.commands[${i}]`), `${what}.commands[${i}]`), + ), + }); + } + + static parseCompletion(data: Json, what = 'completion'): CompletionConfig { + const type = reqString(data, 'type', `${what}.completion`); + if (!['duration', 'requests'].includes(type)) { + throw new ConfigError( + `${what}.completion "type" must be duration or requests (got "${type}")`, + ); + } + const seconds = optNumber(data, 'seconds'); + const requests = optNumber(data, 'requests'); + if (type === 'duration' && (seconds === null || seconds <= 0)) { + throw new ConfigError(`${what}.completion type=duration requires a positive "seconds"`); + } + if (type === 'requests' && (requests === null || requests <= 0)) { + throw new ConfigError(`${what}.completion type=requests requires a positive "requests"`); + } + return new CompletionConfig({ type, seconds, requests }); + } + + static parseKeyspace(data: Json, what = 'keyspace'): KeyspaceConfig { + const keysCount = reqInt(data, 'keys_count', `${what}.keyspace`); + if (keysCount <= 0) throw new ConfigError(`${what}.keyspace "keys_count" must be positive`); + const generationAlg = optString(data, 'generation_alg') ?? 'sequential_int'; + if (!['sequential_int', 'uniform_rand'].includes(generationAlg)) { + throw new ConfigError( + `${what}.keyspace "generation_alg" must be sequential_int or uniform_rand ` + + `(got "${generationAlg}")`, + ); + } + return new KeyspaceConfig({ + keysCount, + keySizeBytes: optNumber(data, 'key_size_bytes'), + keyPrefix: optString(data, 'key_prefix'), + generationAlg, + seed: optNumber(data, 'seed'), + }); + } + + static parseCommand(data: Json, what = 'command'): CommandConfig { + const command = reqString(data, 'command', what); + const weight = optNumber(data, 'weight'); + if (weight !== null && (weight < 0 || weight > 1)) { + // Matches Java's CommandConfig.validate(): weights are fractions of 1. + throw new ConfigError(`${what} "weight" must be between 0 and 1 (got ${weight})`); + } + return new CommandConfig({ + command, + weight, + dataSizeBytes: optNumber(data, 'data_size_bytes'), + }); + } +} diff --git a/node/src/config/phaseConfig.ts b/node/src/config/phaseConfig.ts new file mode 100644 index 0000000..c9186f5 --- /dev/null +++ b/node/src/config/phaseConfig.ts @@ -0,0 +1,58 @@ +/** Configuration for a single benchmark phase. */ + +import type { CommandConfig } from './commandConfig.js'; +import type { CompletionConfig } from './completionConfig.js'; +import type { KeyspaceConfig } from './keyspaceConfig.js'; + +export const DEFAULT_PIPELINE_DEPTH = 1; +export const DEFAULT_WARMUP_REQUESTS = 1; + +export class PhaseConfig { + readonly id: string; + readonly description: string | null; + readonly connections: number; + readonly completion: CompletionConfig; + readonly keyspace: KeyspaceConfig; + readonly commands: CommandConfig[]; + readonly cpsLimit: number; + readonly rpsLimit: number; + readonly pipelineDepth: number; + readonly warmupRequests: number; + + constructor(init: { + id: string; + description?: string | null; + connections: number; + completion: CompletionConfig; + keyspace: KeyspaceConfig; + commands: CommandConfig[]; + cpsLimit?: number | null; + rpsLimit?: number | null; + pipelineDepth?: number | null; + warmupRequests?: number | null; + }) { + this.id = init.id; + this.description = init.description ?? null; + this.connections = init.connections; + this.completion = init.completion; + this.keyspace = init.keyspace; + this.commands = init.commands; + // -1 is the configs' "unlimited" sentinel; null/undefined means the same. + this.cpsLimit = init.cpsLimit ?? -1; + this.rpsLimit = init.rpsLimit ?? -1; + this.pipelineDepth = init.pipelineDepth ?? DEFAULT_PIPELINE_DEPTH; + this.warmupRequests = init.warmupRequests ?? DEFAULT_WARMUP_REQUESTS; + } + + hasCpsLimit(): boolean { + return this.cpsLimit > 0; + } + + hasRpsLimit(): boolean { + return this.rpsLimit > 0; + } + + effectivePipelineDepth(): number { + return this.pipelineDepth > 0 ? this.pipelineDepth : DEFAULT_PIPELINE_DEPTH; + } +} diff --git a/node/src/config/workloadConfig.ts b/node/src/config/workloadConfig.ts new file mode 100644 index 0000000..74c235e --- /dev/null +++ b/node/src/config/workloadConfig.ts @@ -0,0 +1,29 @@ +/** A whole workload: a benchmark profile plus an ordered list of phases. */ + +import type { PhaseConfig } from './phaseConfig.js'; + +export interface BenchmarkProfile { + name?: string; + description?: string; + version?: string; +} + +export class WorkloadConfig { + readonly schemaVersion: string; + readonly benchmarkProfile: BenchmarkProfile; + readonly phases: PhaseConfig[]; + + constructor(init: { + schemaVersion?: string; + benchmarkProfile?: BenchmarkProfile | null; + phases: PhaseConfig[]; + }) { + this.schemaVersion = init.schemaVersion ?? '1.0'; + this.benchmarkProfile = init.benchmarkProfile ?? {}; + this.phases = init.phases; + } + + name(): string { + return this.benchmarkProfile.name ?? 'unnamed'; + } +} diff --git a/node/src/engine/benchmark.ts b/node/src/engine/benchmark.ts new file mode 100644 index 0000000..15a9d48 --- /dev/null +++ b/node/src/engine/benchmark.ts @@ -0,0 +1,414 @@ +/** + * Benchmark engine. + * + * Concurrency model: a single event loop with **one client per connection** (the + * `client == connection` invariant every engine holds) and **one worker per + * connection**, all started together via `Promise.all`. Node is single-threaded + * with async I/O, so this is the faithful analogue of Java's + * virtual-thread-per-client design -- an awaited command parks the worker, not + * the loop, so the other connections keep making progress. + * + * Two loops, mirroring Java (BenchmarkEngine.java:353-452): + * - `pipeline_depth <= 1`: issue one command, await it, record, repeat. + * - `pipeline_depth > 1`: keep up to `pipelineDepth` requests in flight per + * connection, awaiting whichever settles first and immediately refilling. + * + * The request budget is **shared across all workers**, claimed one request at a + * time, exactly as Java does with its per-phase `AtomicLong` + * (BenchmarkEngine.java:249, 363-367). It is deliberately not pre-divided per + * worker: with a shared budget a slow connection cannot cap the run -- faster + * workers absorb the slack and the phase ends when the total budget is spent. + * Pre-splitting would bound wall-clock by the slowest connection and change the + * per-connection distribution, which is a real cross-engine comparability gap. + */ + +import type { BenchmarkClient } from '../client/benchmarkClient.js'; +import { BenchmarkClientFactory } from '../client/factory.js'; +import type { Command, CommandResult } from '../command/command.js'; +import { CommandFactory } from '../command/factory.js'; +import type { DriverConfig } from '../config/driverConfig.js'; +import type { PhaseConfig } from '../config/phaseConfig.js'; +import type { WorkloadConfig } from '../config/workloadConfig.js'; +import { MetricsCollector } from '../metrics/collector.js'; +import { NdjsonWriter } from '../metrics/ndjsonWriter.js'; +import { CommandSelector } from './commandSelector.js'; +import { Counter, KeyGenerator } from './keyGenerator.js'; +import { RateLimiter } from './rateLimiter.js'; + +const PROGRESS_LOG_INTERVAL_MS = 10_000; +const CONNECTION_LOG_INTERVAL = 50; + +export interface Logger { + info(message: string): void; + warn(message: string): void; + error(message: string): void; +} + +export const consoleLogger: Logger = { + info: (message) => console.log(`${new Date().toISOString()} INFO ${message}`), + warn: (message) => console.warn(`${new Date().toISOString()} WARN ${message}`), + error: (message) => console.error(`${new Date().toISOString()} ERROR ${message}`), +}; + +/** + * A shared, monotonically-drained request budget for one phase. + * + * `claim()` is atomic without a lock: it reads and writes with no `await` in + * between, so concurrent workers on the single event-loop thread can never + * interleave inside it. + */ +class RequestBudget { + private remaining: number; + + constructor(total: number) { + this.remaining = total; + } + + claim(): boolean { + if (this.remaining <= 0) return false; + this.remaining -= 1; + return true; + } +} + +export interface BenchmarkEngineOptions { + host: string; + port: number; + driverConfig: DriverConfig; + workloadConfig: WorkloadConfig; + metricsPath: string; + commitId?: string | null; + logger?: Logger; +} + +export class BenchmarkEngine { + private readonly host: string; + private readonly port: number; + private readonly driverConfig: DriverConfig; + private readonly workloadConfig: WorkloadConfig; + private readonly writer: NdjsonWriter; + private readonly commitId: string | null; + private readonly log: Logger; + + constructor(options: BenchmarkEngineOptions) { + this.host = options.host; + this.port = options.port; + this.driverConfig = options.driverConfig; + this.workloadConfig = options.workloadConfig; + this.writer = new NdjsonWriter(options.metricsPath); + this.commitId = options.commitId ?? null; + this.log = options.logger ?? consoleLogger; + } + + async run(): Promise { + this.log.info(`Starting benchmark: ${this.workloadConfig.name()}`); + this.log.info(`Driver: ${this.driverConfig.driverId}, Server mode: ${this.driverConfig.mode}`); + this.log.info('Concurrency: event-loop task-per-connection (one client per connection)'); + this.log.info(`Server: ${this.host}:${this.port}`); + + await this.setupMetadata(); + + for (const phase of this.workloadConfig.phases) { + await this.executePhase(phase); + } + + this.log.info('Benchmark completed'); + } + + /** Best-effort: a version lookup failure must not fail the benchmark. */ + private async setupMetadata(): Promise { + try { + const sample = await BenchmarkClientFactory.createAndConnect( + this.host, + this.port, + this.driverConfig, + ); + const version = sample.driverVersion(); + this.writer.setMetadata({ + commitId: this.commitId, + driverId: this.driverConfig.driverId, + primaryDriverVersion: version, + secondaryDriverId: this.driverConfig.secondaryDriverId(), + secondaryDriverVersion: sample.secondaryDriverVersion?.() ?? null, + }); + this.log.info( + `Metadata: commit=${this.commitId ?? 'N/A'}, driver=${this.driverConfig.driverId}, version=${version}`, + ); + await sample.close(); + } catch (error) { + this.log.warn(`Failed to get driver version for metadata: ${(error as Error).message}`); + this.writer.setMetadata({ + commitId: this.commitId, + driverId: this.driverConfig.driverId, + primaryDriverVersion: 'unknown', + secondaryDriverId: this.driverConfig.secondaryDriverId(), + secondaryDriverVersion: null, + }); + } + } + + private async executePhase(phase: PhaseConfig): Promise { + this.log.info(`=== Starting phase: ${phase.id} (${phase.description ?? ''}) ===`); + + const collector = new MetricsCollector(); + const clients = await this.createClients(phase); + const commands = CommandFactory.createAll(phase.commands); + + let status: string; + try { + if (phase.warmupRequests > 0) await this.warmup(clients, phase.warmupRequests); + + // Created *after* warmup, deliberately. The limiter starts its clock at + // construction, so building it earlier would bank the whole warmup + // duration as credit and release a burst of + // (warmup_duration / interval) requests the moment the workload starts -- + // defeating the evenly-spaced, no-burst property the limiter exists for. + const rateLimiter = phase.hasRpsLimit() ? RateLimiter.create(phase.rpsLimit) : null; + + collector.start(); + status = await this.runWorkload(phase, clients, commands, rateLimiter, collector); + collector.stop(); + } finally { + await this.closeClients(clients); + } + + this.writer.writePhaseResults({ + phaseId: phase.id, + status, + connections: phase.connections, + collector, + }); + this.logPhaseSummary(phase, collector, status); + } + + private async createClients(phase: PhaseConfig): Promise { + this.log.info(`Creating ${phase.connections} connections...`); + const cpsLimiter = phase.hasCpsLimit() ? RateLimiter.create(phase.cpsLimit) : null; + + const clients: BenchmarkClient[] = []; + for (let i = 0; i < phase.connections; i++) { + if (cpsLimiter !== null) await cpsLimiter.acquire(); + clients.push( + await BenchmarkClientFactory.createAndConnect(this.host, this.port, this.driverConfig), + ); + if ((i + 1) % CONNECTION_LOG_INTERVAL === 0) { + this.log.info(`Created ${i + 1}/${phase.connections} connections`); + } + } + this.log.info(`All ${clients.length} connections established`); + return clients; + } + + /** + * Send warmup PINGs on every client, failing fast if any of them errors. + * + * A dead or misconfigured server would otherwise produce a whole phase of + * nothing but errors, which is far harder to diagnose than an upfront throw. + * + * Uses `allSettled`, not `all`: `all` rejects on the first failure while the + * remaining warmup loops keep running unawaited, so `executePhase`'s `finally` + * would close the clients underneath them. Settling every loop first means a + * warmup failure leaves nothing in flight. + */ + private async warmup(clients: BenchmarkClient[], warmupRequests: number): Promise { + this.log.info(`Warmup: ${warmupRequests} PING(s) per client...`); + // Warmup mode lets the recording driver suppress simulated errors, so an + // error_rate workload is not aborted by the very errors it is measuring. + for (const client of clients) client.setWarmupMode?.(true); + let outcomes: PromiseSettledResult[]; + try { + outcomes = await Promise.allSettled( + clients.map(async (client) => { + for (let i = 0; i < warmupRequests; i++) { + const result = await client.ping(); + if (result.error !== undefined) { + throw new Error(`Warmup PING failed: ${result.error.message}`); + } + } + }), + ); + } finally { + for (const client of clients) client.setWarmupMode?.(false); + } + + const failure = outcomes.find((o): o is PromiseRejectedResult => o.status === 'rejected'); + if (failure !== undefined) { + throw failure.reason instanceof Error ? failure.reason : new Error(String(failure.reason)); + } + this.log.info('Warmup completed'); + } + + private async runWorkload( + phase: PhaseConfig, + clients: BenchmarkClient[], + commands: Command[], + rateLimiter: RateLimiter | null, + collector: MetricsCollector, + ): Promise { + const { completion } = phase; + const pipelineDepth = phase.effectivePipelineDepth(); + const seedBase = phase.keyspace.seedValue(); + // Shared across workers for sequential_int, so they collectively emit + // 0, 1, 2, ... exactly as the Java reference does. + const sharedCounter = new Counter(); + + const budget = completion.isRequestBased() ? new RequestBudget(completion.totalRequests()) : null; + const deadlineMs = completion.isDurationBased() + ? Date.now() + completion.durationSeconds() * 1000 + : null; + + const keepGoing = (): boolean => { + if (deadlineMs !== null && Date.now() >= deadlineMs) return false; + if (budget !== null) return budget.claim(); + return true; + }; + + this.log.info( + `Starting ${clients.length} workers (pipeline_depth=${pipelineDepth})...`, + ); + + const progressTimer = setInterval(() => { + this.logProgress(collector, completion.isRequestBased() ? completion.totalRequests() : null); + }, PROGRESS_LOG_INTERVAL_MS); + // Do not let the interval hold the event loop open past the phase. + progressTimer.unref(); + + try { + await Promise.all( + clients.map((client, index) => { + const keyGen = KeyGenerator.createWithSeed(phase.keyspace, seedBase + index, sharedCounter); + const selector = new CommandSelector(commands); + return pipelineDepth <= 1 + ? this.runSyncLoop(client, selector, keyGen, rateLimiter, collector, keepGoing) + : this.runPipelinedLoop( + client, + selector, + keyGen, + rateLimiter, + collector, + keepGoing, + pipelineDepth, + ); + }), + ); + this.log.info(`All operations completed (${collector.totalRequests} total requests)`); + return 'COMPLETED'; + } catch (error) { + this.log.error(`Error during workload execution: ${(error as Error).message}`); + return 'ERROR'; + } finally { + clearInterval(progressTimer); + } + } + + /** One in-flight request per connection (pipeline_depth <= 1). */ + private async runSyncLoop( + client: BenchmarkClient, + selector: CommandSelector, + keyGen: KeyGenerator, + rateLimiter: RateLimiter | null, + collector: MetricsCollector, + keepGoing: () => boolean, + ): Promise { + while (keepGoing()) { + if (rateLimiter !== null) await rateLimiter.acquire(); + collector.record(await this.issue(client, selector, keyGen)); + } + } + + /** + * Up to `pipelineDepth` in-flight requests per connection. + * + * Each slot runs its own claim/issue/record cycle, so a settled request is + * replaced immediately rather than waiting on a whole batch -- the same + * "refill as they land" behaviour as Java's `anyOf` loop, expressed as N + * independent slot loops sharing the connection. + */ + private async runPipelinedLoop( + client: BenchmarkClient, + selector: CommandSelector, + keyGen: KeyGenerator, + rateLimiter: RateLimiter | null, + collector: MetricsCollector, + keepGoing: () => boolean, + pipelineDepth: number, + ): Promise { + const slot = async (): Promise => { + while (keepGoing()) { + if (rateLimiter !== null) await rateLimiter.acquire(); + collector.record(await this.issue(client, selector, keyGen)); + } + }; + await Promise.all(Array.from({ length: pipelineDepth }, slot)); + } + + /** + * Select and run one command. + * + * Drivers already convert a rejection into a failed TimedResult, so a throw + * here means an engine-level bug rather than a server error; record it as a + * failed request and keep the phase running. + */ + private async issue( + client: BenchmarkClient, + selector: CommandSelector, + keyGen: KeyGenerator, + ): Promise { + const command = selector.select(); + // Only advance the key sequence for commands that consume a key, so PING + // does not silently shift the sequence other engines produce. + const key = command.usesKey ? keyGen.nextKey() : ''; + try { + return await command.execute(client, key); + } catch (error) { + return { + commandName: command.name, + latencyMicros: 0, + success: false, + errorMessage: (error as Error).message, + }; + } + } + + private async closeClients(clients: BenchmarkClient[]): Promise { + this.log.info(`Closing ${clients.length} connections...`); + for (const client of clients) { + try { + await client.close(); + } catch (error) { + this.log.warn(`Error closing client: ${(error as Error).message}`); + } + } + } + + private logProgress(collector: MetricsCollector, target: number | null): void { + const elapsedMs = Date.now() - collector.startTime(); + if (elapsedMs <= 0) return; + const current = collector.totalRequests; + const rate = Math.round((current * 1000) / elapsedMs); + if (target !== null) { + const percent = ((current * 100) / target).toFixed(1); + this.log.info(`Progress: ${current}/${target} requests (${percent}%) - ${rate} req/s`); + } else { + this.log.info(`Progress: ${current} requests - ${rate} req/s`); + } + } + + private logPhaseSummary(phase: PhaseConfig, collector: MetricsCollector, status: string): void { + const durationSeconds = collector.durationMillis() / 1000; + const rps = durationSeconds > 0 ? Math.round(collector.totalRequests / durationSeconds) : 0; + this.log.info(`=== Phase ${phase.id} completed: ${status} ===`); + this.log.info( + ` Duration: ${durationSeconds.toFixed(1)}s | Requests: ${collector.totalRequests} | ` + + `Errors: ${collector.totalErrors} | RPS: ${rps}`, + ); + for (const [name, m] of collector.commandMetrics) { + if (m.count() === 0 && m.errors === 0) continue; + this.log.info( + ` ${name}: ${m.requests} req (${m.errors} err) | ` + + `p50=${m.percentile(50)}us p95=${m.percentile(95)}us p99=${m.percentile(99)}us ` + + `p99.9=${m.percentile(99.9)}us | min=${m.min()}us max=${m.max()}us`, + ); + } + } +} diff --git a/node/src/engine/commandSelector.ts b/node/src/engine/commandSelector.ts new file mode 100644 index 0000000..d8e2731 --- /dev/null +++ b/node/src/engine/commandSelector.ts @@ -0,0 +1,43 @@ +/** + * Weighted command selection. + * + * Uses normalized cumulative weights, matching Java's inline CommandSelector + * (BenchmarkEngine.java:520-545) and the Python engine. Selection intentionally + * uses the platform RNG (`Math.random`), not the Java LCG: only *key* generation + * must be cross-engine deterministic. Each worker owns its own selector, so + * there is no shared state. + */ + +import type { Command } from '../command/command.js'; + +export class CommandSelector { + private readonly commands: Command[]; + private readonly cumulativeWeights: number[]; + + constructor(commands: Command[]) { + if (commands.length === 0) throw new Error('CommandSelector requires at least one command'); + this.commands = commands; + this.cumulativeWeights = CommandSelector.buildCumulativeWeights(commands); + } + + select(): Command { + const r = Math.random(); + for (let i = 0; i < this.cumulativeWeights.length; i++) { + if (r <= this.cumulativeWeights[i]!) return this.commands[i]!; + } + return this.commands[this.commands.length - 1]!; + } + + private static buildCumulativeWeights(commands: Command[]): number[] { + let totalWeight = commands.reduce((sum, command) => sum + command.weight, 0); + if (totalWeight === 0) totalWeight = 1.0; + + const cumulative: number[] = []; + let running = 0; + for (const command of commands) { + running += command.weight / totalWeight; + cumulative.push(running); + } + return cumulative; + } +} diff --git a/node/src/engine/javaRandom.ts b/node/src/engine/javaRandom.ts new file mode 100644 index 0000000..ce5efba --- /dev/null +++ b/node/src/engine/javaRandom.ts @@ -0,0 +1,68 @@ +/** + * Faithful port of `java.util.Random` (48-bit LCG). + * + * This guarantees identical `uniform_rand` key sequences across the Java + * (reference), Ruby, C#, Python and Node engines. Java is the canonical source, + * so this port reproduces `nextInt(bound)` exactly -- including the 32-bit + * signed-overflow rejection in the general case, which is what avoids modulo + * bias. (The Ruby port omits that rejection because Ruby integers are + * arbitrary-precision; this port emulates the int32 wraparound so it matches + * the Java reference rather than the Ruby approximation.) + * + * BigInt is mandatory for the state, not a stylistic choice: `seed * MULTIPLIER` + * reaches ~2^83, far past the 2^53 that a JS `number` can represent exactly, so + * a `number`-based port silently diverges from Java after the first step. + * + * @see https://docs.oracle.com/javase/8/docs/api/java/util/Random.html + */ + +const MULTIPLIER = 0x5deece66dn; +const ADDEND = 0xbn; +const MASK = (1n << 48n) - 1n; + +/** Interpret the low 32 bits of `value` as a signed 32-bit integer. */ +export function toInt32(value: number): number { + return value | 0; +} + +export class JavaRandom { + private seed: bigint; + + constructor(seed: number | bigint) { + this.seed = JavaRandom.initialScramble(seed); + } + + setSeed(seed: number | bigint): void { + this.seed = JavaRandom.initialScramble(seed); + } + + /** Return a random int in [0, bound) matching Java's nextInt(int). */ + nextInt(bound: number): number { + if (!Number.isInteger(bound) || bound <= 0) { + throw new Error(`bound must be a positive integer (got ${bound})`); + } + + // Power-of-two fast path (matches Java exactly). + if ((bound & -bound) === bound) { + return Number((BigInt(bound) * BigInt(this.next(31))) >> 31n); + } + + // General case: rejection sampling to avoid modulo bias. The rejection + // condition relies on 32-bit signed overflow, which `| 0` emulates. + for (;;) { + const bits = this.next(31); + const val = bits % bound; + if (toInt32(bits - val + (bound - 1)) >= 0) return val; + } + } + + private static initialScramble(seed: number | bigint): bigint { + return (BigInt(seed) ^ MULTIPLIER) & MASK; + } + + /** Java's `protected int next(int bits)`. Exposed for parity tests. */ + next(bits: number): number { + this.seed = (this.seed * MULTIPLIER + ADDEND) & MASK; + return Number(this.seed >> BigInt(48 - bits)); + } +} diff --git a/node/src/engine/keyGenerator.ts b/node/src/engine/keyGenerator.ts new file mode 100644 index 0000000..c44a87c --- /dev/null +++ b/node/src/engine/keyGenerator.ts @@ -0,0 +1,93 @@ +/** + * Key generator producing sequences identical to the other engines. + * + * - `sequential_int`: keys 0, 1, 2, ... N-1, wrapping around. + * - `uniform_rand`: Java-LCG random keys (see ./javaRandom.ts). + * + * Key formatting matches Java's `String.format("%0Nd", index)`: the numeric part + * is zero-padded to `max(keySizeBytes - prefix.length, 1)` digits. With the + * reference configs (`key_prefix: "bench:"`, `key_size_bytes: 16`) that yields + * `bench:` + 10 digits, e.g. `bench:0000000042`. + * + * Cross-worker semantics follow the Java reference: + * - `sequential_int` uses a counter SHARED across all workers in a phase, so the + * workers collectively emit 0, 1, 2, ... (this is what populates the whole + * keyspace during a WARMUP/populate phase). Pass a shared Counter. + * - `uniform_rand` uses a per-worker RNG seeded `baseSeed + workerIndex`. + */ + +import type { KeyspaceConfig } from '../config/keyspaceConfig.js'; +import { JavaRandom } from './javaRandom.js'; + +/** + * A monotonic 0-based counter, shared across a phase's workers. + * + * Safe to share across concurrent workers: `nextValue` performs its + * read-increment with no `await` in between, so it is atomic on the single + * JS event-loop thread. + */ +export class Counter { + private value: number; + + constructor(start = 0) { + this.value = start; + } + + nextValue(): number { + return this.value++; + } + + reset(): void { + this.value = 0; + } +} + +export class KeyGenerator { + private readonly config: KeyspaceConfig; + private readonly keyPrefix: string; + private readonly keysCount: number; + private readonly paddingWidth: number; + private readonly seed: number; + private readonly sequentialCounter: Counter; + private readonly random: JavaRandom; + + constructor(config: KeyspaceConfig, seedOverride?: number, sequentialCounter?: Counter) { + this.config = config; + this.keyPrefix = config.keyPrefix; + this.keysCount = config.keysCount; + this.paddingWidth = Math.max(config.keySizeBytes - config.keyPrefix.length, 1); + this.seed = seedOverride ?? config.seedValue(); + this.sequentialCounter = sequentialCounter ?? new Counter(); + this.random = new JavaRandom(this.seed); + } + + static create(config: KeyspaceConfig): KeyGenerator { + return new KeyGenerator(config); + } + + /** Per-worker generator with a unique seed and an optional shared counter. */ + static createWithSeed( + config: KeyspaceConfig, + seed: number, + sequentialCounter?: Counter, + ): KeyGenerator { + return new KeyGenerator(config, seed, sequentialCounter); + } + + nextKey(): string { + const rawIndex = this.config.isSequentialInt() + ? this.sequentialCounter.nextValue() + : this.random.nextInt(this.keysCount); + + return this.formatKey(rawIndex % this.keysCount); + } + + reset(): void { + this.sequentialCounter.reset(); + this.random.setSeed(this.seed); + } + + private formatKey(keyIndex: number): string { + return this.keyPrefix + String(keyIndex).padStart(this.paddingWidth, '0'); + } +} diff --git a/node/src/engine/rateLimiter.ts b/node/src/engine/rateLimiter.ts new file mode 100644 index 0000000..d0deef9 --- /dev/null +++ b/node/src/engine/rateLimiter.ts @@ -0,0 +1,66 @@ +/** + * Leaky-bucket rate limiter. + * + * Enforces a constant rate with no burst (evenly-spaced operations), matching + * the Java reference's interval math exactly: + * `intervalNanos = 1_000_000_000 / ratePerSecond` (RateLimiter.java:28). Despite + * what docs/ARCHITECTURE.md says about a "token bucket", every engine actually + * implements this leaky bucket, so this follows the code rather than the doc. + * + * A single limiter is shared across all of a phase's workers. Because JS is + * single-threaded, the check-and-advance of `nextAllowedNanos` has no `await` + * between read and write, so it is atomic -- no CAS loop is needed (Java needs + * one only because its issuers are real threads). + * + * Node-specific wrinkle: `setTimeout` clamps to ~1ms, so at high rates (a 100k + * rps limit is a 10us interval) a timer-based wait would undershoot the target + * badly. Sub-millisecond waits therefore yield via `setImmediate`, which returns + * to the event loop -- letting other connections progress -- without sleeping a + * whole millisecond. + */ + +const NANOS_PER_SECOND = 1_000_000_000n; +const NANOS_PER_MILLI = 1_000_000n; + +/** Yield to the event loop without a clamped timer delay. */ +function yieldToEventLoop(): Promise { + return new Promise((resolve) => setImmediate(resolve)); +} + +function sleepMillis(millis: number): Promise { + return new Promise((resolve) => setTimeout(resolve, millis)); +} + +export class RateLimiter { + readonly ratePerSecond: number; + private readonly intervalNanos: bigint; + private nextAllowedNanos: bigint; + + private constructor(ratePerSecond: number) { + this.ratePerSecond = ratePerSecond; + this.intervalNanos = NANOS_PER_SECOND / BigInt(ratePerSecond); + // The first operation is allowed immediately. + this.nextAllowedNanos = process.hrtime.bigint(); + } + + /** Return a limiter, or null for unlimited (rate <= 0). */ + static create(ratePerSecond: number): RateLimiter | null { + return ratePerSecond > 0 ? new RateLimiter(ratePerSecond) : null; + } + + async acquire(): Promise { + for (;;) { + const now = process.hrtime.bigint(); + if (now >= this.nextAllowedNanos) { + this.nextAllowedNanos += this.intervalNanos; + return; + } + const waitNanos = this.nextAllowedNanos - now; + if (waitNanos >= NANOS_PER_MILLI) { + await sleepMillis(Number(waitNanos / NANOS_PER_MILLI)); + } else { + await yieldToEventLoop(); + } + } + } +} diff --git a/node/src/metrics/collector.ts b/node/src/metrics/collector.ts new file mode 100644 index 0000000..ecca794 --- /dev/null +++ b/node/src/metrics/collector.ts @@ -0,0 +1,111 @@ +/** + * Latency metrics collection. + * + * Single-event-loop design: no locks are needed because `record` runs to + * completion without awaiting, so concurrent workers never interleave inside it. + * Latencies are clamped to 600s before recording, and errors are counted but not + * recorded into the histogram -- matching the other engines. + */ + +import type { CommandResult } from '../command/command.js'; +import { HIGHEST_TRACKABLE_VALUE, newHistogram, type Histogram } from './hdrHistogram.js'; + +export class CommandMetrics { + readonly commandName: string; + requests = 0; + errors = 0; + /** + * Created eagerly (like the Java reference) so the NDJSON `hdr` block and + * summary are always present, even for a command that only ever errors -- an + * empty histogram reports count 0 and zero percentiles. + */ + readonly histogram: Histogram = newHistogram(); + + constructor(commandName: string) { + this.commandName = commandName; + } + + record(result: CommandResult): void { + this.requests += 1; + if (result.success) { + this.histogram.recordValue(Math.min(result.latencyMicros, HIGHEST_TRACKABLE_VALUE)); + } else { + this.errors += 1; + } + } + + count(): number { + return this.histogram.totalCount; + } + + /** + * Lowest recorded latency, matching Java's `Histogram.getMinValue()`. + * + * Deliberately NOT `minNonZeroValue`: that skips a legitimately recorded 0us + * sample, and on an empty histogram it returns Number.MAX_SAFE_INTEGER, which + * would land in the NDJSON as a nonsense min. `getValueAtPercentile(0)` returns + * 0 when empty and Java's `getMinValue()` otherwise -- verified equal for + * 1us..599s (and it is what the Ruby encoder uses). + */ + min(): number { + return this.histogram.getValueAtPercentile(0); + } + + /** + * Highest recorded latency, matching Java's `Histogram.getMaxValue()`. + * + * Deliberately NOT `maxValue`: hdr-histogram-js returns the raw recorded + * sample there, while Java returns the *bucket's* highest equivalent value. At + * 3 significant figures those diverge above ~1000us -- recording 50000us gives + * 50000 in JS but 50015 in Java -- which would make summary.max quietly + * incomparable across engines. `getValueAtPercentile(100)` is Java's value. + */ + max(): number { + return this.histogram.getValueAtPercentile(100); + } + + percentile(pct: number): number { + return this.histogram.getValueAtPercentile(pct); + } +} + +export class MetricsCollector { + readonly commandMetrics = new Map(); + totalRequests = 0; + totalErrors = 0; + private startTimeMs: number | null = null; + private endTimeMs: number | null = null; + + start(): void { + this.startTimeMs = Date.now(); + } + + stop(): void { + this.endTimeMs = Date.now(); + } + + record(result: CommandResult): void { + this.totalRequests += 1; + if (!result.success) this.totalErrors += 1; + + let metrics = this.commandMetrics.get(result.commandName); + if (metrics === undefined) { + metrics = new CommandMetrics(result.commandName); + this.commandMetrics.set(result.commandName, metrics); + } + metrics.record(result); + } + + startTime(): number { + return this.startTimeMs ?? 0; + } + + endTime(): number { + return this.endTimeMs ?? 0; + } + + durationMillis(): number { + if (this.startTimeMs === null || this.endTimeMs === null) return 0; + return this.endTimeMs - this.startTimeMs; + } +} diff --git a/node/src/metrics/hdrHistogram.ts b/node/src/metrics/hdrHistogram.ts new file mode 100644 index 0000000..f6c748b --- /dev/null +++ b/node/src/metrics/hdrHistogram.ts @@ -0,0 +1,48 @@ +/** + * HdrHistogram helpers. + * + * Uses `hdr-histogram-js`, the TypeScript port of HdrHistogram. Its + * `encodeIntoCompressedBase64()` emits the base64-encoded V2 *compressed* + * payload -- the same format Java's `encodeIntoCompressedByteBuffer` + + * `Base64.getEncoder()` produces (NdjsonMetricsWriter.java:164-180) and the same + * the Ruby/Python engines emit -- so payloads are mutually decodable across + * engines for cross-language analysis. (Byte-identity is not guaranteed since + * zlib compression levels may differ, but decodability -- what merge/analysis + * needs -- is.) + * + * The returned string is ALREADY base64 (it starts `HIST`). Never base64 it + * again: double-encoding produces a payload Java and Ruby cannot decode. + * + * Histograms use range (1, 600_000_000, 3): 1 microsecond to 600 seconds at 3 + * significant figures, matching every other engine (Java + * `SynchronizedHistogram(600_000_000, 3)`, C# `LongConcurrentHistogram(1, + * 600_000_000, 3)`, Ruby `HDRHistogram.new(1, 600_000_000, 3)`). + */ + +import * as hdr from 'hdr-histogram-js'; + +export const LOWEST_TRACKABLE_VALUE = 1; +export const HIGHEST_TRACKABLE_VALUE = 600_000_000; // 600 seconds in microseconds +export const SIGNIFICANT_FIGURES = 3; + +export type Histogram = hdr.Histogram; + +export function newHistogram(): Histogram { + return hdr.build({ + bitBucketSize: 64, + autoResize: false, + lowestDiscernibleValue: LOWEST_TRACKABLE_VALUE, + highestTrackableValue: HIGHEST_TRACKABLE_VALUE, + numberOfSignificantValueDigits: SIGNIFICANT_FIGURES, + }); +} + +/** Return the base64 V2-compressed encoding, ready for `payload_b64`. */ +export function encodeBase64(histogram: Histogram): string { + return hdr.encodeIntoCompressedBase64(histogram); +} + +/** Inverse of `encodeBase64`, used by the parity tests. */ +export function decodeBase64(payload: string): Histogram { + return hdr.decodeFromCompressedBase64(payload); +} diff --git a/node/src/metrics/ndjsonWriter.ts b/node/src/metrics/ndjsonWriter.ts new file mode 100644 index 0000000..9aa7d60 --- /dev/null +++ b/node/src/metrics/ndjsonWriter.ts @@ -0,0 +1,122 @@ +/** + * Writes benchmark metrics as NDJSON (newline-delimited JSON), one line per + * phase, so the orchestrator can detect phase completion by watching for new + * lines. + * + * The record shape is byte-for-byte the contract in docs/ARCHITECTURE.md + * "Metrics Output Format" plus the `metadata` block the Java writer emits + * (NdjsonMetricsWriter.java:92-111). The matrix runner treats "exited 0 but wrote + * no record" as a cell failure (run_benchmark_matrix.py:1063-1069), so appending + * really must happen. + */ + +import { appendFileSync, mkdirSync } from 'node:fs'; +import { dirname } from 'node:path'; + +import type { CommandMetrics, MetricsCollector } from './collector.js'; +import { encodeBase64, SIGNIFICANT_FIGURES } from './hdrHistogram.js'; + +export interface Metadata { + commitId?: string | null; + driverId?: string | null; + primaryDriverVersion?: string | null; + secondaryDriverId?: string | null; + secondaryDriverVersion?: string | null; +} + +export class NdjsonWriter { + private readonly outputPath: string; + private metadata: Metadata = {}; + + constructor(outputPath: string) { + this.outputPath = outputPath; + } + + setMetadata(metadata: Metadata): void { + this.metadata = metadata; + } + + writePhaseResults(options: { + phaseId: string; + status: string; + connections: number; + collector: MetricsCollector; + }): void { + const record = this.buildPhaseRecord(options); + const parent = dirname(this.outputPath); + if (parent && parent !== '.') mkdirSync(parent, { recursive: true }); + appendFileSync(this.outputPath, `${JSON.stringify(record)}\n`, 'utf8'); + } + + private buildPhaseRecord(options: { + phaseId: string; + status: string; + connections: number; + collector: MetricsCollector; + }): Record { + const { phaseId, status, connections, collector } = options; + const record: Record = {}; + + const { commitId, driverId, primaryDriverVersion, secondaryDriverId, secondaryDriverVersion } = + this.metadata; + if (commitId != null || driverId != null) { + const metadata: Record = {}; + if (commitId != null) metadata['commit_id'] = commitId; + metadata['timestamp'] = new Date().toISOString(); + if (driverId != null) metadata['driver_id'] = driverId; + if (primaryDriverVersion != null) metadata['primary_driver_version'] = primaryDriverVersion; + if (secondaryDriverId != null) metadata['secondary_driver_id'] = secondaryDriverId; + if (secondaryDriverVersion != null) { + metadata['secondary_driver_version'] = secondaryDriverVersion; + } + record['metadata'] = metadata; + } + + record['phase'] = { + id: phaseId, + status, + start_timestamp: new Date(collector.startTime()).toISOString(), + finish_timestamp: new Date(collector.endTime()).toISOString(), + duration_ms: collector.durationMillis(), + connections, + }; + + record['totals'] = { + requests: collector.totalRequests, + errors: collector.totalErrors, + }; + + const metrics: Record = {}; + for (const [commandName, commandMetrics] of collector.commandMetrics) { + metrics[commandName] = NdjsonWriter.buildCommandRecord(commandMetrics); + } + record['metrics'] = metrics; + + return record; + } + + private static buildCommandRecord(m: CommandMetrics): Record { + return { + requests: m.requests, + errors: m.errors, + latency: { + unit: 'us', + count: m.count(), + summary: { + min: m.min(), + p50: m.percentile(50), + p95: m.percentile(95), + p99: m.percentile(99), + p999: m.percentile(99.9), + max: m.max(), + }, + hdr: { + format: 'hdr', + sigfig: SIGNIFICANT_FIGURES, + // Already base64 -- do NOT encode again (see metrics/hdrHistogram.ts). + payload_b64: encodeBase64(m.histogram), + }, + }, + }; + } +} diff --git a/node/src/version.ts b/node/src/version.ts new file mode 100644 index 0000000..966a9ef --- /dev/null +++ b/node/src/version.ts @@ -0,0 +1,2 @@ +/** Engine version, reported by `--version` and `--info`. */ +export const VERSION = '1.0.0'; diff --git a/node/test/integration/clients.test.ts b/node/test/integration/clients.test.ts new file mode 100644 index 0000000..24548d4 --- /dev/null +++ b/node/test/integration/clients.test.ts @@ -0,0 +1,197 @@ +/** + * Live-server tests for the real drivers. + * + * Skipped unless VALKEY_HOST/VALKEY_PORT point at a running server, matching the + * Java/Ruby/C# integration suites (`make node-integration-test` starts one). + */ + +import assert from 'node:assert/strict'; +import { mkdtempSync, readFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { describe, it } from 'node:test'; + +import { BenchmarkClientFactory } from '../../src/client/factory.js'; +import { ConfigLoader } from '../../src/config/loader.js'; +import { BenchmarkEngine, type Logger } from '../../src/engine/benchmark.js'; +import { decodeBase64 } from '../../src/metrics/hdrHistogram.js'; + +const HOST = process.env['VALKEY_HOST'] ?? 'localhost'; +const PORT = Number(process.env['VALKEY_PORT'] ?? 6379); +const REAL_DRIVERS = ['valkey-glide-node', 'ioredis', 'iovalkey'] as const; + +const silentLogger: Logger = { info: () => {}, warn: () => {}, error: () => {} }; + +/** True when VALKEY_HOST is set, i.e. a server is expected to be reachable. */ +const enabled = process.env['VALKEY_HOST'] !== undefined; +const skip = enabled ? false : 'set VALKEY_HOST to run the live-server tests'; + +function driverConfig(driverId: string, extra: Record = {}) { + return ConfigLoader.parseDriverConfig({ + schema_version: '1.0', + driver_id: driverId, + mode: 'standalone', + specific_driver_config: {}, + ...extra, + }); +} + +for (const driverId of REAL_DRIVERS) { + describe(`${driverId} against a live server`, { skip }, () => { + it('connects, pings, sets, gets and closes', async () => { + const client = await BenchmarkClientFactory.createAndConnect(HOST, PORT, driverConfig(driverId)); + try { + const ping = await client.ping(); + assert.equal(ping.error, undefined); + assert.equal(ping.value, 'PONG'); + assert.ok(ping.latencyMicros >= 0); + + const key = `node-it:${driverId}:${process.pid}`; + const payload = Buffer.from('x'.repeat(64), 'latin1'); + + const set = await client.set(key, payload); + assert.equal(set.error, undefined); + assert.equal(set.value, 'OK'); + + const get = await client.get(key); + assert.equal(get.error, undefined); + // Every driver must return the same decoded shape, or the engines are + // charged for different work: a string of the payload's length. + assert.equal(typeof get.value, 'string'); + assert.equal(get.value, payload.toString('latin1')); + } finally { + await client.close(); + } + }); + + it('returns null for a missing key rather than erroring', async () => { + const client = await BenchmarkClientFactory.createAndConnect(HOST, PORT, driverConfig(driverId)); + try { + const get = await client.get(`node-it:absent:${process.pid}:${Math.random()}`); + assert.equal(get.error, undefined); + assert.equal(get.value, null); + } finally { + await client.close(); + } + }); + + it('reports a real driver version, not "unknown"', async () => { + const client = await BenchmarkClientFactory.create(driverId); + assert.match(client.driverVersion(), /^\d+\.\d+\.\d+/); + }); + + it('applies command_timeout_ms without breaking normal commands', async () => { + const client = await BenchmarkClientFactory.createAndConnect( + HOST, + PORT, + driverConfig(driverId, { command_timeout_ms: 5000 }), + ); + try { + assert.equal((await client.ping()).error, undefined); + } finally { + await client.close(); + } + }); + + it('runs a short workload and writes decodable metrics', async () => { + const metricsPath = join(mkdtempSync(join(tmpdir(), 'resp-bench-node-')), 'metrics.ndjson'); + await new BenchmarkEngine({ + host: HOST, + port: PORT, + driverConfig: driverConfig(driverId), + workloadConfig: ConfigLoader.parseWorkloadConfig({ + benchmark_profile: { name: `${driverId} smoke` }, + phases: [ + { + id: 'STEADY', + connections: 2, + warmup_requests: 1, + completion: { type: 'requests', requests: 200 }, + keyspace: { keys_count: 100, key_prefix: 'node-it:', key_size_bytes: 16 }, + commands: [ + { command: 'set', weight: 0.5, data_size_bytes: 64 }, + { command: 'get', weight: 0.5 }, + ], + }, + ], + }), + metricsPath, + commitId: 'integration-test', + logger: silentLogger, + }).run(); + + const record = JSON.parse(readFileSync(metricsPath, 'utf8').trimEnd().split('\n')[0]!); + assert.equal(record.phase.status, 'COMPLETED'); + assert.equal(record.totals.requests, 200); + assert.equal(record.totals.errors, 0, 'a healthy server should produce no errors'); + assert.equal(record.metadata.driver_id, driverId); + assert.match(record.metadata.primary_driver_version, /^\d+\.\d+\.\d+/); + for (const name of Object.keys(record.metrics)) { + const latency = record.metrics[name].latency; + assert.ok(latency.hdr.payload_b64.startsWith('HIST')); + assert.equal(decodeBase64(latency.hdr.payload_b64).totalCount, latency.count); + assert.ok(latency.summary.p50 >= 0); + assert.ok(latency.summary.max >= latency.summary.p50); + } + }); + + it('runs a pipelined workload', async () => { + const metricsPath = join(mkdtempSync(join(tmpdir(), 'resp-bench-node-')), 'metrics.ndjson'); + await new BenchmarkEngine({ + host: HOST, + port: PORT, + driverConfig: driverConfig(driverId), + workloadConfig: ConfigLoader.parseWorkloadConfig({ + phases: [ + { + id: 'PIPELINED', + connections: 2, + pipeline_depth: 8, + warmup_requests: 1, + completion: { type: 'requests', requests: 200 }, + keyspace: { keys_count: 100, key_prefix: 'node-it:', key_size_bytes: 16 }, + commands: [{ command: 'get', weight: 1.0 }], + }, + ], + }), + metricsPath, + logger: silentLogger, + }).run(); + + const record = JSON.parse(readFileSync(metricsPath, 'utf8').trimEnd().split('\n')[0]!); + assert.equal(record.phase.status, 'COMPLETED'); + // 2 connections x depth 8 = 16 in-flight slots on one shared budget of 200. + assert.equal(record.totals.requests, 200); + assert.equal(record.totals.errors, 0); + }); + }); +} + +describe('unreachable server', { skip }, () => { + it('fails fast instead of recording a phase of pure errors', async () => { + // Port 1 is reserved and never listening. A dead server must abort the run + // with a clear error, not silently produce 100% error metrics. + await assert.rejects( + () => + new BenchmarkEngine({ + host: '127.0.0.1', + port: 1, + driverConfig: driverConfig('ioredis'), + workloadConfig: ConfigLoader.parseWorkloadConfig({ + phases: [ + { + id: 'STEADY', + connections: 1, + warmup_requests: 1, + completion: { type: 'requests', requests: 10 }, + keyspace: { keys_count: 10 }, + commands: [{ command: 'get', weight: 1.0 }], + }, + ], + }), + metricsPath: join(mkdtempSync(join(tmpdir(), 'resp-bench-node-')), 'metrics.ndjson'), + logger: silentLogger, + }).run(), + ); + }); +}); diff --git a/node/test/integration/recordingWorkload.test.ts b/node/test/integration/recordingWorkload.test.ts new file mode 100644 index 0000000..df67dbd --- /dev/null +++ b/node/test/integration/recordingWorkload.test.ts @@ -0,0 +1,330 @@ +/** + * Full-engine tests against the `recording` driver — no server required. + * + * These are the tests that catch engine-level regressions the unit tests cannot: + * the shared request budget, warmup fail-fast, pipelining, rate limiting, and the + * NDJSON a real run produces. + */ + +import assert from 'node:assert/strict'; +import { mkdtempSync, readFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { describe, it } from 'node:test'; + +import { ConfigLoader } from '../../src/config/loader.js'; +import { BenchmarkEngine, type Logger } from '../../src/engine/benchmark.js'; +import { decodeBase64 } from '../../src/metrics/hdrHistogram.js'; + +const silentLogger: Logger = { info: () => {}, warn: () => {}, error: () => {} }; + +function tempMetricsPath(): string { + return join(mkdtempSync(join(tmpdir(), 'resp-bench-node-')), 'metrics.ndjson'); +} + +function readRecords(path: string): Array> { + return readFileSync(path, 'utf8') + .trimEnd() + .split('\n') + .map((line) => JSON.parse(line)); +} + +function driverConfig(specific: Record = {}) { + return ConfigLoader.parseDriverConfig({ + schema_version: '1.0', + driver_id: 'recording', + mode: 'standalone', + specific_driver_config: specific, + }); +} + +function workload(phases: unknown[]) { + return ConfigLoader.parseWorkloadConfig({ + schema_version: '1.0', + benchmark_profile: { name: 'Node engine test' }, + phases, + }); +} + +const STEADY_PHASE = { + id: 'STEADY', + description: 'short steady phase', + connections: 4, + warmup_requests: 1, + completion: { type: 'requests', requests: 200 }, + keyspace: { keys_count: 100, key_prefix: 'e2e:', key_size_bytes: 16 }, + commands: [ + { command: 'set', weight: 0.5, data_size_bytes: 32 }, + { command: 'get', weight: 0.5 }, + ], +}; + +async function run(options: { + phases: unknown[]; + specific?: Record; + commitId?: string; +}): Promise<{ path: string; records: Array> }> { + const path = tempMetricsPath(); + await new BenchmarkEngine({ + host: 'localhost', + port: 6379, + driverConfig: driverConfig(options.specific), + workloadConfig: workload(options.phases), + metricsPath: path, + commitId: options.commitId ?? 'test-commit', + logger: silentLogger, + }).run(); + return { path, records: readRecords(path) }; +} + +describe('recording-driver workload', () => { + it('runs a phase end to end and writes valid NDJSON', async () => { + const { records } = await run({ phases: [STEADY_PHASE] }); + + assert.equal(records.length, 1); + const record = records[0]!; + assert.equal(record['phase'].id, 'STEADY'); + assert.equal(record['phase'].status, 'COMPLETED'); + assert.equal(record['phase'].connections, 4); + assert.equal(record['totals'].requests, 200); + assert.equal(record['totals'].errors, 0); + assert.equal(record['metadata'].commit_id, 'test-commit'); + assert.equal(record['metadata'].driver_id, 'recording'); + assert.equal(record['metadata'].primary_driver_version, '1.0.0'); + + // Both commands were exercised and their counts add up to the total. + const commands = Object.keys(record['metrics']).sort(); + assert.deepEqual(commands, ['GET', 'SET']); + const sum = commands.reduce((acc, name) => acc + record['metrics'][name].requests, 0); + assert.equal(sum, 200); + + for (const name of commands) { + const payload = record['metrics'][name].latency.hdr.payload_b64; + assert.ok(payload.startsWith('HIST')); + assert.equal(decodeBase64(payload).totalCount, record['metrics'][name].latency.count); + } + }); + + it('honours the shared request budget exactly, not a per-worker split', async () => { + // 7 requests across 4 connections does not divide evenly. A pre-split budget + // would round to 4 or 8; a shared budget lands on exactly 7. + const { records } = await run({ + phases: [{ ...STEADY_PHASE, connections: 4, completion: { type: 'requests', requests: 7 } }], + }); + assert.equal(records[0]!['totals'].requests, 7); + }); + + it('honours the budget when connections exceed the request count', async () => { + const { records } = await run({ + phases: [{ ...STEADY_PHASE, connections: 8, completion: { type: 'requests', requests: 3 } }], + }); + assert.equal(records[0]!['totals'].requests, 3); + }); + + it('runs each phase in order, appending one record per phase', async () => { + const { records } = await run({ + phases: [ + { + ...STEADY_PHASE, + id: 'WARMUP', + connections: 2, + completion: { type: 'requests', requests: 20 }, + commands: [{ command: 'set', weight: 1.0, data_size_bytes: 16 }], + }, + { ...STEADY_PHASE, id: 'STEADY', completion: { type: 'requests', requests: 30 } }, + ], + }); + + assert.deepEqual( + records.map((r) => r['phase'].id), + ['WARMUP', 'STEADY'], + ); + assert.equal(records[0]!['totals'].requests, 20); + assert.equal(records[1]!['totals'].requests, 30); + assert.deepEqual(Object.keys(records[0]!['metrics']), ['SET']); + }); + + it('stops a duration-based phase at the deadline', async () => { + const started = Date.now(); + const { records } = await run({ + phases: [{ ...STEADY_PHASE, connections: 2, completion: { type: 'duration', seconds: 1 } }], + }); + const elapsed = Date.now() - started; + + assert.equal(records[0]!['phase'].status, 'COMPLETED'); + assert.ok(records[0]!['totals'].requests > 0, 'a duration phase should do some work'); + assert.ok(elapsed >= 900, `finished suspiciously early: ${elapsed}ms`); + assert.ok(elapsed < 6000, `overran the 1s deadline: ${elapsed}ms`); + }); + + it('keeps pipelined runs on the same shared budget', async () => { + const { records } = await run({ + phases: [ + { + ...STEADY_PHASE, + connections: 3, + pipeline_depth: 4, + completion: { type: 'requests', requests: 50 }, + }, + ], + }); + // 3 connections x depth 4 = 12 in-flight slots all drawing on one budget of + // 50; an over-issuing pipeline would show more than 50 requests. + assert.equal(records[0]!['totals'].requests, 50); + }); + + it('surfaces injected errors as errors, not as latency samples', async () => { + const { records } = await run({ + phases: [{ ...STEADY_PHASE, connections: 2, completion: { type: 'requests', requests: 100 } }], + specific: { error_rate: 1.0, error_message: 'Simulated failure' }, + }); + + const record = records[0]!; + assert.equal(record['totals'].requests, 100); + assert.equal(record['totals'].errors, 100); + for (const name of Object.keys(record['metrics'])) { + const metrics = record['metrics'][name]; + assert.equal(metrics.errors, metrics.requests); + // Failed requests are counted but never recorded into the histogram. + assert.equal(metrics.latency.count, 0); + assert.ok(metrics.latency.hdr.payload_b64.startsWith('HIST')); + } + }); + + it('records a partial error rate on both sides of the split', async () => { + const { records } = await run({ + phases: [{ ...STEADY_PHASE, connections: 2, completion: { type: 'requests', requests: 400 } }], + specific: { error_rate: 0.5 }, + }); + const { requests, errors } = records[0]!['totals']; + assert.equal(requests, 400); + assert.ok(errors > 100 && errors < 300, `error count ${errors} not near half of 400`); + }); + + it('does not count warmup requests toward the phase totals', async () => { + // Warmup runs 3 PINGs on each of 2 connections. If those leaked into the + // measured metrics we would see 6 extra requests and a PING command key. + const { records } = await run({ + phases: [ + { + ...STEADY_PHASE, + connections: 2, + warmup_requests: 3, + completion: { type: 'requests', requests: 10 }, + }, + ], + }); + assert.equal(records[0]!['totals'].requests, 10); + assert.equal('PING' in records[0]!['metrics'], false); + }); + + it('suppresses injected errors during warmup so the phase still runs', async () => { + // Warmup's fail-fast exists to catch an unreachable server. An error_rate + // workload is deliberately measuring errors, so warmup must not abort on + // them -- Java does the same via setWarmupMode. + const { records } = await run({ + phases: [ + { + ...STEADY_PHASE, + connections: 2, + warmup_requests: 2, + completion: { type: 'requests', requests: 20 }, + }, + ], + specific: { error_rate: 1.0, error_message: 'Simulated failure' }, + }); + assert.equal(records[0]!['phase'].status, 'COMPLETED'); + assert.equal(records[0]!['totals'].requests, 20); + assert.equal(records[0]!['totals'].errors, 20); + }); + + it('applies an rps_limit to the whole phase', async () => { + const started = Date.now(); + const { records } = await run({ + phases: [ + { + ...STEADY_PHASE, + connections: 4, + rps_limit: 100, + completion: { type: 'requests', requests: 50 }, + }, + ], + }); + const elapsed = Date.now() - started; + + assert.equal(records[0]!['totals'].requests, 50); + // 50 requests at 100/s cannot finish faster than ~0.49s however many + // connections are issuing them. + assert.ok(elapsed >= 400, `rps_limit was not enforced: ${elapsed}ms for 50 requests at 100/s`); + }); + + it('does not bank warmup time as rate-limiter credit', async () => { + // Regression: the limiter starts its clock at construction, so building it + // before warmup banked the whole warmup duration as credit and released a + // burst of (warmup_duration / interval) requests once the workload began. + // + // Sized so the burst would swallow the entire workload: warmup is 25 PINGs + // at 20ms = ~500ms of idle limiter time, and at 50 rps (a 20ms interval) + // that banks ~25 free requests -- every request this phase issues. A banked + // limiter therefore finishes in ~500ms (warmup only); a correctly-scoped one + // additionally paces 24 intervals, so ~980ms. + const started = Date.now(); + const { records } = await run({ + phases: [ + { + ...STEADY_PHASE, + connections: 4, + warmup_requests: 25, + rps_limit: 50, + completion: { type: 'requests', requests: 25 }, + }, + ], + specific: { operation_delay_micros: 20_000 }, + }); + const elapsed = Date.now() - started; + + assert.equal(records[0]!['totals'].requests, 25); + assert.ok( + elapsed >= 850, + `rate limiter released a warmup-banked burst: 25 requests at 50 rps after ` + + `a ~500ms warmup took only ${elapsed}ms (expected ~980ms)`, + ); + }); + + it('gates connection setup with a cps_limit', async () => { + const started = Date.now(); + await run({ + phases: [ + { + ...STEADY_PHASE, + connections: 5, + cps_limit: 20, + completion: { type: 'requests', requests: 5 }, + }, + ], + }); + const elapsed = Date.now() - started; + // 5 connections at 20/s means the last one opens ~200ms in. + assert.ok(elapsed >= 150, `cps_limit was not enforced: ${elapsed}ms to open 5 connections`); + }); + + it('generates keys inside the configured keyspace', async () => { + const { records } = await run({ + phases: [ + { + ...STEADY_PHASE, + connections: 2, + completion: { type: 'requests', requests: 40 }, + keyspace: { + keys_count: 10, + key_prefix: 'e2e:', + key_size_bytes: 8, + generation_alg: 'uniform_rand', + seed: 12345, + }, + }, + ], + }); + assert.equal(records[0]!['totals'].requests, 40); + }); +}); diff --git a/node/test/unit/collector.test.ts b/node/test/unit/collector.test.ts new file mode 100644 index 0000000..da926e9 --- /dev/null +++ b/node/test/unit/collector.test.ts @@ -0,0 +1,97 @@ +import assert from 'node:assert/strict'; +import { describe, it } from 'node:test'; + +import { CommandMetrics, MetricsCollector } from '../../src/metrics/collector.js'; +import { HIGHEST_TRACKABLE_VALUE } from '../../src/metrics/hdrHistogram.js'; + +describe('CommandMetrics', () => { + it('counts requests and records only successes into the histogram', () => { + const metrics = new CommandMetrics('GET'); + metrics.record({ commandName: 'GET', latencyMicros: 100, success: true }); + metrics.record({ commandName: 'GET', latencyMicros: 200, success: true }); + metrics.record({ commandName: 'GET', latencyMicros: 300, success: false }); + + assert.equal(metrics.requests, 3); + assert.equal(metrics.errors, 1); + // The failed request's latency must not skew the distribution. + assert.equal(metrics.count(), 2); + assert.equal(metrics.max(), 200); + }); + + it('reports zeros for an empty histogram rather than sentinels', () => { + // minNonZeroValue would return Number.MAX_SAFE_INTEGER here and land a + // nonsense min in the NDJSON. + const metrics = new CommandMetrics('GET'); + assert.equal(metrics.count(), 0); + assert.equal(metrics.min(), 0); + assert.equal(metrics.max(), 0); + assert.equal(metrics.percentile(50), 0); + assert.equal(metrics.percentile(99.9), 0); + }); + + it('reports a legitimately recorded 0us sample as min 0', () => { + const metrics = new CommandMetrics('PING'); + metrics.record({ commandName: 'PING', latencyMicros: 0, success: true }); + metrics.record({ commandName: 'PING', latencyMicros: 5, success: true }); + assert.equal(metrics.min(), 0); + }); + + it('matches Java getMinValue/getMaxValue bucket quantization', () => { + // Verified against org.HdrHistogram.Histogram(1, 600_000_000, 3): at 3 + // significant figures Java reports the bucket's equivalent bounds, not the + // raw sample. hdr-histogram-js' maxValue/minNonZeroValue do NOT do this, so + // these anchors keep summary.min/max comparable across engines. + for (const [recorded, expectedMin, expectedMax] of [ + [1, 1, 1], + [100, 100, 100], + [1234, 1234, 1234], + [50_000, 49_984, 50_015], + [599_000_000, 598_736_896, 599_261_183], + ] as const) { + const metrics = new CommandMetrics('GET'); + metrics.record({ commandName: 'GET', latencyMicros: recorded, success: true }); + assert.equal(metrics.min(), expectedMin, `min for ${recorded}`); + assert.equal(metrics.max(), expectedMax, `max for ${recorded}`); + } + }); + + it('clamps a latency above the trackable range instead of throwing', () => { + const metrics = new CommandMetrics('GET'); + metrics.record({ + commandName: 'GET', + latencyMicros: HIGHEST_TRACKABLE_VALUE * 2, + success: true, + }); + assert.equal(metrics.count(), 1); + }); +}); + +describe('MetricsCollector', () => { + it('aggregates totals across commands', () => { + const collector = new MetricsCollector(); + collector.record({ commandName: 'GET', latencyMicros: 10, success: true }); + collector.record({ commandName: 'SET', latencyMicros: 20, success: true }); + collector.record({ commandName: 'SET', latencyMicros: 30, success: false }); + + assert.equal(collector.totalRequests, 3); + assert.equal(collector.totalErrors, 1); + assert.deepEqual([...collector.commandMetrics.keys()], ['GET', 'SET']); + assert.equal(collector.commandMetrics.get('SET')!.requests, 2); + }); + + it('preserves first-seen command order for stable NDJSON output', () => { + const collector = new MetricsCollector(); + collector.record({ commandName: 'SET', latencyMicros: 1, success: true }); + collector.record({ commandName: 'GET', latencyMicros: 1, success: true }); + assert.deepEqual([...collector.commandMetrics.keys()], ['SET', 'GET']); + }); + + it('reports zero duration until both start and stop have run', () => { + const collector = new MetricsCollector(); + assert.equal(collector.durationMillis(), 0); + collector.start(); + assert.equal(collector.durationMillis(), 0); + collector.stop(); + assert.ok(collector.durationMillis() >= 0); + }); +}); diff --git a/node/test/unit/commandSelector.test.ts b/node/test/unit/commandSelector.test.ts new file mode 100644 index 0000000..f29ae9e --- /dev/null +++ b/node/test/unit/commandSelector.test.ts @@ -0,0 +1,59 @@ +import assert from 'node:assert/strict'; +import { describe, it } from 'node:test'; + +import { CommandFactory } from '../../src/command/factory.js'; +import { CommandConfig } from '../../src/config/commandConfig.js'; +import { CommandSelector } from '../../src/engine/commandSelector.js'; + +function selectorFor(weights: Array<[string, number]>): CommandSelector { + return new CommandSelector( + CommandFactory.createAll( + weights.map(([command, weight]) => new CommandConfig({ command, weight })), + ), + ); +} + +function distribution(selector: CommandSelector, draws: number): Map { + const counts = new Map(); + for (let i = 0; i < draws; i++) { + const name = selector.select().name; + counts.set(name, (counts.get(name) ?? 0) + 1); + } + return counts; +} + +describe('CommandSelector', () => { + it('respects an 80/20 weighting within tolerance', () => { + const draws = 20_000; + const counts = distribution(selectorFor([['get', 0.8], ['set', 0.2]]), draws); + const getShare = (counts.get('GET') ?? 0) / draws; + assert.ok(Math.abs(getShare - 0.8) < 0.02, `GET share ${getShare.toFixed(3)} not near 0.8`); + }); + + it('normalizes weights that do not sum to 1', () => { + const draws = 20_000; + const counts = distribution(selectorFor([['get', 0.25], ['set', 0.25]]), draws); + const getShare = (counts.get('GET') ?? 0) / draws; + assert.ok(Math.abs(getShare - 0.5) < 0.02, `GET share ${getShare.toFixed(3)} not near 0.5`); + }); + + it('always returns the sole command', () => { + const selector = selectorFor([['ping', 1.0]]); + for (let i = 0; i < 100; i++) assert.equal(selector.select().name, 'PING'); + }); + + it('never returns a zero-weight command', () => { + const counts = distribution(selectorFor([['get', 1.0], ['set', 0]]), 5000); + assert.equal(counts.get('SET') ?? 0, 0); + }); + + it('falls back to a command when every weight is zero', () => { + // Guards against dividing by a zero total and returning undefined. + const selector = selectorFor([['get', 0], ['set', 0]]); + for (let i = 0; i < 100; i++) assert.ok(['GET', 'SET'].includes(selector.select().name)); + }); + + it('rejects an empty command list', () => { + assert.throws(() => new CommandSelector([]), /at least one command/); + }); +}); diff --git a/node/test/unit/configLoader.test.ts b/node/test/unit/configLoader.test.ts new file mode 100644 index 0000000..13971eb --- /dev/null +++ b/node/test/unit/configLoader.test.ts @@ -0,0 +1,269 @@ +import assert from 'node:assert/strict'; +import { mkdtempSync, writeFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { describe, it } from 'node:test'; + +import { DEFAULT_DATA_SIZE_BYTES } from '../../src/config/commandConfig.js'; +import { + DEFAULT_KEY_PREFIX, + DEFAULT_KEY_SIZE_BYTES, +} from '../../src/config/keyspaceConfig.js'; +import { ConfigError, ConfigLoader } from '../../src/config/loader.js'; + +function writeTempJson(name: string, data: unknown): string { + const dir = mkdtempSync(join(tmpdir(), 'resp-bench-node-')); + const path = join(dir, name); + writeFileSync(path, JSON.stringify(data), 'utf8'); + return path; +} + +const MINIMAL_PHASE = { + id: 'STEADY', + connections: 2, + completion: { type: 'requests', requests: 100 }, + keyspace: { keys_count: 50 }, + commands: [{ command: 'get', weight: 1.0 }], +}; + +describe('ConfigLoader — driver config', () => { + it('parses every schema field', () => { + const config = ConfigLoader.parseDriverConfig({ + schema_version: '1.0', + description: 'test driver', + driver_id: 'ioredis', + mode: 'cluster', + command_timeout_ms: 10_000, + tls: { enabled: true, ca_path: '/tmp/ca.pem' }, + auth: { username: 'u', password: 'p' }, + specific_driver_config: { secondary_driver_id: 'other' }, + }); + + assert.equal(config.schemaVersion, '1.0'); + assert.equal(config.description, 'test driver'); + assert.equal(config.driverId, 'ioredis'); + assert.equal(config.mode, 'cluster'); + assert.equal(config.commandTimeoutMs, 10_000); + assert.equal(config.tlsEnabled(), true); + assert.equal(config.hasAuth(), true); + assert.equal(config.isCluster(), true); + assert.equal(config.isStandalone(), false); + assert.equal(config.secondaryDriverId(), 'other'); + }); + + it('applies defaults for the optional fields', () => { + const config = ConfigLoader.parseDriverConfig({ driver_id: 'ioredis' }); + assert.equal(config.schemaVersion, '1.0'); + assert.equal(config.mode, 'standalone'); + assert.equal(config.commandTimeoutMs, null); + assert.equal(config.tlsEnabled(), false); + assert.equal(config.hasAuth(), false); + assert.deepEqual(config.specificDriverConfig, {}); + assert.equal(config.secondaryDriverId(), null); + }); + + it('treats an empty username and password as no auth', () => { + const config = ConfigLoader.parseDriverConfig({ + driver_id: 'ioredis', + auth: { username: '', password: '' }, + }); + assert.equal(config.hasAuth(), false); + }); + + it('rejects a missing driver_id', () => { + assert.throws(() => ConfigLoader.parseDriverConfig({ mode: 'standalone' }), ConfigError); + }); + + it('rejects an unknown mode', () => { + assert.throws( + () => ConfigLoader.parseDriverConfig({ driver_id: 'ioredis', mode: 'galaxy' }), + /must be standalone, cluster or sentinel/, + ); + }); + + it('loads the real repo driver configs', () => { + const path = writeTempJson('driver.json', { + schema_version: '1.0', + description: 'valkey-glide-node client - default configuration', + driver_id: 'valkey-glide-node', + mode: 'standalone', + specific_driver_config: {}, + }); + assert.equal(ConfigLoader.loadDriverConfig(path).driverId, 'valkey-glide-node'); + }); + + it('reports a readable error for a missing file', () => { + assert.throws(() => ConfigLoader.loadDriverConfig('/nope/missing.json'), /cannot read driver config/); + }); + + it('reports a readable error for malformed JSON', () => { + const dir = mkdtempSync(join(tmpdir(), 'resp-bench-node-')); + const path = join(dir, 'bad.json'); + writeFileSync(path, '{ not json', 'utf8'); + assert.throws(() => ConfigLoader.loadDriverConfig(path), /is not valid JSON/); + }); +}); + +describe('ConfigLoader — workload config', () => { + it('parses a full workload', () => { + const workload = ConfigLoader.parseWorkloadConfig({ + schema_version: '1.0', + benchmark_profile: { name: 'Reference', description: 'd', version: '1.0.0' }, + phases: [ + { + id: 'WARMUP', + description: 'populate', + connections: 1, + cps_limit: -1, + rps_limit: -1, + pipeline_depth: 4, + warmup_requests: 3, + completion: { type: 'requests', requests: 1_000_000 }, + keyspace: { + keys_count: 1_000_000, + key_size_bytes: 16, + key_prefix: 'bench:', + generation_alg: 'uniform_rand', + seed: 12345, + }, + commands: [ + { command: 'get', weight: 0.8 }, + { command: 'set', weight: 0.2, data_size_bytes: 512 }, + ], + }, + ], + }); + + assert.equal(workload.name(), 'Reference'); + assert.equal(workload.phases.length, 1); + const phase = workload.phases[0]!; + assert.equal(phase.id, 'WARMUP'); + assert.equal(phase.connections, 1); + assert.equal(phase.pipelineDepth, 4); + assert.equal(phase.effectivePipelineDepth(), 4); + assert.equal(phase.warmupRequests, 3); + assert.equal(phase.hasCpsLimit(), false); + assert.equal(phase.hasRpsLimit(), false); + assert.equal(phase.completion.isRequestBased(), true); + assert.equal(phase.completion.totalRequests(), 1_000_000); + assert.equal(phase.keyspace.isUniformRand(), true); + assert.equal(phase.keyspace.seedValue(), 12345); + assert.equal(phase.commands[1]!.dataSizeBytes, 512); + }); + + it('applies the cross-engine defaults', () => { + const workload = ConfigLoader.parseWorkloadConfig({ phases: [MINIMAL_PHASE] }); + const phase = workload.phases[0]!; + assert.equal(phase.cpsLimit, -1); + assert.equal(phase.rpsLimit, -1); + assert.equal(phase.pipelineDepth, 1); + assert.equal(phase.warmupRequests, 1); + assert.equal(phase.description, null); + assert.equal(phase.keyspace.keySizeBytes, DEFAULT_KEY_SIZE_BYTES); + assert.equal(phase.keyspace.keyPrefix, DEFAULT_KEY_PREFIX); + assert.equal(phase.keyspace.isSequentialInt(), true); + assert.equal(phase.keyspace.seedValue(), 0); + assert.equal(phase.commands[0]!.dataSizeBytes, DEFAULT_DATA_SIZE_BYTES); + assert.equal(phase.commands[0]!.weight, 1.0); + assert.equal(workload.name(), 'unnamed'); + }); + + it('treats an explicit null as absent, not as null', () => { + // The Ruby/Python engines coerce nulls to defaults; a null leaking through + // would surface as NaN padding or a null prefix deep in the worker loop. + const workload = ConfigLoader.parseWorkloadConfig({ + phases: [ + { + ...MINIMAL_PHASE, + cps_limit: null, + rps_limit: null, + pipeline_depth: null, + warmup_requests: null, + keyspace: { keys_count: 50, key_size_bytes: null, key_prefix: null, generation_alg: null }, + commands: [{ command: 'set', weight: null, data_size_bytes: null }], + }, + ], + }); + const phase = workload.phases[0]!; + assert.equal(phase.cpsLimit, -1); + assert.equal(phase.pipelineDepth, 1); + assert.equal(phase.warmupRequests, 1); + assert.equal(phase.keyspace.keySizeBytes, DEFAULT_KEY_SIZE_BYTES); + assert.equal(phase.keyspace.keyPrefix, DEFAULT_KEY_PREFIX); + assert.equal(phase.keyspace.generationAlg, 'sequential_int'); + assert.equal(phase.commands[0]!.weight, 1.0); + assert.equal(phase.commands[0]!.dataSizeBytes, DEFAULT_DATA_SIZE_BYTES); + }); + + it('lower-cases command names', () => { + const workload = ConfigLoader.parseWorkloadConfig({ + phases: [{ ...MINIMAL_PHASE, commands: [{ command: 'GET', weight: 1 }] }], + }); + assert.equal(workload.phases[0]!.commands[0]!.command, 'get'); + }); + + it('recognises limits when set', () => { + const workload = ConfigLoader.parseWorkloadConfig({ + phases: [{ ...MINIMAL_PHASE, cps_limit: 10, rps_limit: 500 }], + }); + const phase = workload.phases[0]!; + assert.equal(phase.hasCpsLimit(), true); + assert.equal(phase.hasRpsLimit(), true); + }); + + it('rejects a workload with no phases', () => { + assert.throws(() => ConfigLoader.parseWorkloadConfig({ phases: [] }), /non-empty "phases"/); + assert.throws(() => ConfigLoader.parseWorkloadConfig({}), /non-empty "phases"/); + }); + + it('rejects a phase with no commands', () => { + assert.throws( + () => ConfigLoader.parseWorkloadConfig({ phases: [{ ...MINIMAL_PHASE, commands: [] }] }), + /non-empty "commands"/, + ); + }); + + it('rejects non-positive connections', () => { + assert.throws( + () => ConfigLoader.parseWorkloadConfig({ phases: [{ ...MINIMAL_PHASE, connections: 0 }] }), + /"connections" must be positive/, + ); + }); + + it('rejects a completion type that carries no target', () => { + assert.throws( + () => + ConfigLoader.parseWorkloadConfig({ + phases: [{ ...MINIMAL_PHASE, completion: { type: 'duration' } }], + }), + /requires a positive "seconds"/, + ); + assert.throws( + () => + ConfigLoader.parseWorkloadConfig({ + phases: [{ ...MINIMAL_PHASE, completion: { type: 'requests' } }], + }), + /requires a positive "requests"/, + ); + }); + + it('rejects an unknown generation_alg', () => { + assert.throws( + () => + ConfigLoader.parseWorkloadConfig({ + phases: [{ ...MINIMAL_PHASE, keyspace: { keys_count: 10, generation_alg: 'zipf' } }], + }), + /sequential_int or uniform_rand/, + ); + }); + + it('rejects a weight outside 0..1, as Java does', () => { + assert.throws( + () => + ConfigLoader.parseWorkloadConfig({ + phases: [{ ...MINIMAL_PHASE, commands: [{ command: 'get', weight: 5 }] }], + }), + /"weight" must be between 0 and 1/, + ); + }); +}); diff --git a/node/test/unit/factory.test.ts b/node/test/unit/factory.test.ts new file mode 100644 index 0000000..2ab2835 --- /dev/null +++ b/node/test/unit/factory.test.ts @@ -0,0 +1,115 @@ +import assert from 'node:assert/strict'; +import { describe, it } from 'node:test'; + +import { BenchmarkClientFactory } from '../../src/client/factory.js'; +import { packageVersion } from '../../src/client/driverVersion.js'; +import { CommandFactory } from '../../src/command/factory.js'; +import { CommandConfig } from '../../src/config/commandConfig.js'; +import { SetCommand } from '../../src/command/impl/setCommand.js'; + +describe('BenchmarkClientFactory', () => { + it('registers the node driver ids', () => { + assert.deepEqual(BenchmarkClientFactory.supportedDrivers(), [ + 'valkey-glide-node', + 'ioredis', + 'iovalkey', + 'recording', + ]); + }); + + it('does not register a bare valkey-glide', () => { + // scripts/run_benchmark_matrix.py's DRIVER_ENGINE_MAP is global and already + // maps "valkey-glide" to the Java engine. Claiming it here would silently + // reroute Java's glide runs to Node. + assert.equal(BenchmarkClientFactory.supportedDrivers().includes('valkey-glide'), false); + }); + + it('describes every driver for --info', () => { + for (const { driverId, description } of BenchmarkClientFactory.describe()) { + assert.ok(driverId.length > 0); + assert.ok(description.length > 0, `${driverId} has no description`); + } + }); + + it('rejects an unknown driver with the supported list', async () => { + await assert.rejects( + () => BenchmarkClientFactory.create('memcached'), + /Unknown driver: memcached\. Supported: valkey-glide-node, ioredis, iovalkey, recording/, + ); + }); + + it('is case-insensitive on driver_id', async () => { + const client = await BenchmarkClientFactory.create('IoRedis'); + assert.ok(client); + }); + + it('loads the recording driver without a server', async () => { + const client = await BenchmarkClientFactory.create('recording'); + assert.equal(client.driverVersion(), '1.0.0'); + }); +}); + +describe('CommandFactory', () => { + it('supports the cross-engine command set', () => { + assert.deepEqual(CommandFactory.supportedCommands(), ['get', 'set', 'ping']); + }); + + it('builds each command with its weight and name', () => { + const get = CommandFactory.create(new CommandConfig({ command: 'get', weight: 0.8 })); + assert.equal(get.name, 'GET'); + assert.equal(get.weight, 0.8); + assert.equal(get.usesKey, true); + + const ping = CommandFactory.create(new CommandConfig({ command: 'ping' })); + assert.equal(ping.name, 'PING'); + // PING must not consume a generated key: Java's PingCommand ignores the key + // generator, so advancing it here would shift the shared key sequence. + assert.equal(ping.usesKey, false); + }); + + it('accepts an upper-case command name', () => { + assert.equal(CommandFactory.create(new CommandConfig({ command: 'SET' })).name, 'SET'); + }); + + it('rejects an unknown command', () => { + assert.throws( + () => CommandFactory.create(new CommandConfig({ command: 'incr' })), + /Unknown command: incr\. Supported: get, set, ping/, + ); + }); + + it('createAll preserves order', () => { + const commands = CommandFactory.createAll([ + new CommandConfig({ command: 'get', weight: 0.8 }), + new CommandConfig({ command: 'set', weight: 0.2 }), + ]); + assert.deepEqual( + commands.map((c) => c.name), + ['GET', 'SET'], + ); + }); +}); + +describe('SetCommand payload', () => { + it('generates exactly data_size_bytes using the cross-engine pattern', () => { + // Ruby and Python use the same repeated "0123456789ABCDEF" filler. + for (const size of [1, 16, 32, 256, 512, 1000]) { + assert.equal(SetCommand.generateValue(size).length, size); + } + assert.equal(SetCommand.generateValue(20).toString('latin1'), '0123456789ABCDEF0123'); + }); +}); + +describe('packageVersion', () => { + it('reads a version despite a restricted exports map', () => { + // @valkey/valkey-glide does not export ./package.json, so a plain + // require('/package.json') throws ERR_PACKAGE_PATH_NOT_EXPORTED. + assert.match(packageVersion('@valkey/valkey-glide'), /^\d+\.\d+\.\d+/); + assert.match(packageVersion('ioredis'), /^\d+\.\d+\.\d+/); + assert.match(packageVersion('iovalkey'), /^\d+\.\d+\.\d+/); + }); + + it('returns "unknown" for a package that is not installed', () => { + assert.equal(packageVersion('definitely-not-installed-xyz'), 'unknown'); + }); +}); diff --git a/node/test/unit/hdrHistogram.test.ts b/node/test/unit/hdrHistogram.test.ts new file mode 100644 index 0000000..f39bef9 --- /dev/null +++ b/node/test/unit/hdrHistogram.test.ts @@ -0,0 +1,63 @@ +import assert from 'node:assert/strict'; +import { describe, it } from 'node:test'; + +import { + decodeBase64, + encodeBase64, + HIGHEST_TRACKABLE_VALUE, + LOWEST_TRACKABLE_VALUE, + newHistogram, + SIGNIFICANT_FIGURES, +} from '../../src/metrics/hdrHistogram.js'; + +describe('HDR histogram', () => { + it('uses the cross-engine range and precision', () => { + // Java SynchronizedHistogram(600_000_000, 3), C# LongConcurrentHistogram(1, + // 600_000_000, 3), Ruby HDRHistogram.new(1, 600_000_000, 3). + assert.equal(LOWEST_TRACKABLE_VALUE, 1); + assert.equal(HIGHEST_TRACKABLE_VALUE, 600_000_000); + assert.equal(SIGNIFICANT_FIGURES, 3); + const histogram = newHistogram(); + assert.equal(histogram.highestTrackableValue, 600_000_000); + assert.equal(histogram.numberOfSignificantValueDigits, 3); + }); + + it('emits a HIST-prefixed payload, not a double-encoded one', () => { + // Java's writer base64s the compressed bytes, which always start with the + // V2 cookie 0x1c849314 -> "HIST". A payload that does not start with HIST + // means it was base64-encoded twice and Java/Ruby cannot decode it. + const histogram = newHistogram(); + histogram.recordValue(1234); + const payload = encodeBase64(histogram); + assert.ok(payload.startsWith('HIST'), `payload should start with HIST, got ${payload.slice(0, 12)}`); + assert.doesNotMatch(payload, /^SElTVA/, 'payload is base64 of "HIST" — encoded twice'); + }); + + it('round-trips percentiles and total count', () => { + const histogram = newHistogram(); + for (let value = 1; value <= 1000; value++) histogram.recordValue(value); + + const decoded = decodeBase64(encodeBase64(histogram)); + assert.equal(decoded.totalCount, histogram.totalCount); + for (const percentile of [0, 50, 95, 99, 99.9, 100]) { + assert.equal( + decoded.getValueAtPercentile(percentile), + histogram.getValueAtPercentile(percentile), + `p${percentile} differs after round trip`, + ); + } + }); + + it('encodes an empty histogram without throwing', () => { + // An all-errors command still needs an hdr block in the NDJSON. + const payload = encodeBase64(newHistogram()); + assert.ok(payload.startsWith('HIST')); + assert.equal(decodeBase64(payload).totalCount, 0); + }); + + it('records the top of the range', () => { + const histogram = newHistogram(); + histogram.recordValue(HIGHEST_TRACKABLE_VALUE); + assert.equal(histogram.totalCount, 1); + }); +}); diff --git a/node/test/unit/javaRandom.test.ts b/node/test/unit/javaRandom.test.ts new file mode 100644 index 0000000..e1f7002 --- /dev/null +++ b/node/test/unit/javaRandom.test.ts @@ -0,0 +1,84 @@ +/** + * JavaRandom parity tests. + * + * The LCG is anchored to a well-known java.util.Random value, which proves + * byte-for-byte compatibility with the Java reference without needing a JVM. The + * two sequences below were additionally cross-checked against the Python + * engine's JavaRandom, so a failure here means Node has diverged from both. + */ + +import assert from 'node:assert/strict'; +import { describe, it } from 'node:test'; + +import { JavaRandom, toInt32 } from '../../src/engine/javaRandom.js'; + +describe('JavaRandom', () => { + it('matches the documented java.util.Random(0).nextInt() value', () => { + // java.util.Random(0).nextInt() -- i.e. next(32) as a signed int -- is the + // well-documented value -1155484576. This anchors the LCG to real Java. + assert.equal(toInt32(new JavaRandom(0).next(32)), -1155484576); + }); + + it('reproduces the cross-engine sequence for seed 12345, bound 1000', () => { + const rng = new JavaRandom(12345); + const actual = Array.from({ length: 10 }, () => rng.nextInt(1000)); + assert.deepEqual(actual, [251, 80, 241, 828, 55, 84, 375, 802, 501, 389]); + }); + + it('reproduces the cross-engine sequence for a power-of-two bound', () => { + // Exercises the power-of-two fast path, a separate branch in Java. + const rng = new JavaRandom(12345); + const actual = Array.from({ length: 8 }, () => rng.nextInt(256)); + assert.deepEqual(actual, [92, 131, 238, 234, 213, 9, 83, 31]); + }); + + it('is deterministic for a given seed', () => { + const rngA = new JavaRandom(12345); + const rngB = new JavaRandom(12345); + assert.deepEqual( + Array.from({ length: 10 }, () => rngA.nextInt(1000)), + Array.from({ length: 10 }, () => rngB.nextInt(1000)), + ); + }); + + it('produces different sequences for different seeds', () => { + const a = new JavaRandom(12345); + const b = new JavaRandom(54321); + assert.notDeepEqual( + Array.from({ length: 10 }, () => a.nextInt(1000)), + Array.from({ length: 10 }, () => b.nextInt(1000)), + ); + }); + + it('setSeed resets the stream', () => { + const rng = new JavaRandom(12345); + const first = Array.from({ length: 5 }, () => rng.nextInt(1000)); + rng.setSeed(12345); + const second = Array.from({ length: 5 }, () => rng.nextInt(1000)); + assert.deepEqual(first, second); + }); + + it('rejects a non-positive bound', () => { + const rng = new JavaRandom(12345); + assert.throws(() => rng.nextInt(0), /bound must be a positive integer/); + assert.throws(() => rng.nextInt(-1), /bound must be a positive integer/); + }); + + it('stays within the bound', () => { + const rng = new JavaRandom(12345); + for (let i = 0; i < 1000; i++) { + const value = rng.nextInt(100); + assert.ok(value >= 0 && value < 100, `${value} out of range`); + } + }); + + it('stays within power-of-two bounds', () => { + const rng = new JavaRandom(12345); + for (const bound of [2, 4, 8, 16, 256, 1024]) { + for (let i = 0; i < 200; i++) { + const value = rng.nextInt(bound); + assert.ok(value >= 0 && value < bound, `${value} out of range for ${bound}`); + } + } + }); +}); diff --git a/node/test/unit/keyGenerator.test.ts b/node/test/unit/keyGenerator.test.ts new file mode 100644 index 0000000..2f6195e --- /dev/null +++ b/node/test/unit/keyGenerator.test.ts @@ -0,0 +1,133 @@ +import assert from 'node:assert/strict'; +import { describe, it } from 'node:test'; + +import { KeyspaceConfig } from '../../src/config/keyspaceConfig.js'; +import { Counter, KeyGenerator } from '../../src/engine/keyGenerator.js'; + +describe('KeyGenerator', () => { + describe('sequential_int', () => { + it('emits 0..N-1 then wraps', () => { + const gen = KeyGenerator.create( + new KeyspaceConfig({ keysCount: 3, keyPrefix: 'test:', keySizeBytes: 6 }), + ); + assert.deepEqual( + Array.from({ length: 4 }, () => gen.nextKey()), + ['test:0', 'test:1', 'test:2', 'test:0'], + ); + }); + + it('shares one counter across workers, as Java does', () => { + // The whole point of the shared counter: several connections collectively + // populate the keyspace instead of each replaying 0, 1, 2, ... + const config = new KeyspaceConfig({ keysCount: 100, keyPrefix: 'k:', keySizeBytes: 4 }); + const counter = new Counter(); + const workerA = KeyGenerator.createWithSeed(config, 0, counter); + const workerB = KeyGenerator.createWithSeed(config, 1, counter); + assert.deepEqual( + [workerA.nextKey(), workerB.nextKey(), workerA.nextKey(), workerB.nextKey()], + ['k:00', 'k:01', 'k:02', 'k:03'], + ); + }); + + it('gives each generator its own counter when none is shared', () => { + const config = new KeyspaceConfig({ keysCount: 100, keyPrefix: 'k:', keySizeBytes: 4 }); + assert.equal(KeyGenerator.create(config).nextKey(), 'k:00'); + assert.equal(KeyGenerator.create(config).nextKey(), 'k:00'); + }); + }); + + describe('uniform_rand', () => { + it('is reproducible for the same seed', () => { + const config = new KeyspaceConfig({ + keysCount: 1000, + keyPrefix: 'test:', + generationAlg: 'uniform_rand', + seed: 12345, + }); + const a = KeyGenerator.create(config); + const b = KeyGenerator.create(config); + for (let i = 0; i < 100; i++) assert.equal(a.nextKey(), b.nextKey()); + }); + + it('derives a per-worker seed of base + index, as Java does', () => { + const config = new KeyspaceConfig({ + keysCount: 1000, + keyPrefix: 'test:', + generationAlg: 'uniform_rand', + seed: 12345, + }); + const worker0 = KeyGenerator.createWithSeed(config, 12345); + const worker1 = KeyGenerator.createWithSeed(config, 12346); + assert.notEqual(worker0.nextKey(), worker1.nextKey()); + }); + + it('matches the JavaRandom sequence for seed 12345', () => { + // Anchored to the same values as javaRandom.test.ts, so a key-formatting + // change cannot silently break cross-engine key parity. + const gen = KeyGenerator.create( + new KeyspaceConfig({ + keysCount: 1000, + keyPrefix: 'bench:', + keySizeBytes: 16, + generationAlg: 'uniform_rand', + seed: 12345, + }), + ); + assert.deepEqual( + Array.from({ length: 4 }, () => gen.nextKey()), + ['bench:0000000251', 'bench:0000000080', 'bench:0000000241', 'bench:0000000828'], + ); + }); + + it('reset restores the stream', () => { + const gen = KeyGenerator.create( + new KeyspaceConfig({ + keysCount: 1000, + keyPrefix: 'test:', + generationAlg: 'uniform_rand', + seed: 999, + }), + ); + const first = Array.from({ length: 5 }, () => gen.nextKey()); + gen.reset(); + assert.deepEqual( + Array.from({ length: 5 }, () => gen.nextKey()), + first, + ); + }); + }); + + describe('key formatting', () => { + it('zero-pads to key_size_bytes minus the prefix, as Java does', () => { + // Reference configs use key_prefix "bench:" (6) and key_size_bytes 16, + // so the numeric part is 10 digits: bench:0000000042 + const gen = KeyGenerator.create( + new KeyspaceConfig({ keysCount: 1_000_000, keyPrefix: 'bench:', keySizeBytes: 16 }), + ); + const key = gen.nextKey(); + assert.equal(key, 'bench:0000000000'); + assert.equal(key.length, 16); + }); + + it('keeps at least one digit when the prefix fills key_size_bytes', () => { + const gen = KeyGenerator.create( + new KeyspaceConfig({ keysCount: 10, keyPrefix: 'averylongprefix:', keySizeBytes: 4 }), + ); + assert.equal(gen.nextKey(), 'averylongprefix:0'); + }); + + it('does not truncate an index wider than the padding', () => { + const gen = KeyGenerator.create( + new KeyspaceConfig({ keysCount: 1000, keyPrefix: 'k:', keySizeBytes: 4 }), + ); + const keys = Array.from({ length: 101 }, () => gen.nextKey()); + assert.equal(keys[0], 'k:00'); + assert.equal(keys[100], 'k:100'); + }); + + it('applies the documented defaults', () => { + const gen = KeyGenerator.create(new KeyspaceConfig({ keysCount: 10 })); + assert.equal(gen.nextKey(), 'bench:0000000000'); + }); + }); +}); diff --git a/node/test/unit/ndjsonWriter.test.ts b/node/test/unit/ndjsonWriter.test.ts new file mode 100644 index 0000000..7bb3e22 --- /dev/null +++ b/node/test/unit/ndjsonWriter.test.ts @@ -0,0 +1,193 @@ +import assert from 'node:assert/strict'; +import { mkdtempSync, readFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { describe, it } from 'node:test'; + +import { MetricsCollector } from '../../src/metrics/collector.js'; +import { decodeBase64 } from '../../src/metrics/hdrHistogram.js'; +import { NdjsonWriter } from '../../src/metrics/ndjsonWriter.js'; + +function tempPath(name = 'metrics.ndjson'): string { + return join(mkdtempSync(join(tmpdir(), 'resp-bench-node-')), name); +} + +function collectorWith(samples: Array<{ name: string; latency: number; ok?: boolean }>) { + const collector = new MetricsCollector(); + collector.start(); + for (const { name, latency, ok = true } of samples) { + collector.record({ commandName: name, latencyMicros: latency, success: ok }); + } + collector.stop(); + return collector; +} + +function readRecords(path: string): Array> { + return readFileSync(path, 'utf8') + .trimEnd() + .split('\n') + .map((line) => JSON.parse(line)); +} + +describe('NdjsonWriter', () => { + it('writes the documented record shape', () => { + const path = tempPath(); + const writer = new NdjsonWriter(path); + writer.setMetadata({ + commitId: 'abc123', + driverId: 'ioredis', + primaryDriverVersion: '5.11.1', + }); + writer.writePhaseResults({ + phaseId: 'STEADY', + status: 'COMPLETED', + connections: 4, + collector: collectorWith([ + { name: 'GET', latency: 100 }, + { name: 'GET', latency: 200 }, + { name: 'SET', latency: 300 }, + ]), + }); + + const [record] = readRecords(path); + assert.ok(record); + + assert.deepEqual(Object.keys(record).sort(), ['metadata', 'metrics', 'phase', 'totals']); + assert.equal(record['metadata'].commit_id, 'abc123'); + assert.equal(record['metadata'].driver_id, 'ioredis'); + assert.equal(record['metadata'].primary_driver_version, '5.11.1'); + assert.ok(record['metadata'].timestamp); + // Absent secondary driver fields must be omitted, not null. + assert.equal('secondary_driver_id' in record['metadata'], false); + + assert.deepEqual(Object.keys(record['phase']).sort(), [ + 'connections', + 'duration_ms', + 'finish_timestamp', + 'id', + 'start_timestamp', + 'status', + ]); + assert.equal(record['phase'].id, 'STEADY'); + assert.equal(record['phase'].status, 'COMPLETED'); + assert.equal(record['phase'].connections, 4); + assert.match(record['phase'].start_timestamp, /^\d{4}-\d{2}-\d{2}T.*Z$/); + + assert.deepEqual(record['totals'], { requests: 3, errors: 0 }); + + assert.deepEqual(Object.keys(record['metrics']).sort(), ['GET', 'SET']); + const get = record['metrics'].GET; + assert.equal(get.requests, 2); + assert.equal(get.errors, 0); + assert.equal(get.latency.unit, 'us'); + assert.equal(get.latency.count, 2); + assert.deepEqual(Object.keys(get.latency.summary).sort(), [ + 'max', + 'min', + 'p50', + 'p95', + 'p99', + 'p999', + ]); + assert.equal(get.latency.hdr.format, 'hdr'); + assert.equal(get.latency.hdr.sigfig, 3); + assert.ok(get.latency.hdr.payload_b64.startsWith('HIST')); + }); + + it('writes one compact line per phase and appends across phases', () => { + const path = tempPath(); + const writer = new NdjsonWriter(path); + writer.setMetadata({ driverId: 'ioredis' }); + for (const phaseId of ['WARMUP', 'STEADY']) { + writer.writePhaseResults({ + phaseId, + status: 'COMPLETED', + connections: 1, + collector: collectorWith([{ name: 'GET', latency: 10 }]), + }); + } + + const raw = readFileSync(path, 'utf8'); + assert.equal(raw.endsWith('\n'), true); + const lines = raw.trimEnd().split('\n'); + assert.equal(lines.length, 2); + // NDJSON requires no embedded newlines -- no pretty printing. + for (const line of lines) assert.doesNotMatch(line, /\n/); + assert.deepEqual( + lines.map((line) => JSON.parse(line).phase.id), + ['WARMUP', 'STEADY'], + ); + }); + + it('creates the parent directory when it does not exist', () => { + const path = join(mkdtempSync(join(tmpdir(), 'resp-bench-node-')), 'nested', 'deep', 'm.ndjson'); + const writer = new NdjsonWriter(path); + writer.setMetadata({ driverId: 'ioredis' }); + writer.writePhaseResults({ + phaseId: 'P', + status: 'COMPLETED', + connections: 1, + collector: collectorWith([{ name: 'GET', latency: 10 }]), + }); + assert.equal(readRecords(path).length, 1); + }); + + it('emits an hdr block for a command that only ever errored', () => { + // Matches Java: the histogram is created eagerly, so analysis tooling can + // always read metrics..latency.hdr without a null check. + const path = tempPath(); + const writer = new NdjsonWriter(path); + writer.setMetadata({ driverId: 'ioredis' }); + writer.writePhaseResults({ + phaseId: 'ERRORS', + status: 'COMPLETED', + connections: 1, + collector: collectorWith([ + { name: 'GET', latency: 50, ok: false }, + { name: 'GET', latency: 60, ok: false }, + ]), + }); + + const [record] = readRecords(path); + const get = record!['metrics'].GET; + assert.equal(get.requests, 2); + assert.equal(get.errors, 2); + assert.equal(get.latency.count, 0); + assert.deepEqual(get.latency.summary, { min: 0, p50: 0, p95: 0, p99: 0, p999: 0, max: 0 }); + assert.ok(get.latency.hdr.payload_b64.startsWith('HIST')); + assert.equal(decodeBase64(get.latency.hdr.payload_b64).totalCount, 0); + assert.deepEqual(record!['totals'], { requests: 2, errors: 2 }); + }); + + it('omits the metadata block entirely when nothing identifies the run', () => { + const path = tempPath(); + const writer = new NdjsonWriter(path); + writer.writePhaseResults({ + phaseId: 'P', + status: 'COMPLETED', + connections: 1, + collector: collectorWith([{ name: 'GET', latency: 10 }]), + }); + assert.equal('metadata' in readRecords(path)[0]!, false); + }); + + it('carries the secondary driver fields when present', () => { + const path = tempPath(); + const writer = new NdjsonWriter(path); + writer.setMetadata({ + driverId: 'composite', + primaryDriverVersion: '1.0.0', + secondaryDriverId: 'ioredis', + secondaryDriverVersion: '5.11.1', + }); + writer.writePhaseResults({ + phaseId: 'P', + status: 'COMPLETED', + connections: 1, + collector: collectorWith([{ name: 'GET', latency: 10 }]), + }); + const metadata = readRecords(path)[0]!['metadata']; + assert.equal(metadata.secondary_driver_id, 'ioredis'); + assert.equal(metadata.secondary_driver_version, '5.11.1'); + }); +}); diff --git a/node/test/unit/rateLimiter.test.ts b/node/test/unit/rateLimiter.test.ts new file mode 100644 index 0000000..e39f96a --- /dev/null +++ b/node/test/unit/rateLimiter.test.ts @@ -0,0 +1,82 @@ +import assert from 'node:assert/strict'; +import { describe, it } from 'node:test'; + +import { RateLimiter } from '../../src/engine/rateLimiter.js'; + +describe('RateLimiter', () => { + it('returns null for an unlimited rate', () => { + assert.equal(RateLimiter.create(0), null); + assert.equal(RateLimiter.create(-1), null); + }); + + it('returns a limiter for a positive rate', () => { + const limiter = RateLimiter.create(100); + assert.ok(limiter); + assert.equal(limiter.ratePerSecond, 100); + }); + + it('achieves the target rate within 5%', async () => { + // docs/ADDING_LANGUAGE.md's stated tolerance for the rate limiter. + const rate = 200; + const requests = 100; + const limiter = RateLimiter.create(rate)!; + + const start = process.hrtime.bigint(); + for (let i = 0; i < requests; i++) await limiter.acquire(); + const elapsedSeconds = Number(process.hrtime.bigint() - start) / 1e9; + + // The first acquire is free (the bucket starts open), so the limiter only + // paces the remaining requests. + const expectedSeconds = (requests - 1) / rate; + assert.ok( + elapsedSeconds >= expectedSeconds * 0.95, + `finished too fast: ${elapsedSeconds.toFixed(3)}s < ${(expectedSeconds * 0.95).toFixed(3)}s`, + ); + assert.ok( + elapsedSeconds <= expectedSeconds * 1.3, + `finished too slow: ${elapsedSeconds.toFixed(3)}s > ${(expectedSeconds * 1.3).toFixed(3)}s`, + ); + }); + + it('paces a rate whose interval is below setTimeout resolution', async () => { + // 5000/s is a 200us interval -- well under setTimeout's ~1ms floor, so this + // only passes if sub-millisecond waits yield via setImmediate instead. + const rate = 5000; + const requests = 500; + const limiter = RateLimiter.create(rate)!; + + const start = process.hrtime.bigint(); + for (let i = 0; i < requests; i++) await limiter.acquire(); + const elapsedSeconds = Number(process.hrtime.bigint() - start) / 1e9; + + const expectedSeconds = (requests - 1) / rate; + assert.ok( + elapsedSeconds <= expectedSeconds * 2, + `sub-ms pacing overshot badly: ${elapsedSeconds.toFixed(3)}s vs ${expectedSeconds.toFixed(3)}s ` + + '(a clamped setTimeout would take ~10x this)', + ); + }); + + it('shares one budget across concurrent callers', async () => { + const rate = 200; + const perWorker = 25; + const workers = 4; + const limiter = RateLimiter.create(rate)!; + + const start = process.hrtime.bigint(); + await Promise.all( + Array.from({ length: workers }, async () => { + for (let i = 0; i < perWorker; i++) await limiter.acquire(); + }), + ); + const elapsedSeconds = Number(process.hrtime.bigint() - start) / 1e9; + + // The limit is global: 100 requests at 200/s takes ~0.5s regardless of how + // many workers issue them. + const expectedSeconds = (workers * perWorker - 1) / rate; + assert.ok( + elapsedSeconds >= expectedSeconds * 0.95, + `concurrent callers bypassed the limit: ${elapsedSeconds.toFixed(3)}s`, + ); + }); +}); diff --git a/node/test/unit/recordingClient.test.ts b/node/test/unit/recordingClient.test.ts new file mode 100644 index 0000000..19af4f6 --- /dev/null +++ b/node/test/unit/recordingClient.test.ts @@ -0,0 +1,101 @@ +import assert from 'node:assert/strict'; +import { describe, it } from 'node:test'; + +import { RecordingClient } from '../../src/client/impl/recordingClient.js'; +import { ConfigLoader } from '../../src/config/loader.js'; + +function config(specific: Record = {}) { + return ConfigLoader.parseDriverConfig({ + driver_id: 'recording', + mode: 'standalone', + specific_driver_config: specific, + }); +} + +async function connected(specific: Record = {}): Promise { + const client = new RecordingClient(); + await client.connect('localhost', 6379, config(specific)); + return client; +} + +describe('RecordingClient', () => { + it('records the operations it was asked to perform', async () => { + const client = await connected(); + await client.ping(); + await client.set('bench:0000000001', Buffer.from('abc', 'latin1')); + await client.get('bench:0000000001'); + await client.close(); + + assert.deepEqual( + client.operations.map((op) => op.command), + ['CONNECT', 'PING', 'SET', 'GET', 'CLOSE'], + ); + // The keys the engine passed through are observable, so a key-generation + // regression shows up as a wrong key here rather than as silent drift. + assert.deepEqual( + client.operations.filter((op) => op.key !== null).map((op) => op.key), + ['bench:0000000001', 'bench:0000000001'], + ); + assert.ok(client.operations.every((op) => op.success)); + }); + + it('behaves like a key-value store for GET after SET', async () => { + const client = await connected(); + assert.equal((await client.get('missing')).value, null); + await client.set('k', Buffer.from('hello', 'latin1')); + assert.equal((await client.get('k')).value, 'hello'); + }); + + it('returns the cross-engine success values', async () => { + const client = await connected(); + assert.equal((await client.ping()).value, 'PONG'); + assert.equal((await client.set('k', Buffer.alloc(4))).value, 'OK'); + }); + + it('injects errors at error_rate 1.0 with the configured message', async () => { + const client = await connected({ error_rate: 1.0, error_message: 'boom' }); + for (const result of [ + await client.ping(), + await client.get('k'), + await client.set('k', Buffer.alloc(1)), + ]) { + assert.equal(result.error?.message, 'boom'); + assert.equal(result.value, null); + } + assert.ok(client.operations.slice(1).every((op) => !op.success)); + }); + + it('never injects errors while in warmup mode', async () => { + // Mirrors Java's setWarmupMode: the engine's warmup fail-fast must not be + // tripped by errors the workload deliberately injects. + const client = await connected({ error_rate: 1.0 }); + client.setWarmupMode(true); + assert.equal((await client.ping()).error, undefined); + client.setWarmupMode(false); + assert.notEqual((await client.ping()).error, undefined); + }); + + it('does not fail a SET when its own error is injected mid-store', async () => { + // A failed SET must not store, or a later GET would report data the server + // never accepted. + const client = await connected({ error_rate: 1.0 }); + await client.set('k', Buffer.from('nope', 'latin1')); + client.setWarmupMode(true); + assert.equal((await client.get('k')).value, null); + }); + + it('applies a configured operation delay', async () => { + const client = await connected({ operation_delay_micros: 5000 }); + const result = await client.ping(); + assert.ok(result.latencyMicros >= 3000, `latency ${result.latencyMicros}us too low for a 5ms delay`); + }); + + it('reports zero-ish latency with no configured delay', async () => { + const client = await connected(); + assert.ok((await client.ping()).latencyMicros < 5000); + }); + + it('reports a fixed driver version', async () => { + assert.equal(new RecordingClient().driverVersion(), '1.0.0'); + }); +}); diff --git a/node/tsconfig.json b/node/tsconfig.json new file mode 100644 index 0000000..8f1e804 --- /dev/null +++ b/node/tsconfig.json @@ -0,0 +1,23 @@ +{ + "compilerOptions": { + "target": "ES2022", + "lib": ["ES2022"], + "module": "NodeNext", + "moduleResolution": "NodeNext", + "rootDir": ".", + "outDir": "dist", + "sourceMap": true, + "declaration": false, + "incremental": true, + "tsBuildInfoFile": "dist/.tsbuildinfo", + + "strict": true, + "noUncheckedIndexedAccess": true, + "noImplicitOverride": true, + "noFallthroughCasesInSwitch": true, + "exactOptionalPropertyTypes": false, + "forceConsistentCasingInFileNames": true, + "skipLibCheck": true + }, + "include": ["src/**/*.ts", "test/**/*.ts"] +} diff --git a/scripts/generate_graphs.py b/scripts/generate_graphs.py index 177a7e3..3f6a276 100644 --- a/scripts/generate_graphs.py +++ b/scripts/generate_graphs.py @@ -71,6 +71,10 @@ # C# drivers "stackexchange-redis": "csharp", "valkey-glide-csharp": "csharp", + # Node.js drivers + "valkey-glide-node": "node", + "ioredis": "node", + "iovalkey": "node", # Python drivers (future) "redis-py": "python", "aioredis": "python", @@ -118,7 +122,10 @@ def parse_args(): ) parser.add_argument( "--language", - choices=["java", "ruby", "csharp", "python"], + # Keep in sync with the values in DRIVER_LANGUAGE_MAP — argparse rejects + # anything not listed here, so a language added to the map alone fails at + # the CLI rather than silently producing empty graphs. + choices=["java", "ruby", "csharp", "node", "python"], help="Filter results by language (only include drivers for this language)", ) parser.add_argument( diff --git a/scripts/generate_interactive_graphs.py b/scripts/generate_interactive_graphs.py index 3523185..34af16b 100644 --- a/scripts/generate_interactive_graphs.py +++ b/scripts/generate_interactive_graphs.py @@ -680,6 +680,11 @@ def load_cpu_data(results_dir, rps_outlier_map=None): # C# (.NET) drivers — cyan/pink "stackexchange-redis": "#00ACC1", # cyan 600 "valkey-glide-csharp": "#D81B60", # pink 600 + + # Node.js drivers — yellows/browns + "valkey-glide-node": "#F9A825", # yellow 800 + "ioredis": "#6D4C41", # brown 600 + "iovalkey": "#546E7A", # blue grey 600 } DRIVER_FAMILIES = { @@ -694,15 +699,19 @@ def load_cpu_data(results_dir, rps_outlier_map=None): "redisson": "low-level", "stackexchange-redis": "csharp", "valkey-glide-csharp": "csharp", + "valkey-glide-node": "node", + "ioredis": "node", + "iovalkey": "node", } -FAMILY_ORDER = ["spring-data-valkey", "spring-data-redis", "low-level", "csharp"] +FAMILY_ORDER = ["spring-data-valkey", "spring-data-redis", "low-level", "csharp", "node"] FAMILY_LABELS = { "spring-data-valkey": "Spring Data Valkey", "spring-data-redis": "Spring Data Redis", "low-level": "Low-Level Java Drivers", "csharp": "C# (.NET) Drivers", + "node": "Node.js Drivers", } # Fallback color for unknown drivers diff --git a/scripts/run_benchmark_matrix.py b/scripts/run_benchmark_matrix.py index 4bfcf9d..f40ab6c 100644 --- a/scripts/run_benchmark_matrix.py +++ b/scripts/run_benchmark_matrix.py @@ -124,6 +124,10 @@ # C# drivers "stackexchange-redis": "csharp", "valkey-glide-csharp": "csharp", + # Node.js drivers + "valkey-glide-node": "node", + "ioredis": "node", + "iovalkey": "node", # Recording (default to java) "recording": "java", }