From 1118c7a7baa861d275374f30eadaf580e47b6e9b Mon Sep 17 00:00:00 2001 From: James Xin Date: Tue, 8 Sep 2026 10:17:00 -0700 Subject: [PATCH 1/5] Add Node.js benchmark engine (valkey-glide-node, ioredis, iovalkey) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit TypeScript engine under node/, compiled with tsc to node/dist. One event loop, one client per connection, one worker per connection; `pipeline_depth > 1` gives each connection that many independent in-flight slots. Drivers: `valkey-glide-node`, `ioredis`, `iovalkey` (+ `recording` for server-free tests). The GLIDE id is deliberately not bare `valkey-glide` — that is already Java's in the global DRIVER_ENGINE_MAP, and reusing it would reroute Java's glide runs here. Cross-engine parity verified against the Java reference, not assumed: - Key sequences byte-identical: 79,000 keys diffed against Java's real KeyGenerator across both algorithms, 1-16 workers, prime keys_count, and tight prefix padding. javaRandom.ts uses BigInt because the 48-bit LCG multiply reaches ~2^83, past what a JS number holds exactly. - HDR payloads decode in Java (org.HdrHistogram.Histogram) with matching count and percentiles. payload_b64 is encodeIntoCompressedBase64() used directly — it is already base64. - summary.min/max use getValueAtPercentile(0/100), not minNonZeroValue/maxValue. Java returns bucket-equivalent bounds; the JS properties return the raw sample, and they diverge above ~1000us (50000us reads back as 50015 in Java). - Request budget is shared across workers and claimed per request, matching Java's per-phase AtomicLong rather than pre-splitting it. - PING does not consume a key, matching Java's PingCommand. Node-specific fairness controls: ioredis/iovalkey auto-pipelining forced off (it would batch same-tick commands and inflate throughput), reconnects disabled (the default retries forever, hanging a run on a wrong host), uniform string decoding across drivers, SET payloads allocated once, and sub-millisecond rate limits yielding via setImmediate since setTimeout clamps to ~1ms. Harness: Makefile targets, DRIVER_ENGINE_MAP, both graph scripts, driver configs (default/high-throughput/example), schema examples, a benchmark-node CI job, a Node.js block in infra/provision.sh, and config-editor driver list. Docs: node/README.md and docs/BENCHMARKS_NODE.md, which records the measured single-core ceiling (throughput plateaus ~42k rps at 200 connections; at pipeline_depth 16 the process pins 99% of one core at ~72k rps, so past that the engine and not the driver is the limit). Java solves this with parallel issuer threads; the worker_threads equivalent is left as a follow-up to be justified by measurement. Also fixes the HDR range in docs/ADDING_LANGUAGE.md (3600000000 -> 600000000, the value every engine actually uses) and extends its validation checklist with the parity traps found here. 124 tests: unit (no server), full-engine tests via the recording driver, and live-server tests per driver. Signed-off-by: James Xin --- .github/workflows/benchmark.yml | 75 +++- Makefile | 50 ++- README.md | 9 +- config-editor/src/App.tsx | 7 +- configs/drivers/default/ioredis.json | 7 + configs/drivers/default/iovalkey.json | 7 + .../drivers/default/valkey-glide-node.json | 7 + .../drivers/example-ioredis-standalone.json | 7 + .../drivers/example-iovalkey-standalone.json | 7 + .../example-valkey-glide-node-standalone.json | 7 + configs/drivers/high-throughput/ioredis.json | 8 + configs/drivers/high-throughput/iovalkey.json | 8 + .../high-throughput/valkey-glide-node.json | 8 + configs/schemas/driver-config.schema.json | 2 +- docs/ADDING_LANGUAGE.md | 31 +- docs/ARCHITECTURE.md | 19 +- docs/BENCHMARKS_NODE.md | 147 ++++++ docs/CONFIG_SPECIFICATION.md | 5 + infra/provision.sh | 32 +- node/.gitignore | 4 + node/README.md | 141 ++++++ node/package-lock.json | 420 ++++++++++++++++++ node/package.json | 30 ++ node/src/cli.ts | 142 ++++++ node/src/client/benchmarkClient.ts | 59 +++ node/src/client/driverVersion.ts | 45 ++ node/src/client/factory.ts | 81 ++++ node/src/client/impl/glideClient.ts | 88 ++++ node/src/client/impl/ioredisClient.ts | 16 + node/src/client/impl/ioredisFamilyClient.ts | 131 ++++++ node/src/client/impl/iovalkeyClient.ts | 19 + node/src/client/impl/recordingClient.ts | 131 ++++++ node/src/client/timedResult.ts | 12 + node/src/command/command.ts | 19 + node/src/command/factory.ts | 39 ++ node/src/command/impl/getCommand.ts | 23 + node/src/command/impl/pingCommand.ts | 23 + node/src/command/impl/setCommand.ts | 41 ++ node/src/config/commandConfig.ts | 21 + node/src/config/completionConfig.ts | 29 ++ node/src/config/driverConfig.ts | 80 ++++ node/src/config/keyspaceConfig.ts | 41 ++ node/src/config/loader.ts | 198 +++++++++ node/src/config/phaseConfig.ts | 58 +++ node/src/config/workloadConfig.ts | 29 ++ node/src/engine/benchmark.ts | 397 +++++++++++++++++ node/src/engine/commandSelector.ts | 43 ++ node/src/engine/javaRandom.ts | 68 +++ node/src/engine/keyGenerator.ts | 93 ++++ node/src/engine/rateLimiter.ts | 66 +++ node/src/metrics/collector.ts | 111 +++++ node/src/metrics/hdrHistogram.ts | 48 ++ node/src/metrics/ndjsonWriter.ts | 122 +++++ node/src/version.ts | 2 + node/test/integration/clients.test.ts | 197 ++++++++ .../integration/recordingWorkload.test.ts | 297 +++++++++++++ node/test/unit/collector.test.ts | 97 ++++ node/test/unit/commandSelector.test.ts | 59 +++ node/test/unit/configLoader.test.ts | 269 +++++++++++ node/test/unit/factory.test.ts | 115 +++++ node/test/unit/hdrHistogram.test.ts | 63 +++ node/test/unit/javaRandom.test.ts | 84 ++++ node/test/unit/keyGenerator.test.ts | 133 ++++++ node/test/unit/ndjsonWriter.test.ts | 193 ++++++++ node/test/unit/rateLimiter.test.ts | 82 ++++ node/test/unit/recordingClient.test.ts | 101 +++++ node/tsconfig.json | 23 + scripts/generate_graphs.py | 4 + scripts/generate_interactive_graphs.py | 11 +- scripts/run_benchmark_matrix.py | 4 + 70 files changed, 5031 insertions(+), 14 deletions(-) create mode 100644 configs/drivers/default/ioredis.json create mode 100644 configs/drivers/default/iovalkey.json create mode 100644 configs/drivers/default/valkey-glide-node.json create mode 100644 configs/drivers/example-ioredis-standalone.json create mode 100644 configs/drivers/example-iovalkey-standalone.json create mode 100644 configs/drivers/example-valkey-glide-node-standalone.json create mode 100644 configs/drivers/high-throughput/ioredis.json create mode 100644 configs/drivers/high-throughput/iovalkey.json create mode 100644 configs/drivers/high-throughput/valkey-glide-node.json create mode 100644 docs/BENCHMARKS_NODE.md create mode 100644 node/.gitignore create mode 100644 node/README.md create mode 100644 node/package-lock.json create mode 100644 node/package.json create mode 100644 node/src/cli.ts create mode 100644 node/src/client/benchmarkClient.ts create mode 100644 node/src/client/driverVersion.ts create mode 100644 node/src/client/factory.ts create mode 100644 node/src/client/impl/glideClient.ts create mode 100644 node/src/client/impl/ioredisClient.ts create mode 100644 node/src/client/impl/ioredisFamilyClient.ts create mode 100644 node/src/client/impl/iovalkeyClient.ts create mode 100644 node/src/client/impl/recordingClient.ts create mode 100644 node/src/client/timedResult.ts create mode 100644 node/src/command/command.ts create mode 100644 node/src/command/factory.ts create mode 100644 node/src/command/impl/getCommand.ts create mode 100644 node/src/command/impl/pingCommand.ts create mode 100644 node/src/command/impl/setCommand.ts create mode 100644 node/src/config/commandConfig.ts create mode 100644 node/src/config/completionConfig.ts create mode 100644 node/src/config/driverConfig.ts create mode 100644 node/src/config/keyspaceConfig.ts create mode 100644 node/src/config/loader.ts create mode 100644 node/src/config/phaseConfig.ts create mode 100644 node/src/config/workloadConfig.ts create mode 100644 node/src/engine/benchmark.ts create mode 100644 node/src/engine/commandSelector.ts create mode 100644 node/src/engine/javaRandom.ts create mode 100644 node/src/engine/keyGenerator.ts create mode 100644 node/src/engine/rateLimiter.ts create mode 100644 node/src/metrics/collector.ts create mode 100644 node/src/metrics/hdrHistogram.ts create mode 100644 node/src/metrics/ndjsonWriter.ts create mode 100644 node/src/version.ts create mode 100644 node/test/integration/clients.test.ts create mode 100644 node/test/integration/recordingWorkload.test.ts create mode 100644 node/test/unit/collector.test.ts create mode 100644 node/test/unit/commandSelector.test.ts create mode 100644 node/test/unit/configLoader.test.ts create mode 100644 node/test/unit/factory.test.ts create mode 100644 node/test/unit/hdrHistogram.test.ts create mode 100644 node/test/unit/javaRandom.test.ts create mode 100644 node/test/unit/keyGenerator.test.ts create mode 100644 node/test/unit/ndjsonWriter.test.ts create mode 100644 node/test/unit/rateLimiter.test.ts create mode 100644 node/test/unit/recordingClient.test.ts create mode 100644 node/tsconfig.json diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml index 8aebd34..13f3a85 100644 --- a/.github/workflows/benchmark.yml +++ b/.github/workflows/benchmark.yml @@ -152,8 +152,81 @@ jobs: path: ${{ steps.names.outputs.result_file }} retention-days: 30 + benchmark-node: + runs-on: ubuntu-latest + + strategy: + fail-fast: false + matrix: + driver: + - configs/drivers/default/valkey-glide-node.json + - configs/drivers/default/ioredis.json + - configs/drivers/default/iovalkey.json + workload: + - configs/workloads/reference/basic-standalone-single-client-1M-reqs.json + + steps: + - uses: actions/checkout@v4 + + # Node 22 LTS: ioredis 6 and node-redis 6 both require >= 20, and 18 is EOL. + - name: Set up Node.js + uses: actions/setup-node@v4 + with: + node-version: '22' + cache: npm + cache-dependency-path: node/package-lock.json + + - name: Install build dependencies + run: | + sudo apt-get update + sudo apt-get install -y build-essential + + - name: Start Valkey server + run: | + # Use Makefile target which builds from source and configures with persistence disabled + make server-standalone-start + # Wait for server to be ready + sleep 2 + # Verify server is up and persistence is disabled + work/valkey/bin/valkey-cli ping + work/valkey/bin/valkey-cli CONFIG GET save + + - name: Build Node.js benchmark + run: cd node && npm ci && npm run build + + - name: Extract names for result file + id: names + run: | + DRIVER_NAME=$(basename ${{ matrix.driver }} .json) + WORKLOAD_NAME=$(basename ${{ matrix.workload }} .json) + echo "driver_name=$DRIVER_NAME" >> $GITHUB_OUTPUT + echo "workload_name=$WORKLOAD_NAME" >> $GITHUB_OUTPUT + echo "result_file=results/github-runner/reference/${DRIVER_NAME}-${WORKLOAD_NAME}.ndjson" >> $GITHUB_OUTPUT + + - name: Run benchmark + run: | + mkdir -p results/github-runner/reference + node node/dist/src/cli.js \ + --server localhost:6379 \ + --driver ${{ matrix.driver }} \ + --workload ${{ matrix.workload }} \ + --metrics ${{ steps.names.outputs.result_file }} \ + --commit-id ${{ github.sha }} + + - name: Stop Valkey server + if: always() + run: | + make server-standalone-stop || true + + - name: Upload results + uses: actions/upload-artifact@v4 + with: + name: benchmark-node-${{ steps.names.outputs.driver_name }}-${{ steps.names.outputs.workload_name }} + path: ${{ steps.names.outputs.result_file }} + retention-days: 30 + generate-graphs: - needs: [benchmark-java, benchmark-ruby] + needs: [benchmark-java, benchmark-ruby, benchmark-node] runs-on: ubuntu-latest permissions: contents: write diff --git a/Makefile b/Makefile index db7d5d2..ee1d868 100644 --- a/Makefile +++ b/Makefile @@ -34,6 +34,8 @@ WORK_DIR=$(shell pwd)/work python-build python-test python-run python-clean \ ruby-build ruby-test ruby-run ruby-clean ruby-info \ csharp-build csharp-test csharp-run csharp-clean csharp-info \ + node-build node-test node-unit-test node-integration-test \ + node-run node-clean node-info \ config-editor-build config-editor-dev # ============================================================================ @@ -74,6 +76,13 @@ help: @echo " make csharp-clean Clean C# build artifacts" @echo " make csharp-info Show supported C# drivers and commands" @echo "" + @echo "Node.js Engine:" + @echo " make node-build Install deps and compile the Node.js engine" + @echo " make node-test Run Node.js tests (unit + integration)" + @echo " make node-run Run Node.js benchmark (requires DRIVER and WORKLOAD)" + @echo " make node-clean Clean Node.js build artifacts" + @echo " make node-info Show supported Node.js drivers and commands" + @echo "" @echo "Config Editor:" @echo " make config-editor-build Build config editor UI" @echo " make config-editor-dev Run config editor in development mode" @@ -387,6 +396,41 @@ csharp-clean: csharp-info: csharp-build dotnet run --project $(CSHARP_PROJECT) -c Release -- --info +# ============================================================================ +# Node.js Engine +# ============================================================================ + +# tsc emits into node/dist mirroring the source tree, so src/cli.ts -> dist/src/cli.js +NODE_CLI=node/dist/src/cli.js + +node-build: + cd node && npm ci && npm run build + +node-test: node-unit-test node-integration-test + +node-unit-test: node-build + cd node && node --test dist/test/unit/ + +# The live-server tests skip themselves unless VALKEY_HOST is set, so the server +# has to be up before this runs. +node-integration-test: node-build server-standalone-start + sleep 1 + cd node && VALKEY_HOST=localhost VALKEY_PORT=6379 node --test dist/test/integration/ + $(MAKE) server-standalone-stop + +node-run: node-build + node $(NODE_CLI) \ + --server $(SERVER) \ + --driver $(DRIVER) \ + --workload $(WORKLOAD) \ + --metrics $(METRICS_OUTPUT) + +node-info: node-build + node $(NODE_CLI) --info + +node-clean: + cd node && rm -rf dist node_modules coverage + # ============================================================================ # Config Editor # ============================================================================ @@ -447,8 +491,8 @@ test-scripts-all: java-build # All Languages # ============================================================================ -build-all: java-build ruby-build csharp-build python-build +build-all: java-build ruby-build csharp-build node-build python-build -test-all: java-test ruby-test csharp-test python-test +test-all: java-test ruby-test csharp-test node-test python-test -clean-all: java-clean ruby-clean csharp-clean python-clean clean +clean-all: java-clean ruby-clean csharp-clean node-clean python-clean clean diff --git a/README.md b/README.md index 5f55ed1..f14aa08 100644 --- a/README.md +++ b/README.md @@ -13,6 +13,7 @@ A multi-language benchmark suite for RESP protocol (Redis/Valkey) compatible dat ### Prerequisites - Python 3.8+, Java 21+, Maven +- Node.js 20+ (for the Node.js engine) - Make - A server CLI (`valkey-cli`) for the matrix runner's readiness probe and per-cell FLUSHALL — the Makefile's `server-*` targets build one into @@ -103,6 +104,7 @@ Thread-based system metrics collector that runs alongside benchmarks, collecting | Java | ✅ Ready | Jedis, Lettuce, Valkey-Glide, Redisson, Spring Data Valkey/Redis | | Ruby | ✅ Ready | redis-rb, valkey-glide-ruby | | C# | ✅ Ready | valkey-glide-csharp, StackExchange.Redis | +| Node.js | ✅ Ready | valkey-glide-node, ioredis, iovalkey | | Python | 🚧 Planned | redis-py, aioredis, valkey-glide | ## Project Structure @@ -133,6 +135,7 @@ resp-bench/ ├── java/ # Java benchmark engine ├── ruby/ # Ruby benchmark engine ├── csharp/ # C# (.NET 10) benchmark engine +├── node/ # Node.js (TypeScript) benchmark engine ├── docs/ │ ├── ARCHITECTURE.md # System architecture │ ├── BENCHMARK_MATRIX.md # Matrix orchestrator docs @@ -140,7 +143,8 @@ resp-bench/ │ ├── CONFIG_SPECIFICATION.md # Configuration format spec │ ├── BENCHMARKS_JAVA.md # Java benchmark details │ ├── BENCHMARKS_CSHARP.md # C# benchmark details -│ └── BENCHMARKS_RUBY.md # Ruby benchmark details +│ ├── BENCHMARKS_RUBY.md # Ruby benchmark details +│ └── BENCHMARKS_NODE.md # Node.js benchmark details └── graphs/interactive/ # Generated HTML graphs ``` @@ -197,6 +201,7 @@ See [docs/CONFIG_SPECIFICATION.md](docs/CONFIG_SPECIFICATION.md) for full detail | `make java-test` | Run Java unit tests | | `make ruby-test` | Run Ruby tests | | `make csharp-test` | Run C# tests | +| `make node-test` | Run Node.js tests (unit + integration) | ### Engines @@ -205,8 +210,10 @@ See [docs/CONFIG_SPECIFICATION.md](docs/CONFIG_SPECIFICATION.md) for full detail | `make java-run` | Run Java engine (DRIVER, WORKLOAD, SERVER) | | `make ruby-run` | Run Ruby engine (DRIVER, WORKLOAD, SERVER) | | `make csharp-run` | Run C# engine (DRIVER, WORKLOAD, SERVER) | +| `make node-run` | Run Node.js engine (DRIVER, WORKLOAD, SERVER) | | `make java-build` | Build Java JAR | | `make csharp-build` | Build C# executable | +| `make node-build` | Install deps and compile the Node.js engine | ### Server Management diff --git a/config-editor/src/App.tsx b/config-editor/src/App.tsx index 7b8e89f..492df38 100644 --- a/config-editor/src/App.tsx +++ b/config-editor/src/App.tsx @@ -35,7 +35,12 @@ interface CommandConfig { data_size_bytes?: number } -const DRIVERS = ['jedis', 'lettuce', 'valkey-glide', 'redisson', 'spring-data-valkey', 'spring-data-redis'] +const DRIVERS = [ + // Java + 'jedis', 'lettuce', 'valkey-glide', 'redisson', 'spring-data-valkey', 'spring-data-redis', + // Node.js + 'valkey-glide-node', 'ioredis', 'iovalkey', +] const COMMANDS = ['set', 'get', 'ping'] const ALGORITHMS = ['sequential_int', 'uniform_rand'] diff --git a/configs/drivers/default/ioredis.json b/configs/drivers/default/ioredis.json new file mode 100644 index 0000000..8261f9d --- /dev/null +++ b/configs/drivers/default/ioredis.json @@ -0,0 +1,7 @@ +{ + "schema_version": "1.0", + "description": "ioredis client - default configuration", + "driver_id": "ioredis", + "mode": "standalone", + "specific_driver_config": {} +} diff --git a/configs/drivers/default/iovalkey.json b/configs/drivers/default/iovalkey.json new file mode 100644 index 0000000..e4793ec --- /dev/null +++ b/configs/drivers/default/iovalkey.json @@ -0,0 +1,7 @@ +{ + "schema_version": "1.0", + "description": "iovalkey client - default configuration", + "driver_id": "iovalkey", + "mode": "standalone", + "specific_driver_config": {} +} diff --git a/configs/drivers/default/valkey-glide-node.json b/configs/drivers/default/valkey-glide-node.json new file mode 100644 index 0000000..bc25ae1 --- /dev/null +++ b/configs/drivers/default/valkey-glide-node.json @@ -0,0 +1,7 @@ +{ + "schema_version": "1.0", + "description": "Valkey GLIDE for Node.js client - default configuration", + "driver_id": "valkey-glide-node", + "mode": "standalone", + "specific_driver_config": {} +} diff --git a/configs/drivers/example-ioredis-standalone.json b/configs/drivers/example-ioredis-standalone.json new file mode 100644 index 0000000..9b594f5 --- /dev/null +++ b/configs/drivers/example-ioredis-standalone.json @@ -0,0 +1,7 @@ +{ + "schema_version": "1.0", + "description": "ioredis client, standalone mode", + "driver_id": "ioredis", + "mode": "standalone", + "specific_driver_config": {} +} diff --git a/configs/drivers/example-iovalkey-standalone.json b/configs/drivers/example-iovalkey-standalone.json new file mode 100644 index 0000000..978fa32 --- /dev/null +++ b/configs/drivers/example-iovalkey-standalone.json @@ -0,0 +1,7 @@ +{ + "schema_version": "1.0", + "description": "iovalkey client, standalone mode", + "driver_id": "iovalkey", + "mode": "standalone", + "specific_driver_config": {} +} diff --git a/configs/drivers/example-valkey-glide-node-standalone.json b/configs/drivers/example-valkey-glide-node-standalone.json new file mode 100644 index 0000000..5c5bb3d --- /dev/null +++ b/configs/drivers/example-valkey-glide-node-standalone.json @@ -0,0 +1,7 @@ +{ + "schema_version": "1.0", + "description": "Valkey GLIDE for Node.js client, standalone mode", + "driver_id": "valkey-glide-node", + "mode": "standalone", + "specific_driver_config": {} +} diff --git a/configs/drivers/high-throughput/ioredis.json b/configs/drivers/high-throughput/ioredis.json new file mode 100644 index 0000000..d214e79 --- /dev/null +++ b/configs/drivers/high-throughput/ioredis.json @@ -0,0 +1,8 @@ +{ + "schema_version": "1.0", + "description": "ioredis client - high-throughput configuration", + "driver_id": "ioredis", + "mode": "standalone", + "specific_driver_config": {}, + "command_timeout_ms": 10000 +} diff --git a/configs/drivers/high-throughput/iovalkey.json b/configs/drivers/high-throughput/iovalkey.json new file mode 100644 index 0000000..6a1d347 --- /dev/null +++ b/configs/drivers/high-throughput/iovalkey.json @@ -0,0 +1,8 @@ +{ + "schema_version": "1.0", + "description": "iovalkey client - high-throughput configuration", + "driver_id": "iovalkey", + "mode": "standalone", + "specific_driver_config": {}, + "command_timeout_ms": 10000 +} diff --git a/configs/drivers/high-throughput/valkey-glide-node.json b/configs/drivers/high-throughput/valkey-glide-node.json new file mode 100644 index 0000000..abc7a6a --- /dev/null +++ b/configs/drivers/high-throughput/valkey-glide-node.json @@ -0,0 +1,8 @@ +{ + "schema_version": "1.0", + "description": "Valkey GLIDE for Node.js client - high-throughput configuration", + "driver_id": "valkey-glide-node", + "mode": "standalone", + "specific_driver_config": {}, + "command_timeout_ms": 10000 +} diff --git a/configs/schemas/driver-config.schema.json b/configs/schemas/driver-config.schema.json index e06cf4e..c20cccc 100644 --- a/configs/schemas/driver-config.schema.json +++ b/configs/schemas/driver-config.schema.json @@ -18,7 +18,7 @@ "driver_id": { "type": "string", "description": "Identifier for the client driver implementation", - "examples": ["jedis", "lettuce", "valkey-glide", "redisson", "stackexchange-redis", "valkey-glide-csharp", "redis-py", "aioredis"] + "examples": ["jedis", "lettuce", "valkey-glide", "redisson", "stackexchange-redis", "valkey-glide-csharp", "valkey-glide-node", "ioredis", "iovalkey", "redis-py", "aioredis"] }, "mode": { "type": "string", diff --git a/docs/ADDING_LANGUAGE.md b/docs/ADDING_LANGUAGE.md index c8adab4..5ddb8c8 100644 --- a/docs/ADDING_LANGUAGE.md +++ b/docs/ADDING_LANGUAGE.md @@ -7,7 +7,10 @@ This guide explains how to add support for a new programming language to resp-be Before adding a new language, ensure you understand: - [Architecture](ARCHITECTURE.md) - Overall system design - [Configuration Specification](CONFIG_SPECIFICATION.md) - Config format details -- Existing implementations (Java is the reference implementation) +- Existing implementations: **Java is the reference implementation**; Ruby, C# and + Node.js follow it. For an async/event-loop language, `node/` is the closest model + and its README documents the parity traps worth knowing up front (RNG width, HDR + encoding, percentile-vs-raw min/max, shared request budget). ## Step-by-Step Guide @@ -178,9 +181,10 @@ class MetricsCollector: def record(self, command: str, latency_us: int, success: bool) -> None: if command not in self.command_metrics: - # 1µs to 1 hour, 3 significant figures + # 1µs to 600s, 3 significant figures (must match the other engines: + # Java/C#/Ruby/Node all use a max of 600_000_000µs, not 1 hour) self.command_metrics[command] = CommandMetrics( - histogram=HdrHistogram(1, 3600000000, 3) + histogram=HdrHistogram(1, 600000000, 3) ) metrics = self.command_metrics[command] @@ -389,9 +393,28 @@ Before submitting a new language engine: - [ ] Config parsing handles all schema fields - [ ] Key generator produces identical sequences (test with seed=12345) +- [ ] Keys are zero-padded to `key_size_bytes` (Java: `"%0" + max(1, key_size_bytes - prefix.length) + "d"`) +- [ ] The `uniform_rand` PRNG matches `java.util.Random` **including** the int32 + overflow check in `nextInt`'s rejection branch. Watch the arithmetic width: + the 48-bit LCG multiply exceeds what a double-based number type holds exactly + (this bites JavaScript, where `BigInt` is required) +- [ ] `sequential_int` uses a counter **shared** across all workers; `uniform_rand` + uses a per-worker PRNG seeded `base_seed + worker_index` +- [ ] The request budget is **shared** across workers and claimed one request at a + time, not pre-divided per worker (Java: one `AtomicLong` per phase) +- [ ] PING does **not** consume a generated key (Java's `PingCommand` ignores the + key generator, so consuming one shifts every subsequent key) - [ ] Rate limiter achieves target rates within 5% tolerance - [ ] Metrics output matches NDJSON schema exactly -- [ ] HdrHistogram produces compatible base64 payloads +- [ ] HdrHistogram produces compatible base64 payloads. If your library's encode + already returns base64, use it **directly** — encoding it again yields a + payload Java and Ruby cannot decode +- [ ] `summary.min`/`max` match Java's `getMinValue()`/`getMaxValue()`, which return + the *bucket's* equivalent bounds. Many ports expose a raw min/max property + instead; those diverge above ~1000µs at 3 significant figures. Prefer + `getValueAtPercentile(0)` / `getValueAtPercentile(100)` +- [ ] `warmup_requests`, `cps_limit`, `rps_limit`, `pipeline_depth`, + `command_timeout_ms`, `tls` and `auth` are all actually honoured, not just parsed - [ ] All unit tests pass - [ ] Integration tests pass against live server - [ ] Documentation complete diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md index 6463d7e..f1ac564 100644 --- a/docs/ARCHITECTURE.md +++ b/docs/ARCHITECTURE.md @@ -156,6 +156,18 @@ class BenchmarkClient(ABC): async def close(self): ... ``` +**Node.js (TypeScript):** +```typescript +interface BenchmarkClient { + connect(host: string, port: number, config: DriverConfig): Promise; + ping(): Promise>; + get(key: string): Promise>; + set(key: string, value: Buffer): Promise>; + close(): Promise; + driverVersion(): string; +} +``` + ## Metrics Output Format All engines produce identical NDJSON output: @@ -236,10 +248,15 @@ Different languages use appropriate concurrency primitives: | C# | Task-per-client with async/await (.NET 8+) | | Python | asyncio with async/await | | Go | goroutines and channels | -| Node.js | Promise/async-await | +| Node.js | One event loop, worker-per-connection (Promise/async-await) | The key requirement is that N connections can operate concurrently, each potentially with pipeline_depth in-flight requests. +**Node.js caveat:** the engine is single-threaded, so one CPU core bounds the whole +run. Past that point measurements reflect the engine rather than the client. See +[BENCHMARKS_NODE.md](BENCHMARKS_NODE.md) § "The Single-Core Ceiling" for the +measured plateau and how to tell when you have hit it. + ## Parallel Command Issuers (Java) At high connection counts (128+), a single command-issuing thread becomes a CPU bottleneck — saturating one core on semaphore contention, key generation (`String.format()`), and round-robin scanning. To address this, the Java engine supports **parallel command issuer threads** that partition client connections across multiple threads. diff --git a/docs/BENCHMARKS_NODE.md b/docs/BENCHMARKS_NODE.md new file mode 100644 index 0000000..00e2a4b --- /dev/null +++ b/docs/BENCHMARKS_NODE.md @@ -0,0 +1,147 @@ +# Node.js Benchmarks + +Details specific to the Node.js engine (`node/`). See [ARCHITECTURE.md](ARCHITECTURE.md) +for the cross-engine design and [../node/README.md](../node/README.md) for build and +usage. + +## Drivers + +| `driver_id` | Package | Notes | +|---|---|---| +| `valkey-glide-node` | `@valkey/valkey-glide` | Valkey GLIDE for Node.js. Prebuilt native binaries per platform. | +| `ioredis` | `ioredis` | The most widely used Node.js Redis client. | +| `iovalkey` | `iovalkey` | The Valkey-maintained fork of ioredis; API-identical. | +| `recording` | — | Synthetic-latency client for server-free testing. | + +The GLIDE id is `valkey-glide-node`, **not** `valkey-glide` — the latter belongs to +the Java engine in the global `DRIVER_ENGINE_MAP`. + +## Concurrency Model + +One event loop, one client per connection, one worker per connection. An awaited +command parks its worker, not the loop, so other connections keep progressing — +the analogue of Java's virtual-thread-per-client design. + +`pipeline_depth > 1` gives each connection that many independent +issue/await/record slots, so a settled request is replaced immediately rather than +waiting for a batch. + +The per-phase request budget is shared across workers and claimed one request at a +time (matching Java's `AtomicLong`), so a slow connection cannot cap the run. + +## The Single-Core Ceiling + +**Node runs the whole engine on one thread, so one CPU core is the hard ceiling.** +This is the single most important caveat when comparing Node numbers to Java or C#, +which spread issuing across threads. + +Measured locally (Apple M-series laptop, Valkey 8 co-located on the same host, +100% GET, 512B values, 10k keys, 5s phases). **Absolute numbers here are not +publishable results** — client and server contend for the same cores. The *shape* +is the point: + +Throughput vs connections (`pipeline_depth=1`): + +| Connections | glide RPS | ioredis RPS | iovalkey RPS | +|---|---|---|---| +| 1 | 1,139 | 1,584 | 1,646 | +| 10 | 8,424 | 8,467 | 10,360 | +| 50 | 25,065 | 23,071 | 26,268 | +| 100 | 33,639 | 34,898 | 35,087 | +| 200 | 41,812 | 42,748 | 41,144 | + +Doubling 100 → 200 connections buys only ~20% more throughput while p50 latency +roughly doubles (2.6ms → 4.5ms). That plateau is the event loop saturating, not the +clients. + +Throughput vs `pipeline_depth` at 10 connections (ioredis), with process CPU: + +| `pipeline_depth` | RPS | p50 | p99 | CPU (of one core) | +|---|---|---|---|---| +| 1 | 10,124 | 929µs | 2,055µs | 29% | +| 4 | 32,194 | 1,157µs | 2,731µs | 52% | +| 16 | 72,631 | 1,955µs | 4,089µs | 99% | + +CPU rises in lockstep with throughput and pins at 99% of a single core, where +throughput stops scaling. **The engine, not the client, is the limit past that +point.** + +Practical guidance: + +- Prefer raising `pipeline_depth` over raising `connections` to reach high + throughput on Node — it is far cheaper per unit of RPS. +- When comparing Node against Java/C#, check whether the Node process is CPU-bound. + If it is at ~100% of a core, you are measuring the engine, not the driver. +- Java addresses the same ceiling with parallel command-issuer threads (see + [ARCHITECTURE.md](ARCHITECTURE.md) § "Parallel Command Issuers"). The Node + equivalent would be `worker_threads` with a client partition per worker. That is + deliberately **not** implemented — it is a follow-up, to be justified by + measurements rather than assumed. + +## Fairness Controls + +Node-specific hazards with no analogue in the other engines, each handled +explicitly: + +- **Auto-pipelining is forced off** (`enableAutoPipelining: false`). ioredis and + iovalkey can transparently batch commands issued in the same event-loop tick, + which would inflate throughput against every other engine while looking like a + driver win. +- **Reconnects are disabled** (`retryStrategy: () => null`). ioredis' default + retries forever, so a wrong host would hang a run rather than fail it, and a + mid-phase reconnect would fold connection setup into request latency. +- **Response decoding is uniform.** `get` returns a `string` in all three drivers, + so none is charged for a different amount of decoding. GLIDE returns strings by + default; we do not opt one driver into bytes. +- **SET payloads are allocated once** per command object, so GC churn from payload + construction is not attributed to the driver. +- **Sub-millisecond rate limits work.** `setTimeout` clamps to ~1ms, so the limiter + yields via `setImmediate` below that. A 100k rps limit is a 10µs interval; a + timer-based wait would undershoot by ~100×. +- **Memory is not comparable to the JVM.** The system monitor's RSS samples include + V8 heap growth, which grows and collects on a different schedule from the JVM's. + +## Cross-Engine Parity + +Verified against the Java reference rather than assumed: + +- **Key sequences are byte-identical to Java.** 79,000 keys diffed against Java's + real `KeyGenerator` across `sequential_int` and `uniform_rand`, 1–16 workers, + prime `keys_count`, and prefix-width edge cases. `javaRandom.ts` ports + `java.util.Random` with `BigInt` (the 48-bit LCG multiply reaches ~2^83, past + what a JS `number` holds exactly) and is anchored to + `new Random(0).nextInt() == -1155484576`. +- **HDR payloads decode in Java.** `payload_b64` is + `encodeIntoCompressedBase64()` used directly — it is already base64, so encoding + it again would produce something Java cannot read. Verified by decoding a + Node-produced payload with `org.HdrHistogram.Histogram`: identical count and + percentiles. Range `(1, 600_000_000, 3)`, as in every engine. +- **`summary.min`/`max` match Java's quantization.** Java's `getMinValue()`/ + `getMaxValue()` return the bucket's equivalent bounds, while hdr-histogram-js' + `minNonZeroValue`/`maxValue` return the raw sample — they diverge above ~1000µs + (50000µs recorded reads back as 50015 in Java). The engine uses + `getValueAtPercentile(0)` and `getValueAtPercentile(100)`, which match Java exactly. +- **PING does not consume a key**, matching Java's `PingCommand`, so mixing PING + into a workload does not shift the key sequence other engines would produce. + +## Reproducing the Numbers Above + +```bash +make server-standalone-start + +# Populate the keyspace, then sweep connections. +make node-run \ + DRIVER=configs/drivers/default/ioredis.json \ + WORKLOAD=configs/workloads/reference/basic-standalone-single-client-1M-reqs.json \ + METRICS_OUTPUT=output/node-ioredis.ndjson + +make server-standalone-stop +``` + +For a full matrix across drivers and connection counts, use the orchestrator — +Node drivers are registered in `DRIVER_ENGINE_MAP`, so it dispatches to +`make node-run` automatically: + +```bash +python scripts/run_benchmark_matrix.py --matrix configs/matrices/.json +``` diff --git a/docs/CONFIG_SPECIFICATION.md b/docs/CONFIG_SPECIFICATION.md index 6a4c0b7..153ea73 100644 --- a/docs/CONFIG_SPECIFICATION.md +++ b/docs/CONFIG_SPECIFICATION.md @@ -54,6 +54,11 @@ resp-bench uses two JSON configuration files: - `stackexchange-redis` - StackExchange.Redis client - `valkey-glide-csharp` - Valkey GLIDE C# client +**Node.js:** +- `valkey-glide-node` - Valkey GLIDE Node.js client (not `valkey-glide`, which is Java's) +- `ioredis` - ioredis client +- `iovalkey` - iovalkey client (the Valkey-maintained ioredis fork) + **Python (planned):** - `redis-py` - redis-py synchronous client - `redis-py-async` - redis-py async client diff --git a/infra/provision.sh b/infra/provision.sh index 1c3699f..1de54b7 100755 --- a/infra/provision.sh +++ b/infra/provision.sh @@ -213,6 +213,35 @@ env_add "export DOTNET_CLI_TELEMETRY_OPTOUT=1" env_add "export DOTNET_NOLOGO=1" "${DOTNET_ROOT}/dotnet" --info 2>&1 | head -n3 | sed 's/^/[provision] /' || true +# ═════════════════════════════════════════════════════════════════════════════ +# LANGUAGE: Node.js (20+) — engines: valkey-glide-node, ioredis, iovalkey. +# node/package.json declares "engines": {"node": ">=20"}; distro feeds often +# ship 18 (EOL) or older, so install from NodeSource, which is version-pinned +# and works on both dnf and apt. valkey-glide ships prebuilt native binaries +# per platform, so no compiler is needed beyond build-essential (already +# installed above for the server build). +# ═════════════════════════════════════════════════════════════════════════════ + +NODE_MAJOR="${NODE_MAJOR:-22}" +log "LANGUAGE: Node.js — installing Node ${NODE_MAJOR} + npm" +node_major_installed() { + command -v node >/dev/null 2>&1 && + [ "$(node -p 'process.versions.node.split(".")[0]' 2>/dev/null || echo 0)" -ge 20 ] +} +if ! node_major_installed; then + if [ "${PKG}" = "dnf" ]; then + curl -fsSL "https://rpm.nodesource.com/setup_${NODE_MAJOR}.x" | ${SUDO} bash - + pkg_install nodejs -- nodejs + else + curl -fsSL "https://deb.nodesource.com/setup_${NODE_MAJOR}.x" | ${SUDO} bash - + pkg_install nodejs -- nodejs + fi +else + log "Node.js $(node -v) already present (>= 20)" +fi +node -v 2>&1 | sed 's/^/[provision] /' || true +npm -v 2>&1 | sed 's/^/[provision] /' || true + # ═════════════════════════════════════════════════════════════════════════════ # LANGUAGE: Python (3.9+ + pip) — used by the matrix orchestrator and graph # generator (scripts/*.py). Engine deps are installed from the repo's pinned @@ -241,7 +270,6 @@ python --version 2>&1 | sed 's/^/[provision] /' || true # ───────────────────────────────────────────────────────────────────────────── # FUTURE ENGINES (leave room — see resp-bench plan §5.5): -# Node.js (#13): pkg_install nodejs npm -- nodejs npm + `npm ci` warm-up # Go (#14): install the Go toolchain + `go mod download` # PHP (#15): pkg_install php php-cli composer -- php-cli composer # Add each as its own "LANGUAGE:" block above, mirroring the pattern. @@ -295,6 +323,8 @@ if [ "${SKIP_WARM_CACHES:-0}" != "1" ]; then make -C "${REPO_DIR}" ruby-build || log "WARNING: Ruby warm-up build failed (see above)" log "warming C# build cache (dotnet build)" make -C "${REPO_DIR}" csharp-build || log "WARNING: C# warm-up build failed (see above)" + log "warming Node.js deps (npm ci + tsc)" + make -C "${REPO_DIR}" node-build || log "WARNING: Node.js warm-up build failed (see above)" else log "SKIP_WARM_CACHES=1 — skipping engine cache warm-up" fi diff --git a/node/.gitignore b/node/.gitignore new file mode 100644 index 0000000..c19bb02 --- /dev/null +++ b/node/.gitignore @@ -0,0 +1,4 @@ +node_modules/ +dist/ +coverage/ +*.tsbuildinfo diff --git a/node/README.md b/node/README.md new file mode 100644 index 0000000..af7834f --- /dev/null +++ b/node/README.md @@ -0,0 +1,141 @@ +# resp-bench Node.js Engine + +Node.js implementation of the resp-bench benchmark suite, written in TypeScript. + +## Supported Drivers + +| `driver_id` | Package | Notes | +|---|---|---| +| `valkey-glide-node` | [`@valkey/valkey-glide`](https://www.npmjs.com/package/@valkey/valkey-glide) | Valkey GLIDE for Node.js. Ships prebuilt native binaries per platform. | +| `ioredis` | [`ioredis`](https://www.npmjs.com/package/ioredis) | The most widely used Node.js Redis client. | +| `iovalkey` | [`iovalkey`](https://www.npmjs.com/package/iovalkey) | The Valkey-maintained fork of ioredis; API-identical. | +| `recording` | — | In-memory synthetic-latency client for server-free tests. | + +> **Note:** the GLIDE driver id is `valkey-glide-node`, not `valkey-glide`. The +> latter is already the Java engine's id in the global `DRIVER_ENGINE_MAP` in +> `scripts/run_benchmark_matrix.py`, and reusing it would reroute Java's runs here. + +## Requirements + +- Node.js **20 or newer** (`package.json` declares `engines.node >= 20`). CI uses 22 LTS. + +## Build + +```bash +npm ci && npm run build # or: make node-build +``` + +TypeScript compiles to `dist/`, mirroring the source tree, so the entry point is +`dist/src/cli.js`. + +## Usage + +```bash +node dist/src/cli.js \ + --server localhost:6379 \ + --driver ../configs/drivers/default/ioredis.json \ + --workload ../configs/workloads/reference/basic-standalone-single-client-1M-reqs.json \ + --metrics ../output/node.ndjson +``` + +Or through the Makefile, from the repository root: + +```bash +make node-run \ + DRIVER=configs/drivers/default/valkey-glide-node.json \ + WORKLOAD=configs/workloads/reference/basic-standalone-single-client-1M-reqs.json \ + METRICS_OUTPUT=output/node.ndjson + +make node-info # list supported drivers and commands +``` + +## Tests + +```bash +make node-test # unit + integration (starts/stops a server) +cd node && npm run test:unit # unit only, no server needed +``` + +The live-server tests skip themselves unless `VALKEY_HOST` is set: + +```bash +cd node && VALKEY_HOST=localhost VALKEY_PORT=6379 npm run test:integration +``` + +The `recording` driver lets the full engine — phases, warmup, budget, pipelining, +rate limiting, NDJSON — be exercised with no server at all. + +## Concurrency Model + +A single event loop with **one client per connection** and **one worker per +connection**, all started together. Node is single-threaded with async I/O, so an +awaited command parks that worker rather than the loop, and the other connections +keep making progress. This is the analogue of Java's +virtual-thread-per-client design. + +`pipeline_depth > 1` is supported: each connection runs that many independent +issue/await/record slots, so a settled request is replaced immediately. + +The per-phase request budget is **shared across all workers** and claimed one +request at a time, matching the Java reference's `AtomicLong`. It is deliberately +not pre-divided per worker — a shared budget lets fast connections absorb a slow +one's slack, so wall-clock is not bounded by the slowest connection. + +## Cross-Engine Parity + +Verified against the Java reference engine, not assumed: + +- **Key sequences are byte-identical.** 79,000 keys were diffed against Java's + actual `KeyGenerator` across both algorithms, 1–16 workers, prime `keys_count`, + and tight padding. `src/engine/javaRandom.ts` is a `java.util.Random` port + anchored to `new Random(0).nextInt() == -1155484576`; it uses `BigInt` because + the 48-bit LCG multiply overflows a JS `number`. +- **HDR payloads are mutually decodable.** `payload_b64` is + `encodeIntoCompressedBase64()` used directly (it is *already* base64 — encoding + it again would produce something Java cannot read). Java's + `Histogram.decodeFromCompressedByteBuffer` reads our payloads with matching + count and percentiles. Range is `(1, 600_000_000, 3)`, as in every engine. +- **`summary.min`/`max` use `getValueAtPercentile(0/100)`**, not + `minNonZeroValue`/`maxValue`. Java reports the bucket's equivalent bounds, and + hdr-histogram-js' properties return the raw sample — they diverge above ~1000µs + (recording 50000µs yields 50000 in JS but 50015 in Java). +- **PING does not consume a key**, matching Java's `PingCommand`, so mixing PING + into a workload does not shift the key sequence. + +## Node-Specific Fairness Notes + +Read these before comparing Node numbers to another engine: + +- **Auto-pipelining is explicitly disabled.** ioredis and iovalkey can + transparently batch commands issued in the same event-loop tick, which would + inflate throughput against every other engine. `enableAutoPipelining: false`. +- **Reconnects are disabled** (`retryStrategy: () => null`). ioredis otherwise + retries forever, so a wrong host would hang a run instead of failing it, and a + mid-phase reconnect would fold connection setup into request latency. +- **All drivers decode responses the same way.** `get` returns a `string` in all + three, so none is charged for a different amount of decoding. +- **SET payloads are built once** per command object, not per request, so the + driver is not charged for the engine's own allocation churn. +- **Sub-millisecond rate limits work.** `setTimeout` clamps to ~1 ms, so the + limiter yields via `setImmediate` for shorter intervals; a 100k rps limit is a + 10 µs interval and a timer-based wait would undershoot it by ~100×. +- **RSS is not comparable to the JVM's.** The system monitor's memory samples + include V8 heap growth and GC timing, which behave differently from the JVM's. +- **One event loop is one core.** At high connection counts the engine itself, not + the client, may become the ceiling. See `docs/BENCHMARKS_NODE.md`. + +## Layout + +``` +node/ +├── src/ +│ ├── cli.ts # arg parsing, --info, error boundary +│ ├── client/ # driver interface, factory, per-driver impls +│ ├── command/ # GET / SET / PING +│ ├── config/ # JSON config parsing + validation +│ ├── engine/ # benchmark loops, key gen, RNG, rate limiter +│ └── metrics/ # HDR histogram, collector, NDJSON writer +└── test/ + ├── unit/ # no server, no optional deps + └── integration/ # recording driver + live-server tests +``` diff --git a/node/package-lock.json b/node/package-lock.json new file mode 100644 index 0000000..068c475 --- /dev/null +++ b/node/package-lock.json @@ -0,0 +1,420 @@ +{ + "name": "resp-bench-node", + "version": "1.0.0", + "lockfileVersion": 3, + "requires": true, + "packages": { + "": { + "name": "resp-bench-node", + "version": "1.0.0", + "license": "Apache-2.0", + "dependencies": { + "@valkey/valkey-glide": "^2.5.2", + "hdr-histogram-js": "^3.0.1", + "ioredis": "^5.11.1", + "iovalkey": "^0.4.0" + }, + "devDependencies": { + "@types/node": "^20.19.0", + "typescript": "^5.9.2" + }, + "engines": { + "node": ">=20" + } + }, + "node_modules/@assemblyscript/loader": { + "version": "0.19.23", + "resolved": "https://registry.npmjs.org/@assemblyscript/loader/-/loader-0.19.23.tgz", + "integrity": "sha512-ulkCYfFbYj01ie1MDOyxv2F6SpRN1TOj7fQxbP07D6HmeR+gr2JLSmINKjga2emB+b1L2KGrFKBTc+e00p54nw==", + "license": "Apache-2.0" + }, + "node_modules/@ioredis/commands": { + "version": "1.10.0", + "resolved": "https://registry.npmjs.org/@ioredis/commands/-/commands-1.10.0.tgz", + "integrity": "sha512-UmeW7z4LfctwoQ5wkhVzgq8tXkreED2xZGpX+Bg+zA+WJFZCT6c062AfCK/Dfk81xZnnwdhJCUMkitihRaoC2Q==", + "license": "MIT" + }, + "node_modules/@iovalkey/commands": { + "version": "0.1.0", + "resolved": "https://registry.npmjs.org/@iovalkey/commands/-/commands-0.1.0.tgz", + "integrity": "sha512-/B9W4qKSSITDii5nkBCHyPkIkAi+ealUtr1oqBJsLxjSRLka4pxun2VvMNSmcwgAMxgXtQfl0qRv7TE+udPJzg==", + "license": "MIT" + }, + "node_modules/@protobufjs/aspromise": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/@protobufjs/aspromise/-/aspromise-1.1.2.tgz", + "integrity": "sha512-j+gKExEuLmKwvz3OgROXtrJ2UG2x8Ch2YZUxahh+s1F2HZ+wAceUNLkvy6zKCPVRkU++ZWQrdxsUeQXmcg4uoQ==", + "license": "BSD-3-Clause" + }, + "node_modules/@protobufjs/base64": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/@protobufjs/base64/-/base64-1.1.2.tgz", + "integrity": "sha512-AZkcAA5vnN/v4PDqKyMR5lx7hZttPDgClv83E//FMNhR2TMcLUhfRUBHCmSl0oi9zMgDDqRUJkSxO3wm85+XLg==", + "license": "BSD-3-Clause" + }, + "node_modules/@protobufjs/codegen": { + "version": "2.0.5", + "resolved": "https://registry.npmjs.org/@protobufjs/codegen/-/codegen-2.0.5.tgz", + "integrity": "sha512-zgXFLzW3Ap33e6d0Wlj4MGIm6Ce8O89n/apUaGNB/jx+hw+ruWEp7EwGUshdLKVRCxZW12fp9r40E1mQrf/34g==", + "license": "BSD-3-Clause" + }, + "node_modules/@protobufjs/eventemitter": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/@protobufjs/eventemitter/-/eventemitter-1.1.1.tgz", + "integrity": "sha512-vW1GmwMZNnL+gMRaovlh9yZX74kc+TTU3FObkkurpMaRtBfLP3ldjS9KQWlwZgraRE0+dheEEoAxdzcJQ8eXZg==", + "license": "BSD-3-Clause" + }, + "node_modules/@protobufjs/fetch": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/@protobufjs/fetch/-/fetch-1.1.1.tgz", + "integrity": "sha512-GpptLrs57adMSuHi3VNj0mAF8dwh36LMaYF6XyJ6JMWlVsc+t42tm1HSEDmOs3A8fC9yyeisgLhsTVQokOZ0zw==", + "license": "BSD-3-Clause", + "dependencies": { + "@protobufjs/aspromise": "^1.1.1" + } + }, + "node_modules/@protobufjs/float": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/@protobufjs/float/-/float-1.0.2.tgz", + "integrity": "sha512-Ddb+kVXlXst9d+R9PfTIxh1EdNkgoRe5tOX6t01f1lYWOvJnSPDBlG241QLzcyPdoNTsblLUdujGSE4RzrTZGQ==", + "license": "BSD-3-Clause" + }, + "node_modules/@protobufjs/path": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/@protobufjs/path/-/path-1.1.2.tgz", + "integrity": "sha512-6JOcJ5Tm08dOHAbdR3GrvP+yUUfkjG5ePsHYczMFLq3ZmMkAD98cDgcT2iA1lJ9NVwFd4tH/iSSoe44YWkltEA==", + "license": "BSD-3-Clause" + }, + "node_modules/@protobufjs/pool": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/@protobufjs/pool/-/pool-1.1.0.tgz", + "integrity": "sha512-0kELaGSIDBKvcgS4zkjz1PeddatrjYcmMWOlAuAPwAeccUrPHdUqo/J6LiymHHEiJT5NrF1UVwxY14f+fy4WQw==", + "license": "BSD-3-Clause" + }, + "node_modules/@protobufjs/utf8": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/@protobufjs/utf8/-/utf8-1.1.2.tgz", + "integrity": "sha512-b1UQwcEZ4yCnMCD8DAL1VlbvBJE9/IX4FTIp7BG1xYpf29SLazLSrqUkj4w7Y5y7cCVP6E5tcqqcI0xemPkHug==", + "license": "BSD-3-Clause" + }, + "node_modules/@types/node": { + "version": "20.19.43", + "resolved": "https://registry.npmjs.org/@types/node/-/node-20.19.43.tgz", + "integrity": "sha512-6oYBAi5ikg4Pl+kGsoYtawUMBT2zZMCvPNF7pVLnHZfd1zf38DRiWn/gT01RYCdUqkv7Fhr+C9ot4/tb+2sVvA==", + "license": "MIT", + "dependencies": { + "undici-types": "~6.21.0" + } + }, + "node_modules/@valkey/valkey-glide": { + "version": "2.5.2", + "resolved": "https://registry.npmjs.org/@valkey/valkey-glide/-/valkey-glide-2.5.2.tgz", + "integrity": "sha512-8d3m2NyveQ2Ws4OUd7Zz3zsPP835Nqi4Rm3ZMoLdUjTfnEJejIMPJGR8b09foiPEG4GXV9j+QWyDRuM9VHOuVw==", + "license": "Apache-2.0", + "dependencies": { + "long": "5", + "protobufjs": "^7.6.3" + }, + "engines": { + "node": ">=16" + }, + "optionalDependencies": { + "@valkey/valkey-glide-darwin-arm64": "2.5.2", + "@valkey/valkey-glide-darwin-x64": "2.5.2", + "@valkey/valkey-glide-linux-arm64-gnu": "2.5.2", + "@valkey/valkey-glide-linux-arm64-musl": "2.5.2", + "@valkey/valkey-glide-linux-x64-gnu": "2.5.2", + "@valkey/valkey-glide-linux-x64-musl": "2.5.2" + } + }, + "node_modules/@valkey/valkey-glide-darwin-arm64": { + "version": "2.5.2", + "resolved": "https://registry.npmjs.org/@valkey/valkey-glide-darwin-arm64/-/valkey-glide-darwin-arm64-2.5.2.tgz", + "integrity": "sha512-6kmFsyBrMj8kRF72ZiKpLsT0WZrRQdM3P2p9vJskRaXbRiMGmLVoqj8vv1tYcu/xRr7bI6aEONOlwKbHojRLkQ==", + "cpu": [ + "arm64" + ], + "license": "Apache-2.0", + "optional": true, + "os": [ + "darwin" + ] + }, + "node_modules/@valkey/valkey-glide-darwin-x64": { + "version": "2.5.2", + "resolved": "https://registry.npmjs.org/@valkey/valkey-glide-darwin-x64/-/valkey-glide-darwin-x64-2.5.2.tgz", + "integrity": "sha512-2RgllAJlYvoKirMKDIseYfX7LNFOF06UohqRB4LM2U3Y9mjE/ssNrPy+netFYdqz0aWcgYajzCMr2jXeckmoEg==", + "cpu": [ + "x64" + ], + "license": "Apache-2.0", + "optional": true, + "os": [ + "darwin" + ] + }, + "node_modules/@valkey/valkey-glide-linux-arm64-gnu": { + "version": "2.5.2", + "resolved": "https://registry.npmjs.org/@valkey/valkey-glide-linux-arm64-gnu/-/valkey-glide-linux-arm64-gnu-2.5.2.tgz", + "integrity": "sha512-EVWT7VT9porB670AxjQ+b/g/Dx5Nm/GaMy9irCzdDdnnTOwPyHvK1kPFzMATNuyDgXd5L504i99u8iHL1GpDEw==", + "cpu": [ + "arm64" + ], + "license": "Apache-2.0", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@valkey/valkey-glide-linux-arm64-musl": { + "version": "2.5.2", + "resolved": "https://registry.npmjs.org/@valkey/valkey-glide-linux-arm64-musl/-/valkey-glide-linux-arm64-musl-2.5.2.tgz", + "integrity": "sha512-CFTrYPWIrh+k66eqH3/FvUlMZ4S7p328iFTK+qsHxyYH3CnhAmKMaOHcptFKz5n7ypT8L/XuOBl+NIQgUrJGQA==", + "cpu": [ + "arm64" + ], + "license": "Apache-2.0", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@valkey/valkey-glide-linux-x64-gnu": { + "version": "2.5.2", + "resolved": "https://registry.npmjs.org/@valkey/valkey-glide-linux-x64-gnu/-/valkey-glide-linux-x64-gnu-2.5.2.tgz", + "integrity": "sha512-osWsmJ+cg2iZjjNZ5IHQUMg/Xb/TQVyBvDxKA11vpLuEFMulQYfUfw+LWo0PwnG2mGJGKHhDnZ8KZAJpd03LWQ==", + "cpu": [ + "x64" + ], + "license": "Apache-2.0", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@valkey/valkey-glide-linux-x64-musl": { + "version": "2.5.2", + "resolved": "https://registry.npmjs.org/@valkey/valkey-glide-linux-x64-musl/-/valkey-glide-linux-x64-musl-2.5.2.tgz", + "integrity": "sha512-gJCma4s3j3AUQk3k2Kcxe3ImWpla4e9jr41f1w9VwZd6CkWuZQdk1PsLpdROvtaBxUiRiq+RZvvLKvGf0md1Kg==", + "cpu": [ + "x64" + ], + "license": "Apache-2.0", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/base64-js": { + "version": "1.5.1", + "resolved": "https://registry.npmjs.org/base64-js/-/base64-js-1.5.1.tgz", + "integrity": "sha512-AKpaYlHn8t4SVbOHCy+b5+KKgvR4vrsD8vbvrbiQJps7fKDTkjkDry6ji0rUJjC0kzbNePLwzxq8iypo41qeWA==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ], + "license": "MIT" + }, + "node_modules/cluster-key-slot": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/cluster-key-slot/-/cluster-key-slot-1.1.1.tgz", + "integrity": "sha512-rwHwUfXL40Chm1r08yrhU3qpUvdVlgkKNeyeGPOxnW8/SyVDvgRaed/Uz54AqWNaTCAThlj6QAs3TZcKI0xDEw==", + "license": "Apache-2.0", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/debug": { + "version": "4.4.3", + "resolved": "https://registry.npmjs.org/debug/-/debug-4.4.3.tgz", + "integrity": "sha512-RGwwWnwQvkVfavKVt22FGLw+xYSdzARwm0ru6DhTVA3umU5hZc28V3kO4stgYryrTlLpuvgI9GiijltAjNbcqA==", + "license": "MIT", + "dependencies": { + "ms": "^2.1.3" + }, + "engines": { + "node": ">=6.0" + }, + "peerDependenciesMeta": { + "supports-color": { + "optional": true + } + } + }, + "node_modules/denque": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/denque/-/denque-2.1.0.tgz", + "integrity": "sha512-HVQE3AAb/pxF8fQAoiqpvg9i3evqug3hoiwakOyZAwJm+6vZehbkYXZ0l4JxS+I3QxM97v5aaRNhj8v5oBhekw==", + "license": "Apache-2.0", + "engines": { + "node": ">=0.10" + } + }, + "node_modules/hdr-histogram-js": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/hdr-histogram-js/-/hdr-histogram-js-3.0.1.tgz", + "integrity": "sha512-l3GSdZL1Jr1C0kyb461tUjEdrRPZr8Qry7jByltf5JGrA0xvqOSrxRBfcrJqqV/AMEtqqhHhC6w8HW0gn76tRQ==", + "license": "BSD-2-Clause", + "dependencies": { + "@assemblyscript/loader": "^0.19.21", + "base64-js": "^1.2.0", + "pako": "^1.0.3" + }, + "engines": { + "node": ">=14" + } + }, + "node_modules/ioredis": { + "version": "5.11.1", + "resolved": "https://registry.npmjs.org/ioredis/-/ioredis-5.11.1.tgz", + "integrity": "sha512-ehuGcf94bQXhfagULNXrJdfnWO38v070jxSx/qE87Kjzmu2fU7ro5EFAb+OPituLqgfyuQaym5DlrNydW2sJ9A==", + "license": "MIT", + "dependencies": { + "@ioredis/commands": "1.10.0", + "cluster-key-slot": "1.1.1", + "debug": "4.4.3", + "denque": "2.1.0", + "redis-errors": "1.2.0", + "redis-parser": "3.0.0", + "standard-as-callback": "2.1.0" + }, + "engines": { + "node": ">=12.22.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/ioredis" + } + }, + "node_modules/iovalkey": { + "version": "0.4.0", + "resolved": "https://registry.npmjs.org/iovalkey/-/iovalkey-0.4.0.tgz", + "integrity": "sha512-OSUKxJ+s44CLdUTaicX4+pVrBN/zHKIYwr2oKhRDuczr19FwcSCXEZHzcMUYWZNh1mNSCV9bNJHW/T6cHVm9Aw==", + "license": "MIT", + "dependencies": { + "@iovalkey/commands": "^0.1.0", + "cluster-key-slot": "^1.1.0", + "debug": "^4.3.4", + "denque": "^2.1.0", + "lodash.defaults": "^4.2.0", + "lodash.isarguments": "^3.1.0", + "redis-errors": "^1.2.0", + "redis-parser": "^3.0.0", + "standard-as-callback": "^2.1.0" + }, + "engines": { + "node": ">=18.12.0" + } + }, + "node_modules/lodash.defaults": { + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/lodash.defaults/-/lodash.defaults-4.2.0.tgz", + "integrity": "sha512-qjxPLHd3r5DnsdGacqOMU6pb/avJzdh9tFX2ymgoZE27BmjXrNy/y4LoaiTeAb+O3gL8AfpJGtqfX/ae2leYYQ==", + "license": "MIT" + }, + "node_modules/lodash.isarguments": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/lodash.isarguments/-/lodash.isarguments-3.1.0.tgz", + "integrity": "sha512-chi4NHZlZqZD18a0imDHnZPrDeBbTtVN7GXMwuGdRH9qotxAjYs3aVLKc7zNOG9eddR5Ksd8rvFEBc9SsggPpg==", + "license": "MIT" + }, + "node_modules/long": { + "version": "5.3.2", + "resolved": "https://registry.npmjs.org/long/-/long-5.3.2.tgz", + "integrity": "sha512-mNAgZ1GmyNhD7AuqnTG3/VQ26o760+ZYBPKjPvugO8+nLbYfX6TVpJPseBvopbdY+qpZ/lKUnmEc1LeZYS3QAA==", + "license": "Apache-2.0" + }, + "node_modules/ms": { + "version": "2.1.3", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", + "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==", + "license": "MIT" + }, + "node_modules/pako": { + "version": "1.0.11", + "resolved": "https://registry.npmjs.org/pako/-/pako-1.0.11.tgz", + "integrity": "sha512-4hLB8Py4zZce5s4yd9XzopqwVv/yGNhV1Bl8NTmCq1763HeK2+EwVTv+leGeL13Dnh2wfbqowVPXCIO0z4taYw==", + "license": "(MIT AND Zlib)" + }, + "node_modules/protobufjs": { + "version": "7.6.6", + "resolved": "https://registry.npmjs.org/protobufjs/-/protobufjs-7.6.6.tgz", + "integrity": "sha512-dYDWdjSl5RNb7SgPxGQcRU+GtvP7s2fpkrY0r432PcOIaZ0/rBcxEZnQN67iJhFuQiVw754JDoPruPCNdGsbjg==", + "hasInstallScript": true, + "license": "BSD-3-Clause", + "dependencies": { + "@protobufjs/aspromise": "^1.1.2", + "@protobufjs/base64": "^1.1.2", + "@protobufjs/codegen": "^2.0.5", + "@protobufjs/eventemitter": "^1.1.1", + "@protobufjs/fetch": "^1.1.1", + "@protobufjs/float": "^1.0.2", + "@protobufjs/path": "^1.1.2", + "@protobufjs/pool": "^1.1.0", + "@protobufjs/utf8": "^1.1.1", + "@types/node": ">=13.7.0", + "long": "^5.3.2" + }, + "engines": { + "node": ">=12.0.0" + } + }, + "node_modules/redis-errors": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/redis-errors/-/redis-errors-1.2.0.tgz", + "integrity": "sha512-1qny3OExCf0UvUV/5wpYKf2YwPcOqXzkwKKSmKHiE6ZMQs5heeE/c8eXK+PNllPvmjgAbfnsbpkGZWy8cBpn9w==", + "license": "MIT", + "engines": { + "node": ">=4" + } + }, + "node_modules/redis-parser": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/redis-parser/-/redis-parser-3.0.0.tgz", + "integrity": "sha512-DJnGAeenTdpMEH6uAJRK/uiyEIH9WVsUmoLwzudwGJUwZPp80PDBWPHXSAGNPwNvIXAbe7MSUB1zQFugFml66A==", + "license": "MIT", + "dependencies": { + "redis-errors": "^1.0.0" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/standard-as-callback": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/standard-as-callback/-/standard-as-callback-2.1.0.tgz", + "integrity": "sha512-qoRRSyROncaz1z0mvYqIE4lCd9p2R90i6GxW3uZv5ucSu8tU7B5HXUP1gG8pVZsYNVaXjk8ClXHPttLyxAL48A==", + "license": "MIT" + }, + "node_modules/typescript": { + "version": "5.9.3", + "resolved": "https://registry.npmjs.org/typescript/-/typescript-5.9.3.tgz", + "integrity": "sha512-jl1vZzPDinLr9eUt3J/t7V6FgNEw9QjvBPdysz9KfQDD41fQrC2Y4vKQdiaUpFT4bXlb1RHhLpp8wtm6M5TgSw==", + "dev": true, + "license": "Apache-2.0", + "bin": { + "tsc": "bin/tsc", + "tsserver": "bin/tsserver" + }, + "engines": { + "node": ">=14.17" + } + }, + "node_modules/undici-types": { + "version": "6.21.0", + "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-6.21.0.tgz", + "integrity": "sha512-iwDZqg0QAGrg9Rav5H4n0M64c3mkR59cJ6wQp+7C4nI0gsmExaedaYLNO44eT4AtBBwjbTiGPMlt2Md0T9H9JQ==", + "license": "MIT" + } + } +} diff --git a/node/package.json b/node/package.json new file mode 100644 index 0000000..35e1f7c --- /dev/null +++ b/node/package.json @@ -0,0 +1,30 @@ +{ + "name": "resp-bench-node", + "version": "1.0.0", + "description": "resp-bench Node.js benchmark engine", + "license": "Apache-2.0", + "private": true, + "type": "module", + "engines": { + "node": ">=20" + }, + "scripts": { + "build": "tsc", + "clean": "rm -rf dist", + "typecheck": "tsc --noEmit", + "test": "npm run build && node --test dist/test/", + "test:unit": "npm run build && node --test dist/test/unit/", + "test:integration": "npm run build && node --test dist/test/integration/", + "start": "node dist/src/cli.js" + }, + "dependencies": { + "@valkey/valkey-glide": "^2.5.2", + "hdr-histogram-js": "^3.0.1", + "ioredis": "^5.11.1", + "iovalkey": "^0.4.0" + }, + "devDependencies": { + "@types/node": "^20.19.0", + "typescript": "^5.9.2" + } +} diff --git a/node/src/cli.ts b/node/src/cli.ts new file mode 100644 index 0000000..9c6fef3 --- /dev/null +++ b/node/src/cli.ts @@ -0,0 +1,142 @@ +/** + * Command-line interface for the resp-bench Node.js engine. + * + * Implements the shared cross-engine CLI contract: `--server`, `--driver`, + * `--workload`, `--metrics`, plus `--info`, `--commit-id` (used by CI), and + * `--version`. Deliberately no `--concurrency` flag: the engine has a single + * execution model, and pipelining comes from the workload's `pipeline_depth`. + */ + +import { existsSync } from 'node:fs'; +import { parseArgs } from 'node:util'; + +import { BenchmarkClientFactory } from './client/factory.js'; +import { CommandFactory } from './command/factory.js'; +import { ConfigLoader } from './config/loader.js'; +import { BenchmarkEngine } from './engine/benchmark.js'; +import { VERSION } from './version.js'; + +const DEFAULT_SERVER = 'localhost:6379'; +const DEFAULT_PORT = 6379; + +const USAGE = `resp-bench Node.js engine v${VERSION} + +Usage: + node dist/src/cli.js --driver --workload --metrics [options] + +Options: + --server Server address (default: ${DEFAULT_SERVER}) + --driver Driver configuration JSON (required) + --workload Workload configuration JSON (required) + --metrics Metrics NDJSON output path (required) + --commit-id Git commit ID recorded in the metrics metadata + --info Show supported drivers and commands + --version Show the engine version + --help Show this message +`; + +function parseServer(server: string): { host: string; port: number } { + const separator = server.lastIndexOf(':'); + if (separator === -1) return { host: server || 'localhost', port: DEFAULT_PORT }; + const host = server.slice(0, separator) || 'localhost'; + const port = Number(server.slice(separator + 1)); + if (!Number.isInteger(port) || port <= 0) { + throw new Error(`invalid --server "${server}": port must be a positive integer`); + } + return { host, port }; +} + +function printInfo(): void { + const lines = [ + `resp-bench Node.js Engine v${VERSION}`, + '', + 'Supported Drivers:', + ...BenchmarkClientFactory.describe().map( + ({ driverId, description }) => ` - ${driverId.padEnd(20)} : ${description}`, + ), + '', + 'Supported Commands:', + ...CommandFactory.describe().map(({ name, description }) => ` - ${name.padEnd(10)} : ${description}`), + '', + 'Supported Key Generation Algorithms:', + ' - sequential_int : Sequential integers (0 to keys_count), shared across connections', + ' - uniform_rand : Uniform random, java.util.Random-compatible per connection', + '', + 'Supported Completion Types:', + ' - duration : Run for the specified seconds', + ' - requests : Run until the shared request budget is exhausted', + '', + 'Concurrency: event-loop task-per-connection (one client per connection)', + ]; + console.log(lines.join('\n')); +} + +export async function main(argv: string[] = process.argv.slice(2)): Promise { + let options; + try { + ({ values: options } = parseArgs({ + args: argv, + options: { + server: { type: 'string', default: DEFAULT_SERVER }, + driver: { type: 'string' }, + workload: { type: 'string' }, + metrics: { type: 'string' }, + 'commit-id': { type: 'string' }, + info: { type: 'boolean', default: false }, + version: { type: 'boolean', default: false }, + help: { type: 'boolean', default: false }, + }, + strict: true, + })); + } catch (error) { + console.error(`Error: ${(error as Error).message}`); + console.error(USAGE); + return 1; + } + + if (options.help) { + console.log(USAGE); + return 0; + } + if (options.version) { + console.log(`resp-bench Node.js Engine v${VERSION}`); + return 0; + } + if (options.info) { + printInfo(); + return 0; + } + + try { + const missing = (['driver', 'workload', 'metrics'] as const).filter((flag) => !options[flag]); + if (missing.length > 0) { + throw new Error(`missing required options: ${missing.map((f) => `--${f}`).join(', ')}`); + } + for (const flag of ['driver', 'workload'] as const) { + if (!existsSync(options[flag]!)) { + throw new Error(`${flag} config not found: ${options[flag]}`); + } + } + + const { host, port } = parseServer(options.server!); + const engine = new BenchmarkEngine({ + host, + port, + driverConfig: ConfigLoader.loadDriverConfig(options.driver!), + workloadConfig: ConfigLoader.loadWorkloadConfig(options.workload!), + metricsPath: options.metrics!, + commitId: options['commit-id'] ?? null, + }); + await engine.run(); + return 0; + } catch (error) { + console.error(`Error: ${(error as Error).message}`); + if (process.env['DEBUG']) console.error((error as Error).stack); + return 1; + } +} + +// `import.meta.url` check keeps the module importable by tests without running. +if (process.argv[1] && import.meta.url === `file://${process.argv[1]}`) { + process.exitCode = await main(); +} diff --git a/node/src/client/benchmarkClient.ts b/node/src/client/benchmarkClient.ts new file mode 100644 index 0000000..214e655 --- /dev/null +++ b/node/src/client/benchmarkClient.ts @@ -0,0 +1,59 @@ +/** + * The interface every driver implements. + * + * One client instance maps to exactly one transport connection (the + * `client == connection` invariant shared by all engines); the engine never + * shares a client across workers. + * + * `measure()` is the single place latency is captured, so every driver reports it + * identically: `process.hrtime.bigint()` around the awaited command, truncated to + * whole microseconds, recorded even when the command throws. + */ + +import type { DriverConfig } from '../config/driverConfig.js'; +import type { TimedResult } from './timedResult.js'; + +export interface BenchmarkClient { + connect(host: string, port: number, config: DriverConfig): Promise; + ping(): Promise>; + get(key: string): Promise>; + set(key: string, value: Buffer): Promise>; + close(): Promise; + driverVersion(): string; + /** Secondary driver version, for composite drivers only. */ + secondaryDriverVersion?(): string | null; + /** + * Mark the client as warming up, mirroring Java's `setWarmupMode` + * (BenchmarkEngine.java:212-237). + * + * Real drivers ignore this. The recording driver uses it to suppress simulated + * errors so that an `error_rate` workload does not abort in warmup — the + * warmup fail-fast is meant to catch an unreachable server, not injected + * errors the phase is deliberately measuring. + */ + setWarmupMode?(warmup: boolean): void; +} + +const NANOS_PER_MICRO = 1000n; + +/** + * Await `operation` and record its latency in microseconds. + * + * Errors are captured, not thrown: the engine records a failed request and keeps + * going, matching the other engines. Latency is measured on the error path too. + */ +export async function measure(operation: () => Promise): Promise> { + const start = process.hrtime.bigint(); + try { + const value = await operation(); + const latencyMicros = Number((process.hrtime.bigint() - start) / NANOS_PER_MICRO); + return { value, latencyMicros }; + } catch (error) { + const latencyMicros = Number((process.hrtime.bigint() - start) / NANOS_PER_MICRO); + return { + value: null, + latencyMicros, + error: error instanceof Error ? error : new Error(String(error)), + }; + } +} diff --git a/node/src/client/driverVersion.ts b/node/src/client/driverVersion.ts new file mode 100644 index 0000000..bf7c933 --- /dev/null +++ b/node/src/client/driverVersion.ts @@ -0,0 +1,45 @@ +/** + * Reads an installed package's version for the NDJSON `metadata` block. + * + * `require('/package.json')` is NOT usable here: `@valkey/valkey-glide` + * declares an `exports` map with no `./package.json` entry, so the subpath import + * fails with ERR_PACKAGE_PATH_NOT_EXPORTED. Resolving the module's entry point + * and walking up to the nearest package.json with `fs` sidesteps `exports` + * entirely (it only governs specifier resolution, not file reads) and works for + * every driver. + */ + +import { readFileSync } from 'node:fs'; +import { createRequire } from 'node:module'; +import { dirname, join } from 'node:path'; + +const require = createRequire(import.meta.url); + +/** Depth to walk up from the resolved entry point looking for package.json. */ +const MAX_WALK_UP = 6; + +export function packageVersion(name: string): string { + let dir: string; + try { + dir = dirname(require.resolve(name)); + } catch { + return 'unknown'; + } + + for (let i = 0; i < MAX_WALK_UP; i++) { + try { + const manifest = JSON.parse(readFileSync(join(dir, 'package.json'), 'utf8')) as { + name?: string; + version?: string; + }; + // Guard against picking up a nested manifest of a different package. + if (manifest.name === name && typeof manifest.version === 'string') return manifest.version; + } catch { + // Not here (or unreadable) -- keep walking up. + } + const parent = dirname(dir); + if (parent === dir) break; + dir = parent; + } + return 'unknown'; +} diff --git a/node/src/client/factory.ts b/node/src/client/factory.ts new file mode 100644 index 0000000..a5dac3d --- /dev/null +++ b/node/src/client/factory.ts @@ -0,0 +1,81 @@ +/** + * Driver registry: maps `driver_id` to a client implementation. + * + * Implementations are loaded with a dynamic `import()` so `--info` and the unit + * tests work even if a driver's native bits are missing or broken on this + * platform -- only the driver actually requested gets loaded. + * + * The ids matter beyond this file. `DRIVER_ENGINE_MAP` in + * scripts/run_benchmark_matrix.py is a single global map shared by every engine, + * and `valkey-glide` there already means *Java*. Hence `valkey-glide-node`: a + * bare `valkey-glide` here would silently reroute Java's glide runs to Node. + */ + +import type { DriverConfig } from '../config/driverConfig.js'; +import type { BenchmarkClient } from './benchmarkClient.js'; + +interface DriverEntry { + description: string; + load: () => Promise; +} + +const DRIVERS = new Map([ + [ + 'valkey-glide-node', + { + description: 'Valkey GLIDE for Node.js (@valkey/valkey-glide)', + load: async () => new (await import('./impl/glideClient.js')).GlideBenchmarkClient(), + }, + ], + [ + 'ioredis', + { + description: 'ioredis — the most widely used Node.js Redis client', + load: async () => new (await import('./impl/ioredisClient.js')).IoredisBenchmarkClient(), + }, + ], + [ + 'iovalkey', + { + description: 'iovalkey — the Valkey-maintained fork of ioredis', + load: async () => new (await import('./impl/iovalkeyClient.js')).IovalkeyBenchmarkClient(), + }, + ], + [ + 'recording', + { + description: 'In-memory synthetic-latency client (no server required)', + load: async () => new (await import('./impl/recordingClient.js')).RecordingClient(), + }, + ], +]); + +export class BenchmarkClientFactory { + static supportedDrivers(): string[] { + return [...DRIVERS.keys()]; + } + + static describe(): Array<{ driverId: string; description: string }> { + return [...DRIVERS.entries()].map(([driverId, { description }]) => ({ driverId, description })); + } + + static async create(driverId: string): Promise { + const entry = DRIVERS.get((driverId ?? '').toLowerCase()); + if (entry === undefined) { + throw new Error( + `Unknown driver: ${driverId}. Supported: ${BenchmarkClientFactory.supportedDrivers().join(', ')}`, + ); + } + return entry.load(); + } + + static async createAndConnect( + host: string, + port: number, + config: DriverConfig, + ): Promise { + const client = await BenchmarkClientFactory.create(config.driverId); + await client.connect(host, port, config); + return client; + } +} diff --git a/node/src/client/impl/glideClient.ts b/node/src/client/impl/glideClient.ts new file mode 100644 index 0000000..74e3d2d --- /dev/null +++ b/node/src/client/impl/glideClient.ts @@ -0,0 +1,88 @@ +/** + * valkey-glide driver (`@valkey/valkey-glide`). + * + * One GlideClient per connection, honouring the `client == connection` invariant + * shared across engines. + * + * Two glide-specific shapes to note: + * - `close()` is **synchronous** (returns void, not a Promise), unlike every + * other driver here. + * - `get()` returns a `string` by default. That is deliberate and matches the + * Python engine and the ioredis/iovalkey clients here, so all drivers are + * charged for the same UTF-8 decode. Do not switch one driver to bytes. + */ + +import type { + GlideClient, + GlideClientConfiguration, + GlideClusterClient, + GlideClusterClientConfiguration, + ServerCredentials, +} from '@valkey/valkey-glide'; + +import type { DriverConfig } from '../../config/driverConfig.js'; +import { measure, type BenchmarkClient } from '../benchmarkClient.js'; +import { packageVersion } from '../driverVersion.js'; +import type { TimedResult } from '../timedResult.js'; + +const PACKAGE = '@valkey/valkey-glide'; + +export class GlideBenchmarkClient implements BenchmarkClient { + private client: GlideClient | GlideClusterClient | null = null; + + async connect(host: string, port: number, config: DriverConfig): Promise { + const glide = await import('@valkey/valkey-glide'); + + const addresses = [{ host, port }]; + const credentials: ServerCredentials | undefined = config.hasAuth() + ? ({ + password: config.auth?.password ?? '', + ...(config.auth?.username ? { username: config.auth.username } : {}), + } as ServerCredentials) + : undefined; + + const shared = { + addresses, + useTLS: config.tlsEnabled(), + ...(credentials ? { credentials } : {}), + ...(config.commandTimeoutMs ? { requestTimeout: config.commandTimeoutMs } : {}), + }; + + this.client = config.isCluster() + ? await glide.GlideClusterClient.createClient(shared as GlideClusterClientConfiguration) + : await glide.GlideClient.createClient(shared as GlideClientConfiguration); + } + + private requireClient(): GlideClient | GlideClusterClient { + if (this.client === null) throw new Error('glide client is not connected'); + return this.client; + } + + async ping(): Promise> { + const client = this.requireClient(); + return measure(async () => String(await client.ping())); + } + + async get(key: string): Promise> { + const client = this.requireClient(); + return measure(async () => { + const value = await client.get(key); + return value === null ? null : String(value); + }) as Promise>; + } + + async set(key: string, value: Buffer): Promise> { + const client = this.requireClient(); + return measure(async () => String(await client.set(key, value))); + } + + async close(): Promise { + // Synchronous in glide -- there is nothing to await. + this.client?.close(); + this.client = null; + } + + driverVersion(): string { + return packageVersion(PACKAGE); + } +} diff --git a/node/src/client/impl/ioredisClient.ts b/node/src/client/impl/ioredisClient.ts new file mode 100644 index 0000000..85b609b --- /dev/null +++ b/node/src/client/impl/ioredisClient.ts @@ -0,0 +1,16 @@ +/** ioredis driver — the most widely used Node.js Redis client. */ + +import { IoredisFamilyClient, type RedisModuleLike } from './ioredisFamilyClient.js'; + +export class IoredisBenchmarkClient extends IoredisFamilyClient { + protected override packageName(): string { + return 'ioredis'; + } + + protected override async loadModule(): Promise { + const module = await import('ioredis'); + // ioredis' constructors carry overloads that RedisModuleLike narrows to the + // one form this engine calls; the shapes are compatible at runtime. + return { Redis: module.Redis, Cluster: module.Cluster } as unknown as RedisModuleLike; + } +} diff --git a/node/src/client/impl/ioredisFamilyClient.ts b/node/src/client/impl/ioredisFamilyClient.ts new file mode 100644 index 0000000..2959874 --- /dev/null +++ b/node/src/client/impl/ioredisFamilyClient.ts @@ -0,0 +1,131 @@ +/** + * Shared implementation for the ioredis-family drivers. + * + * `ioredis` and `iovalkey` (the Valkey-maintained fork) expose the same + * constructor options and the same `Redis`/`Cluster` exports, so both drivers + * differ only in which module they load and which package they report a version + * for. They are described structurally here rather than against either package's + * types, so neither becomes a compile-time dependency of the other's driver. + * + * Fairness note: `enableAutoPipelining` is forced **off**. Left on (it is off by + * default, but that default has changed before) ioredis transparently batches + * commands issued in the same event-loop tick, which would inflate throughput + * against every other engine and silently make the comparison meaningless. + */ + +import { readFileSync } from 'node:fs'; + +import type { DriverConfig } from '../../config/driverConfig.js'; +import { measure, type BenchmarkClient } from '../benchmarkClient.js'; +import { packageVersion } from '../driverVersion.js'; +import type { TimedResult } from '../timedResult.js'; + +/** The slice of the ioredis surface this engine uses. */ +interface RedisLike { + connect(): Promise; + ping(): Promise; + get(key: string): Promise; + set(key: string, value: Buffer): Promise; + quit(): Promise; + disconnect(): void; +} + +export interface RedisModuleLike { + Redis: new (options: Record) => RedisLike; + Cluster: new ( + nodes: Array<{ host: string; port: number }>, + options: Record, + ) => RedisLike; +} + +function buildTlsOptions(config: DriverConfig): Record | undefined { + if (!config.tlsEnabled()) return undefined; + const tls: Record = {}; + if (config.tls?.ca_path) tls['ca'] = readFileSync(config.tls.ca_path); + if (config.tls?.cert_path) tls['cert'] = readFileSync(config.tls.cert_path); + if (config.tls?.key_path) tls['key'] = readFileSync(config.tls.key_path); + if (config.tls?.verify_hostname === false) tls['rejectUnauthorized'] = false; + return tls; +} + +export abstract class IoredisFamilyClient implements BenchmarkClient { + private client: RedisLike | null = null; + + /** npm package name, used for both loading and version reporting. */ + protected abstract packageName(): string; + + protected abstract loadModule(): Promise; + + async connect(host: string, port: number, config: DriverConfig): Promise { + const module = await this.loadModule(); + const tls = buildTlsOptions(config); + + const options: Record = { + // Batching commands issued in one tick would not be comparable to the + // other engines -- keep every request a distinct round trip. + enableAutoPipelining: false, + // Connect explicitly below so a connection failure surfaces here rather + // than as a first-command error, and so cps_limit really gates setup. + lazyConnect: true, + // Fail a stuck request instead of retrying it under a different latency. + maxRetriesPerRequest: 0, + // Never silently reconnect. ioredis' default retryStrategy retries + // forever, so a wrong host would hang the run instead of failing it, and a + // mid-phase reconnect would fold connection setup into request latency. + retryStrategy: () => null, + // Bound the initial connect so an unreachable host fails fast. + connectTimeout: config.commandTimeoutMs ?? 10_000, + ...(config.hasAuth() && config.auth?.username ? { username: config.auth.username } : {}), + ...(config.hasAuth() && config.auth?.password ? { password: config.auth.password } : {}), + ...(config.commandTimeoutMs ? { commandTimeout: config.commandTimeoutMs } : {}), + ...(tls ? { tls } : {}), + }; + + if (config.isCluster()) { + this.client = new module.Cluster([{ host, port }], { + lazyConnect: true, + redisOptions: options, + }); + } else { + this.client = new module.Redis({ ...options, host, port }); + } + await this.client.connect(); + } + + private requireClient(): RedisLike { + if (this.client === null) throw new Error(`${this.packageName()} client is not connected`); + return this.client; + } + + async ping(): Promise> { + const client = this.requireClient(); + return measure(() => client.ping()); + } + + async get(key: string): Promise> { + const client = this.requireClient(); + return measure(() => client.get(key)) as Promise>; + } + + async set(key: string, value: Buffer): Promise> { + const client = this.requireClient(); + return measure(() => client.set(key, value)) as Promise>; + } + + async close(): Promise { + if (this.client === null) return; + const client = this.client; + this.client = null; + try { + await client.quit(); + } catch { + // A server that already dropped the connection makes QUIT reject; the + // socket still has to go, or the process will not exit. + client.disconnect(); + } + } + + driverVersion(): string { + return packageVersion(this.packageName()); + } +} diff --git a/node/src/client/impl/iovalkeyClient.ts b/node/src/client/impl/iovalkeyClient.ts new file mode 100644 index 0000000..0340d77 --- /dev/null +++ b/node/src/client/impl/iovalkeyClient.ts @@ -0,0 +1,19 @@ +/** + * iovalkey driver — the Valkey-maintained fork of ioredis. + * + * API-identical to ioredis, so all behaviour lives in the shared base class. + */ + +import { IoredisFamilyClient, type RedisModuleLike } from './ioredisFamilyClient.js'; + +export class IovalkeyBenchmarkClient extends IoredisFamilyClient { + protected override packageName(): string { + return 'iovalkey'; + } + + protected override async loadModule(): Promise { + const module = await import('iovalkey'); + // Same overload-narrowing as the ioredis driver; iovalkey mirrors its API. + return { Redis: module.Redis, Cluster: module.Cluster } as unknown as RedisModuleLike; + } +} diff --git a/node/src/client/impl/recordingClient.ts b/node/src/client/impl/recordingClient.ts new file mode 100644 index 0000000..2b60618 --- /dev/null +++ b/node/src/client/impl/recordingClient.ts @@ -0,0 +1,131 @@ +/** + * In-memory recording client for server-free testing. + * + * Records operations and supports simulated latency and error injection via + * `specific_driver_config` (`operation_delay_micros`, `delay_variation_micros`, + * `error_rate`, `error_message`). This lets the integration tests exercise the + * whole engine without a live server, mirroring the Ruby/Python recording + * drivers. + */ + +import type { DriverConfig } from '../../config/driverConfig.js'; +import type { BenchmarkClient } from '../benchmarkClient.js'; +import type { TimedResult } from '../timedResult.js'; + +export interface RecordedOperation { + command: string; + key: string | null; + value: Buffer | null; + success: boolean; + errorMessage: string | null; +} + +const NANOS_PER_MICRO = 1000n; + +function readNumber(source: Record, key: string, fallback: number): number { + const value = source[key]; + return typeof value === 'number' && Number.isFinite(value) ? value : fallback; +} + +/** Sleep for whole microseconds, tolerating setTimeout's ~1ms floor. */ +function sleepMicros(micros: number): Promise { + return new Promise((resolve) => setTimeout(resolve, micros / 1000)); +} + +export class RecordingClient implements BenchmarkClient { + readonly operations: RecordedOperation[] = []; + private readonly storedData = new Map(); + private operationDelayMicros = 0; + private delayVariationMicros = 0; + private errorRate = 0; + private errorMessage = 'Simulated error'; + private warmupMode = false; + + async connect(_host: string, _port: number, config: DriverConfig): Promise { + const cfg = config.specificDriverConfig; + this.operationDelayMicros = readNumber(cfg, 'operation_delay_micros', 0); + this.delayVariationMicros = readNumber(cfg, 'delay_variation_micros', 0); + this.errorRate = readNumber(cfg, 'error_rate', 0); + const message = cfg['error_message']; + if (typeof message === 'string') this.errorMessage = message; + this.record('CONNECT', null, null, true, null); + } + + async ping(): Promise> { + const { latencyMicros, success } = await this.simulate(); + this.record('PING', null, null, success, success ? null : this.errorMessage); + return success + ? { value: 'PONG', latencyMicros } + : { value: null, latencyMicros, error: new Error(this.errorMessage) }; + } + + async get(key: string): Promise> { + const { latencyMicros, success } = await this.simulate(); + this.record('GET', key, null, success, success ? null : this.errorMessage); + if (!success) return { value: null, latencyMicros, error: new Error(this.errorMessage) }; + const stored = this.storedData.get(key); + return { value: stored === undefined ? null : stored.toString('latin1'), latencyMicros }; + } + + async set(key: string, value: Buffer): Promise> { + const { latencyMicros, success } = await this.simulate(); + if (success) this.storedData.set(key, value); + this.record('SET', key, value, success, success ? null : this.errorMessage); + return success + ? { value: 'OK', latencyMicros } + : { value: null, latencyMicros, error: new Error(this.errorMessage) }; + } + + async close(): Promise { + this.record('CLOSE', null, null, true, null); + } + + driverVersion(): string { + return '1.0.0'; + } + + /** + * Suppress simulated errors during warmup, matching Java's recording client. + * The engine's warmup fail-fast exists to catch an unreachable server, so + * injected errors must not trip it. + */ + setWarmupMode(warmup: boolean): void { + this.warmupMode = warmup; + } + + private record( + command: string, + key: string | null, + value: Buffer | null, + success: boolean, + errorMessage: string | null, + ): void { + this.operations.push({ command, key, value, success, errorMessage }); + } + + private async simulate(): Promise<{ latencyMicros: number; success: boolean }> { + const start = process.hrtime.bigint(); + const delayMicros = this.calculateDelayMicros(); + if (delayMicros > 0) await sleepMicros(delayMicros); + const latencyMicros = Number((process.hrtime.bigint() - start) / NANOS_PER_MICRO); + return { latencyMicros, success: !this.shouldSimulateError() }; + } + + private calculateDelayMicros(): number { + if (this.operationDelayMicros <= 0) return 0; + let delay = this.operationDelayMicros; + if (this.delayVariationMicros > 0) { + const spread = 2 * this.delayVariationMicros + 1; + const variation = Math.floor(Math.random() * spread) - this.delayVariationMicros; + delay = Math.max(0, delay + variation); + } + return delay; + } + + private shouldSimulateError(): boolean { + if (this.warmupMode) return false; + if (this.errorRate <= 0) return false; + if (this.errorRate >= 1) return true; + return Math.random() < this.errorRate; + } +} diff --git a/node/src/client/timedResult.ts b/node/src/client/timedResult.ts new file mode 100644 index 0000000..3d36c5a --- /dev/null +++ b/node/src/client/timedResult.ts @@ -0,0 +1,12 @@ +/** + * The outcome of a single measured command. + * + * `error === undefined` means success. Latency is populated on both paths so a + * failure still contributes a measured duration, matching the other engines. + */ +export interface TimedResult { + readonly value: T | null; + /** Command latency in whole microseconds. Recorded even on error. */ + readonly latencyMicros: number; + readonly error?: Error; +} diff --git a/node/src/command/command.ts b/node/src/command/command.ts new file mode 100644 index 0000000..8c74699 --- /dev/null +++ b/node/src/command/command.ts @@ -0,0 +1,19 @@ +/** A benchmark command: a weighted operation the engine can issue. */ + +import type { BenchmarkClient } from '../client/benchmarkClient.js'; + +export interface CommandResult { + readonly commandName: string; + readonly latencyMicros: number; + readonly success: boolean; + readonly errorMessage?: string; +} + +export interface Command { + /** Upper-case command name, used as the NDJSON metrics key (GET/SET/PING). */ + readonly name: string; + readonly weight: number; + /** Whether this command consumes a generated key (PING does not). */ + readonly usesKey: boolean; + execute(client: BenchmarkClient, key: string): Promise; +} diff --git a/node/src/command/factory.ts b/node/src/command/factory.ts new file mode 100644 index 0000000..635a85a --- /dev/null +++ b/node/src/command/factory.ts @@ -0,0 +1,39 @@ +/** Maps a workload's `command` strings to Command implementations. */ + +import type { CommandConfig } from '../config/commandConfig.js'; +import type { Command } from './command.js'; +import { GetCommand } from './impl/getCommand.js'; +import { PingCommand } from './impl/pingCommand.js'; +import { SetCommand } from './impl/setCommand.js'; + +type CommandBuilder = (config: CommandConfig) => Command; + +const BUILDERS = new Map([ + ['get', { build: (c) => new GetCommand(c), description: 'GET key' }], + ['set', { build: (c) => new SetCommand(c), description: 'SET key value' }], + ['ping', { build: (c) => new PingCommand(c), description: 'PING' }], +]); + +export class CommandFactory { + static supportedCommands(): string[] { + return [...BUILDERS.keys()]; + } + + static describe(): Array<{ name: string; description: string }> { + return [...BUILDERS.entries()].map(([name, { description }]) => ({ name, description })); + } + + static create(config: CommandConfig): Command { + const entry = BUILDERS.get(config.command.toLowerCase()); + if (entry === undefined) { + throw new Error( + `Unknown command: ${config.command}. Supported: ${CommandFactory.supportedCommands().join(', ')}`, + ); + } + return entry.build(config); + } + + static createAll(configs: CommandConfig[]): Command[] { + return configs.map((config) => CommandFactory.create(config)); + } +} diff --git a/node/src/command/impl/getCommand.ts b/node/src/command/impl/getCommand.ts new file mode 100644 index 0000000..723e512 --- /dev/null +++ b/node/src/command/impl/getCommand.ts @@ -0,0 +1,23 @@ +import type { BenchmarkClient } from '../../client/benchmarkClient.js'; +import type { CommandConfig } from '../../config/commandConfig.js'; +import type { Command, CommandResult } from '../command.js'; + +export class GetCommand implements Command { + readonly name = 'GET'; + readonly weight: number; + readonly usesKey = true; + + constructor(config: CommandConfig) { + this.weight = config.weight; + } + + async execute(client: BenchmarkClient, key: string): Promise { + const result = await client.get(key); + return { + commandName: this.name, + latencyMicros: result.latencyMicros, + success: result.error === undefined, + ...(result.error ? { errorMessage: result.error.message } : {}), + }; + } +} diff --git a/node/src/command/impl/pingCommand.ts b/node/src/command/impl/pingCommand.ts new file mode 100644 index 0000000..86ac70f --- /dev/null +++ b/node/src/command/impl/pingCommand.ts @@ -0,0 +1,23 @@ +import type { BenchmarkClient } from '../../client/benchmarkClient.js'; +import type { CommandConfig } from '../../config/commandConfig.js'; +import type { Command, CommandResult } from '../command.js'; + +export class PingCommand implements Command { + readonly name = 'PING'; + readonly weight: number; + readonly usesKey = false; + + constructor(config: CommandConfig) { + this.weight = config.weight; + } + + async execute(client: BenchmarkClient): Promise { + const result = await client.ping(); + return { + commandName: this.name, + latencyMicros: result.latencyMicros, + success: result.error === undefined, + ...(result.error ? { errorMessage: result.error.message } : {}), + }; + } +} diff --git a/node/src/command/impl/setCommand.ts b/node/src/command/impl/setCommand.ts new file mode 100644 index 0000000..eae42ff --- /dev/null +++ b/node/src/command/impl/setCommand.ts @@ -0,0 +1,41 @@ +import type { BenchmarkClient } from '../../client/benchmarkClient.js'; +import type { CommandConfig } from '../../config/commandConfig.js'; +import type { Command, CommandResult } from '../command.js'; + +/** + * Deterministic filler pattern, matching the Ruby and Python engines. Java uses + * random bytes instead, but only the payload *length* affects RESP framing and + * server work, and a fixed pattern makes runs reproducible. + */ +const PATTERN = '0123456789ABCDEF'; + +export class SetCommand implements Command { + readonly name = 'SET'; + readonly weight: number; + readonly usesKey = true; + /** + * Built once at construction, not per request: allocating a fresh payload in + * the hot loop would charge the driver for the engine's own GC churn. + */ + private readonly value: Buffer; + + constructor(config: CommandConfig) { + this.weight = config.weight; + this.value = SetCommand.generateValue(config.dataSizeBytes); + } + + async execute(client: BenchmarkClient, key: string): Promise { + const result = await client.set(key, this.value); + return { + commandName: this.name, + latencyMicros: result.latencyMicros, + success: result.error === undefined, + ...(result.error ? { errorMessage: result.error.message } : {}), + }; + } + + static generateValue(size: number): Buffer { + const repeats = Math.floor(size / PATTERN.length) + 1; + return Buffer.from(PATTERN.repeat(repeats).slice(0, size), 'latin1'); + } +} diff --git a/node/src/config/commandConfig.ts b/node/src/config/commandConfig.ts new file mode 100644 index 0000000..eb85f02 --- /dev/null +++ b/node/src/config/commandConfig.ts @@ -0,0 +1,21 @@ +/** A single weighted command entry within a phase. */ + +/** + * Default SET payload size. 256 is the cross-engine default: Java + * (SetCommand.java `getDataSizeBytesOrDefault(256)`), C# (SetCommand.cs), + * Ruby and Python all use it. Do not change without changing them too. + */ +export const DEFAULT_DATA_SIZE_BYTES = 256; + +export class CommandConfig { + readonly command: string; + readonly weight: number; + readonly dataSizeBytes: number; + + constructor(init: { command: string; weight?: number | null; dataSizeBytes?: number | null }) { + this.command = init.command.toLowerCase(); + // A missing weight defaults to 1.0 (matching Java), rather than NaN. + this.weight = init.weight ?? 1.0; + this.dataSizeBytes = init.dataSizeBytes ?? DEFAULT_DATA_SIZE_BYTES; + } +} diff --git a/node/src/config/completionConfig.ts b/node/src/config/completionConfig.ts new file mode 100644 index 0000000..1dd8770 --- /dev/null +++ b/node/src/config/completionConfig.ts @@ -0,0 +1,29 @@ +/** Phase completion criteria: run for a duration, or until a request count. */ + +export class CompletionConfig { + readonly type: string; + readonly seconds: number | null; + readonly requests: number | null; + + constructor(init: { type: string; seconds?: number | null; requests?: number | null }) { + this.type = init.type; + this.seconds = init.seconds ?? null; + this.requests = init.requests ?? null; + } + + isDurationBased(): boolean { + return this.type === 'duration'; + } + + isRequestBased(): boolean { + return this.type === 'requests'; + } + + durationSeconds(): number { + return this.seconds ?? 0; + } + + totalRequests(): number { + return this.requests ?? 0; + } +} diff --git a/node/src/config/driverConfig.ts b/node/src/config/driverConfig.ts new file mode 100644 index 0000000..c432107 --- /dev/null +++ b/node/src/config/driverConfig.ts @@ -0,0 +1,80 @@ +/** + * Driver (client library) configuration. + * + * Maps to configs/schemas/driver-config.schema.json. Field names and defaults + * mirror the Java/Ruby/Python engines so the same JSON files work unchanged + * across every engine. + */ + +export interface TlsConfig { + enabled?: boolean; + cert_path?: string; + key_path?: string; + ca_path?: string; + verify_hostname?: boolean; +} + +export interface AuthConfig { + username?: string; + password?: string; +} + +export const DEFAULT_MODE = 'standalone'; + +export class DriverConfig { + readonly schemaVersion: string; + readonly description: string | null; + readonly driverId: string; + readonly mode: string; + readonly commandTimeoutMs: number | null; + readonly tls: TlsConfig | null; + readonly auth: AuthConfig | null; + readonly specificDriverConfig: Record; + + constructor(init: { + schemaVersion?: string; + description?: string | null; + driverId: string; + mode?: string | null; + commandTimeoutMs?: number | null; + tls?: TlsConfig | null; + auth?: AuthConfig | null; + specificDriverConfig?: Record | null; + }) { + this.schemaVersion = init.schemaVersion ?? '1.0'; + this.description = init.description ?? null; + this.driverId = init.driverId; + this.mode = init.mode ?? DEFAULT_MODE; + this.commandTimeoutMs = init.commandTimeoutMs ?? null; + this.tls = init.tls ?? null; + this.auth = init.auth ?? null; + this.specificDriverConfig = init.specificDriverConfig ?? {}; + } + + /** Secondary driver id for composite drivers (e.g. Java's spring-data-*). */ + secondaryDriverId(): string | null { + const value = this.specificDriverConfig['secondary_driver_id']; + return typeof value === 'string' ? value : null; + } + + isStandalone(): boolean { + return this.mode === 'standalone'; + } + + isCluster(): boolean { + return this.mode === 'cluster'; + } + + isSentinel(): boolean { + return this.mode === 'sentinel'; + } + + tlsEnabled(): boolean { + return this.tls?.enabled === true; + } + + /** Username/password only count as auth when at least one is non-empty. */ + hasAuth(): boolean { + return Boolean(this.auth && (this.auth.password || this.auth.username)); + } +} diff --git a/node/src/config/keyspaceConfig.ts b/node/src/config/keyspaceConfig.ts new file mode 100644 index 0000000..b20b4f9 --- /dev/null +++ b/node/src/config/keyspaceConfig.ts @@ -0,0 +1,41 @@ +/** Key-generation configuration for a benchmark phase. */ + +export const DEFAULT_KEY_SIZE_BYTES = 16; +export const DEFAULT_KEY_PREFIX = 'bench:'; +export const DEFAULT_GENERATION_ALG = 'sequential_int'; + +export class KeyspaceConfig { + readonly keysCount: number; + readonly keySizeBytes: number; + readonly keyPrefix: string; + readonly generationAlg: string; + readonly seed: number | null; + + constructor(init: { + keysCount: number; + keySizeBytes?: number | null; + keyPrefix?: string | null; + generationAlg?: string | null; + seed?: number | null; + }) { + this.keysCount = init.keysCount; + // A null/undefined value falls back to the default rather than staying + // null, matching the Ruby/Python engines. + this.keySizeBytes = init.keySizeBytes ?? DEFAULT_KEY_SIZE_BYTES; + this.keyPrefix = init.keyPrefix ?? DEFAULT_KEY_PREFIX; + this.generationAlg = init.generationAlg ?? DEFAULT_GENERATION_ALG; + this.seed = init.seed ?? null; + } + + isSequentialInt(): boolean { + return this.generationAlg === 'sequential_int'; + } + + isUniformRand(): boolean { + return this.generationAlg === 'uniform_rand'; + } + + seedValue(): number { + return this.seed ?? 0; + } +} diff --git a/node/src/config/loader.ts b/node/src/config/loader.ts new file mode 100644 index 0000000..000a914 --- /dev/null +++ b/node/src/config/loader.ts @@ -0,0 +1,198 @@ +/** + * Loads driver and workload configuration from the shared configs/ JSON files. + * + * Field names and defaults mirror the Java/Ruby/Python engines exactly, so the + * same JSON is consumed identically by every engine. Unlike those engines this + * one also validates the required fields up front: a typo'd config should fail + * with a clear message rather than surface later as a null-shaped error deep in + * a worker loop. + */ + +import { readFileSync } from 'node:fs'; + +import { CommandConfig } from './commandConfig.js'; +import { CompletionConfig } from './completionConfig.js'; +import { DriverConfig } from './driverConfig.js'; +import { KeyspaceConfig } from './keyspaceConfig.js'; +import { PhaseConfig } from './phaseConfig.js'; +import { WorkloadConfig } from './workloadConfig.js'; + +type Json = Record; + +export class ConfigError extends Error {} + +function asRecord(value: unknown, what: string): Json { + if (value === null || typeof value !== 'object' || Array.isArray(value)) { + throw new ConfigError(`${what} must be a JSON object`); + } + return value as Json; +} + +function optString(data: Json, key: string): string | null { + const value = data[key]; + if (value === undefined || value === null) return null; + if (typeof value !== 'string') throw new ConfigError(`"${key}" must be a string`); + return value; +} + +function reqString(data: Json, key: string, what: string): string { + const value = optString(data, key); + if (value === null || value === '') throw new ConfigError(`${what} is missing required "${key}"`); + return value; +} + +function optNumber(data: Json, key: string): number | null { + const value = data[key]; + if (value === undefined || value === null) return null; + if (typeof value !== 'number' || !Number.isFinite(value)) { + throw new ConfigError(`"${key}" must be a finite number`); + } + return value; +} + +function reqInt(data: Json, key: string, what: string): number { + const value = optNumber(data, key); + if (value === null) throw new ConfigError(`${what} is missing required "${key}"`); + if (!Number.isInteger(value)) throw new ConfigError(`"${key}" must be an integer`); + return value; +} + +function readJson(path: string, what: string): Json { + let text: string; + try { + text = readFileSync(path, 'utf8'); + } catch (error) { + throw new ConfigError(`cannot read ${what} "${path}": ${(error as Error).message}`); + } + try { + return asRecord(JSON.parse(text), what); + } catch (error) { + if (error instanceof ConfigError) throw error; + throw new ConfigError(`${what} "${path}" is not valid JSON: ${(error as Error).message}`); + } +} + +export class ConfigLoader { + static loadDriverConfig(path: string): DriverConfig { + return ConfigLoader.parseDriverConfig(readJson(path, 'driver config')); + } + + static loadWorkloadConfig(path: string): WorkloadConfig { + return ConfigLoader.parseWorkloadConfig(readJson(path, 'workload config')); + } + + static parseDriverConfig(data: Json): DriverConfig { + const mode = optString(data, 'mode') ?? 'standalone'; + if (!['standalone', 'cluster', 'sentinel'].includes(mode)) { + throw new ConfigError( + `driver config "mode" must be standalone, cluster or sentinel (got "${mode}")`, + ); + } + return new DriverConfig({ + schemaVersion: optString(data, 'schema_version') ?? '1.0', + description: optString(data, 'description'), + driverId: reqString(data, 'driver_id', 'driver config'), + mode, + commandTimeoutMs: optNumber(data, 'command_timeout_ms'), + tls: (data['tls'] ?? null) as DriverConfig['tls'], + auth: (data['auth'] ?? null) as DriverConfig['auth'], + specificDriverConfig: (data['specific_driver_config'] ?? {}) as Record, + }); + } + + static parseWorkloadConfig(data: Json): WorkloadConfig { + const rawPhases = data['phases']; + if (!Array.isArray(rawPhases) || rawPhases.length === 0) { + throw new ConfigError('workload config must have a non-empty "phases" array'); + } + return new WorkloadConfig({ + schemaVersion: optString(data, 'schema_version') ?? '1.0', + benchmarkProfile: (data['benchmark_profile'] ?? {}) as WorkloadConfig['benchmarkProfile'], + phases: rawPhases.map((phase, index) => + ConfigLoader.parsePhase(asRecord(phase, `phases[${index}]`), index), + ), + }); + } + + static parsePhase(data: Json, index = 0): PhaseConfig { + const what = `phases[${index}]`; + const rawCommands = data['commands']; + if (!Array.isArray(rawCommands) || rawCommands.length === 0) { + throw new ConfigError(`${what} must have a non-empty "commands" array`); + } + const connections = reqInt(data, 'connections', what); + if (connections <= 0) throw new ConfigError(`${what} "connections" must be positive`); + + return new PhaseConfig({ + id: reqString(data, 'id', what), + description: optString(data, 'description'), + connections, + cpsLimit: optNumber(data, 'cps_limit'), + rpsLimit: optNumber(data, 'rps_limit'), + pipelineDepth: optNumber(data, 'pipeline_depth'), + warmupRequests: optNumber(data, 'warmup_requests'), + completion: ConfigLoader.parseCompletion( + asRecord(data['completion'] ?? {}, `${what}.completion`), + what, + ), + keyspace: ConfigLoader.parseKeyspace( + asRecord(data['keyspace'] ?? {}, `${what}.keyspace`), + what, + ), + commands: rawCommands.map((command, i) => + ConfigLoader.parseCommand(asRecord(command, `${what}.commands[${i}]`), `${what}.commands[${i}]`), + ), + }); + } + + static parseCompletion(data: Json, what = 'completion'): CompletionConfig { + const type = reqString(data, 'type', `${what}.completion`); + if (!['duration', 'requests'].includes(type)) { + throw new ConfigError( + `${what}.completion "type" must be duration or requests (got "${type}")`, + ); + } + const seconds = optNumber(data, 'seconds'); + const requests = optNumber(data, 'requests'); + if (type === 'duration' && (seconds === null || seconds <= 0)) { + throw new ConfigError(`${what}.completion type=duration requires a positive "seconds"`); + } + if (type === 'requests' && (requests === null || requests <= 0)) { + throw new ConfigError(`${what}.completion type=requests requires a positive "requests"`); + } + return new CompletionConfig({ type, seconds, requests }); + } + + static parseKeyspace(data: Json, what = 'keyspace'): KeyspaceConfig { + const keysCount = reqInt(data, 'keys_count', `${what}.keyspace`); + if (keysCount <= 0) throw new ConfigError(`${what}.keyspace "keys_count" must be positive`); + const generationAlg = optString(data, 'generation_alg') ?? 'sequential_int'; + if (!['sequential_int', 'uniform_rand'].includes(generationAlg)) { + throw new ConfigError( + `${what}.keyspace "generation_alg" must be sequential_int or uniform_rand ` + + `(got "${generationAlg}")`, + ); + } + return new KeyspaceConfig({ + keysCount, + keySizeBytes: optNumber(data, 'key_size_bytes'), + keyPrefix: optString(data, 'key_prefix'), + generationAlg, + seed: optNumber(data, 'seed'), + }); + } + + static parseCommand(data: Json, what = 'command'): CommandConfig { + const command = reqString(data, 'command', what); + const weight = optNumber(data, 'weight'); + if (weight !== null && (weight < 0 || weight > 1)) { + // Matches Java's CommandConfig.validate(): weights are fractions of 1. + throw new ConfigError(`${what} "weight" must be between 0 and 1 (got ${weight})`); + } + return new CommandConfig({ + command, + weight, + dataSizeBytes: optNumber(data, 'data_size_bytes'), + }); + } +} diff --git a/node/src/config/phaseConfig.ts b/node/src/config/phaseConfig.ts new file mode 100644 index 0000000..c9186f5 --- /dev/null +++ b/node/src/config/phaseConfig.ts @@ -0,0 +1,58 @@ +/** Configuration for a single benchmark phase. */ + +import type { CommandConfig } from './commandConfig.js'; +import type { CompletionConfig } from './completionConfig.js'; +import type { KeyspaceConfig } from './keyspaceConfig.js'; + +export const DEFAULT_PIPELINE_DEPTH = 1; +export const DEFAULT_WARMUP_REQUESTS = 1; + +export class PhaseConfig { + readonly id: string; + readonly description: string | null; + readonly connections: number; + readonly completion: CompletionConfig; + readonly keyspace: KeyspaceConfig; + readonly commands: CommandConfig[]; + readonly cpsLimit: number; + readonly rpsLimit: number; + readonly pipelineDepth: number; + readonly warmupRequests: number; + + constructor(init: { + id: string; + description?: string | null; + connections: number; + completion: CompletionConfig; + keyspace: KeyspaceConfig; + commands: CommandConfig[]; + cpsLimit?: number | null; + rpsLimit?: number | null; + pipelineDepth?: number | null; + warmupRequests?: number | null; + }) { + this.id = init.id; + this.description = init.description ?? null; + this.connections = init.connections; + this.completion = init.completion; + this.keyspace = init.keyspace; + this.commands = init.commands; + // -1 is the configs' "unlimited" sentinel; null/undefined means the same. + this.cpsLimit = init.cpsLimit ?? -1; + this.rpsLimit = init.rpsLimit ?? -1; + this.pipelineDepth = init.pipelineDepth ?? DEFAULT_PIPELINE_DEPTH; + this.warmupRequests = init.warmupRequests ?? DEFAULT_WARMUP_REQUESTS; + } + + hasCpsLimit(): boolean { + return this.cpsLimit > 0; + } + + hasRpsLimit(): boolean { + return this.rpsLimit > 0; + } + + effectivePipelineDepth(): number { + return this.pipelineDepth > 0 ? this.pipelineDepth : DEFAULT_PIPELINE_DEPTH; + } +} diff --git a/node/src/config/workloadConfig.ts b/node/src/config/workloadConfig.ts new file mode 100644 index 0000000..74c235e --- /dev/null +++ b/node/src/config/workloadConfig.ts @@ -0,0 +1,29 @@ +/** A whole workload: a benchmark profile plus an ordered list of phases. */ + +import type { PhaseConfig } from './phaseConfig.js'; + +export interface BenchmarkProfile { + name?: string; + description?: string; + version?: string; +} + +export class WorkloadConfig { + readonly schemaVersion: string; + readonly benchmarkProfile: BenchmarkProfile; + readonly phases: PhaseConfig[]; + + constructor(init: { + schemaVersion?: string; + benchmarkProfile?: BenchmarkProfile | null; + phases: PhaseConfig[]; + }) { + this.schemaVersion = init.schemaVersion ?? '1.0'; + this.benchmarkProfile = init.benchmarkProfile ?? {}; + this.phases = init.phases; + } + + name(): string { + return this.benchmarkProfile.name ?? 'unnamed'; + } +} diff --git a/node/src/engine/benchmark.ts b/node/src/engine/benchmark.ts new file mode 100644 index 0000000..67c5aeb --- /dev/null +++ b/node/src/engine/benchmark.ts @@ -0,0 +1,397 @@ +/** + * Benchmark engine. + * + * Concurrency model: a single event loop with **one client per connection** (the + * `client == connection` invariant every engine holds) and **one worker per + * connection**, all started together via `Promise.all`. Node is single-threaded + * with async I/O, so this is the faithful analogue of Java's + * virtual-thread-per-client design -- an awaited command parks the worker, not + * the loop, so the other connections keep making progress. + * + * Two loops, mirroring Java (BenchmarkEngine.java:353-452): + * - `pipeline_depth <= 1`: issue one command, await it, record, repeat. + * - `pipeline_depth > 1`: keep up to `pipelineDepth` requests in flight per + * connection, awaiting whichever settles first and immediately refilling. + * + * The request budget is **shared across all workers**, claimed one request at a + * time, exactly as Java does with its per-phase `AtomicLong` + * (BenchmarkEngine.java:249, 363-367). It is deliberately not pre-divided per + * worker: with a shared budget a slow connection cannot cap the run -- faster + * workers absorb the slack and the phase ends when the total budget is spent. + * Pre-splitting would bound wall-clock by the slowest connection and change the + * per-connection distribution, which is a real cross-engine comparability gap. + */ + +import type { BenchmarkClient } from '../client/benchmarkClient.js'; +import { BenchmarkClientFactory } from '../client/factory.js'; +import type { Command, CommandResult } from '../command/command.js'; +import { CommandFactory } from '../command/factory.js'; +import type { DriverConfig } from '../config/driverConfig.js'; +import type { PhaseConfig } from '../config/phaseConfig.js'; +import type { WorkloadConfig } from '../config/workloadConfig.js'; +import { MetricsCollector } from '../metrics/collector.js'; +import { NdjsonWriter } from '../metrics/ndjsonWriter.js'; +import { CommandSelector } from './commandSelector.js'; +import { Counter, KeyGenerator } from './keyGenerator.js'; +import { RateLimiter } from './rateLimiter.js'; + +const PROGRESS_LOG_INTERVAL_MS = 10_000; +const CONNECTION_LOG_INTERVAL = 50; + +export interface Logger { + info(message: string): void; + warn(message: string): void; + error(message: string): void; +} + +export const consoleLogger: Logger = { + info: (message) => console.log(`${new Date().toISOString()} INFO ${message}`), + warn: (message) => console.warn(`${new Date().toISOString()} WARN ${message}`), + error: (message) => console.error(`${new Date().toISOString()} ERROR ${message}`), +}; + +/** + * A shared, monotonically-drained request budget for one phase. + * + * `claim()` is atomic without a lock: it reads and writes with no `await` in + * between, so concurrent workers on the single event-loop thread can never + * interleave inside it. + */ +class RequestBudget { + private remaining: number; + + constructor(total: number) { + this.remaining = total; + } + + claim(): boolean { + if (this.remaining <= 0) return false; + this.remaining -= 1; + return true; + } +} + +export interface BenchmarkEngineOptions { + host: string; + port: number; + driverConfig: DriverConfig; + workloadConfig: WorkloadConfig; + metricsPath: string; + commitId?: string | null; + logger?: Logger; +} + +export class BenchmarkEngine { + private readonly host: string; + private readonly port: number; + private readonly driverConfig: DriverConfig; + private readonly workloadConfig: WorkloadConfig; + private readonly writer: NdjsonWriter; + private readonly commitId: string | null; + private readonly log: Logger; + + constructor(options: BenchmarkEngineOptions) { + this.host = options.host; + this.port = options.port; + this.driverConfig = options.driverConfig; + this.workloadConfig = options.workloadConfig; + this.writer = new NdjsonWriter(options.metricsPath); + this.commitId = options.commitId ?? null; + this.log = options.logger ?? consoleLogger; + } + + async run(): Promise { + this.log.info(`Starting benchmark: ${this.workloadConfig.name()}`); + this.log.info(`Driver: ${this.driverConfig.driverId}, Server mode: ${this.driverConfig.mode}`); + this.log.info('Concurrency: event-loop task-per-connection (one client per connection)'); + this.log.info(`Server: ${this.host}:${this.port}`); + + await this.setupMetadata(); + + for (const phase of this.workloadConfig.phases) { + await this.executePhase(phase); + } + + this.log.info('Benchmark completed'); + } + + /** Best-effort: a version lookup failure must not fail the benchmark. */ + private async setupMetadata(): Promise { + try { + const sample = await BenchmarkClientFactory.createAndConnect( + this.host, + this.port, + this.driverConfig, + ); + const version = sample.driverVersion(); + this.writer.setMetadata({ + commitId: this.commitId, + driverId: this.driverConfig.driverId, + primaryDriverVersion: version, + secondaryDriverId: this.driverConfig.secondaryDriverId(), + secondaryDriverVersion: sample.secondaryDriverVersion?.() ?? null, + }); + this.log.info( + `Metadata: commit=${this.commitId ?? 'N/A'}, driver=${this.driverConfig.driverId}, version=${version}`, + ); + await sample.close(); + } catch (error) { + this.log.warn(`Failed to get driver version for metadata: ${(error as Error).message}`); + this.writer.setMetadata({ + commitId: this.commitId, + driverId: this.driverConfig.driverId, + primaryDriverVersion: 'unknown', + secondaryDriverId: this.driverConfig.secondaryDriverId(), + secondaryDriverVersion: null, + }); + } + } + + private async executePhase(phase: PhaseConfig): Promise { + this.log.info(`=== Starting phase: ${phase.id} (${phase.description ?? ''}) ===`); + + const collector = new MetricsCollector(); + const clients = await this.createClients(phase); + const commands = CommandFactory.createAll(phase.commands); + const rateLimiter = phase.hasRpsLimit() ? RateLimiter.create(phase.rpsLimit) : null; + + let status: string; + try { + if (phase.warmupRequests > 0) await this.warmup(clients, phase.warmupRequests); + + collector.start(); + status = await this.runWorkload(phase, clients, commands, rateLimiter, collector); + collector.stop(); + } finally { + await this.closeClients(clients); + } + + this.writer.writePhaseResults({ + phaseId: phase.id, + status, + connections: phase.connections, + collector, + }); + this.logPhaseSummary(phase, collector, status); + } + + private async createClients(phase: PhaseConfig): Promise { + this.log.info(`Creating ${phase.connections} connections...`); + const cpsLimiter = phase.hasCpsLimit() ? RateLimiter.create(phase.cpsLimit) : null; + + const clients: BenchmarkClient[] = []; + for (let i = 0; i < phase.connections; i++) { + if (cpsLimiter !== null) await cpsLimiter.acquire(); + clients.push( + await BenchmarkClientFactory.createAndConnect(this.host, this.port, this.driverConfig), + ); + if ((i + 1) % CONNECTION_LOG_INTERVAL === 0) { + this.log.info(`Created ${i + 1}/${phase.connections} connections`); + } + } + this.log.info(`All ${clients.length} connections established`); + return clients; + } + + /** + * Send warmup PINGs on every client, failing fast if any of them errors. + * + * A dead or misconfigured server would otherwise produce a whole phase of + * nothing but errors, which is far harder to diagnose than an upfront throw. + */ + private async warmup(clients: BenchmarkClient[], warmupRequests: number): Promise { + this.log.info(`Warmup: ${warmupRequests} PING(s) per client...`); + // Warmup mode lets the recording driver suppress simulated errors, so an + // error_rate workload is not aborted by the very errors it is measuring. + for (const client of clients) client.setWarmupMode?.(true); + try { + await Promise.all( + clients.map(async (client) => { + for (let i = 0; i < warmupRequests; i++) { + const result = await client.ping(); + if (result.error !== undefined) { + throw new Error(`Warmup PING failed: ${result.error.message}`); + } + } + }), + ); + } finally { + for (const client of clients) client.setWarmupMode?.(false); + } + this.log.info('Warmup completed'); + } + + private async runWorkload( + phase: PhaseConfig, + clients: BenchmarkClient[], + commands: Command[], + rateLimiter: RateLimiter | null, + collector: MetricsCollector, + ): Promise { + const { completion } = phase; + const pipelineDepth = phase.effectivePipelineDepth(); + const seedBase = phase.keyspace.seedValue(); + // Shared across workers for sequential_int, so they collectively emit + // 0, 1, 2, ... exactly as the Java reference does. + const sharedCounter = new Counter(); + + const budget = completion.isRequestBased() ? new RequestBudget(completion.totalRequests()) : null; + const deadlineMs = completion.isDurationBased() + ? Date.now() + completion.durationSeconds() * 1000 + : null; + + const keepGoing = (): boolean => { + if (deadlineMs !== null && Date.now() >= deadlineMs) return false; + if (budget !== null) return budget.claim(); + return true; + }; + + this.log.info( + `Starting ${clients.length} workers (pipeline_depth=${pipelineDepth})...`, + ); + + const progressTimer = setInterval(() => { + this.logProgress(collector, completion.isRequestBased() ? completion.totalRequests() : null); + }, PROGRESS_LOG_INTERVAL_MS); + // Do not let the interval hold the event loop open past the phase. + progressTimer.unref(); + + try { + await Promise.all( + clients.map((client, index) => { + const keyGen = KeyGenerator.createWithSeed(phase.keyspace, seedBase + index, sharedCounter); + const selector = new CommandSelector(commands); + return pipelineDepth <= 1 + ? this.runSyncLoop(client, selector, keyGen, rateLimiter, collector, keepGoing) + : this.runPipelinedLoop( + client, + selector, + keyGen, + rateLimiter, + collector, + keepGoing, + pipelineDepth, + ); + }), + ); + this.log.info(`All operations completed (${collector.totalRequests} total requests)`); + return 'COMPLETED'; + } catch (error) { + this.log.error(`Error during workload execution: ${(error as Error).message}`); + return 'ERROR'; + } finally { + clearInterval(progressTimer); + } + } + + /** One in-flight request per connection (pipeline_depth <= 1). */ + private async runSyncLoop( + client: BenchmarkClient, + selector: CommandSelector, + keyGen: KeyGenerator, + rateLimiter: RateLimiter | null, + collector: MetricsCollector, + keepGoing: () => boolean, + ): Promise { + while (keepGoing()) { + if (rateLimiter !== null) await rateLimiter.acquire(); + collector.record(await this.issue(client, selector, keyGen)); + } + } + + /** + * Up to `pipelineDepth` in-flight requests per connection. + * + * Each slot runs its own claim/issue/record cycle, so a settled request is + * replaced immediately rather than waiting on a whole batch -- the same + * "refill as they land" behaviour as Java's `anyOf` loop, expressed as N + * independent slot loops sharing the connection. + */ + private async runPipelinedLoop( + client: BenchmarkClient, + selector: CommandSelector, + keyGen: KeyGenerator, + rateLimiter: RateLimiter | null, + collector: MetricsCollector, + keepGoing: () => boolean, + pipelineDepth: number, + ): Promise { + const slot = async (): Promise => { + while (keepGoing()) { + if (rateLimiter !== null) await rateLimiter.acquire(); + collector.record(await this.issue(client, selector, keyGen)); + } + }; + await Promise.all(Array.from({ length: pipelineDepth }, slot)); + } + + /** + * Select and run one command. + * + * Drivers already convert a rejection into a failed TimedResult, so a throw + * here means an engine-level bug rather than a server error; record it as a + * failed request and keep the phase running. + */ + private async issue( + client: BenchmarkClient, + selector: CommandSelector, + keyGen: KeyGenerator, + ): Promise { + const command = selector.select(); + // Only advance the key sequence for commands that consume a key, so PING + // does not silently shift the sequence other engines produce. + const key = command.usesKey ? keyGen.nextKey() : ''; + try { + return await command.execute(client, key); + } catch (error) { + return { + commandName: command.name, + latencyMicros: 0, + success: false, + errorMessage: (error as Error).message, + }; + } + } + + private async closeClients(clients: BenchmarkClient[]): Promise { + this.log.info(`Closing ${clients.length} connections...`); + for (const client of clients) { + try { + await client.close(); + } catch (error) { + this.log.warn(`Error closing client: ${(error as Error).message}`); + } + } + } + + private logProgress(collector: MetricsCollector, target: number | null): void { + const elapsedMs = Date.now() - collector.startTime(); + if (elapsedMs <= 0) return; + const current = collector.totalRequests; + const rate = Math.round((current * 1000) / elapsedMs); + if (target !== null) { + const percent = ((current * 100) / target).toFixed(1); + this.log.info(`Progress: ${current}/${target} requests (${percent}%) - ${rate} req/s`); + } else { + this.log.info(`Progress: ${current} requests - ${rate} req/s`); + } + } + + private logPhaseSummary(phase: PhaseConfig, collector: MetricsCollector, status: string): void { + const durationSeconds = collector.durationMillis() / 1000; + const rps = durationSeconds > 0 ? Math.round(collector.totalRequests / durationSeconds) : 0; + this.log.info(`=== Phase ${phase.id} completed: ${status} ===`); + this.log.info( + ` Duration: ${durationSeconds.toFixed(1)}s | Requests: ${collector.totalRequests} | ` + + `Errors: ${collector.totalErrors} | RPS: ${rps}`, + ); + for (const [name, m] of collector.commandMetrics) { + if (m.count() === 0 && m.errors === 0) continue; + this.log.info( + ` ${name}: ${m.requests} req (${m.errors} err) | ` + + `p50=${m.percentile(50)}us p95=${m.percentile(95)}us p99=${m.percentile(99)}us ` + + `p99.9=${m.percentile(99.9)}us | min=${m.min()}us max=${m.max()}us`, + ); + } + } +} diff --git a/node/src/engine/commandSelector.ts b/node/src/engine/commandSelector.ts new file mode 100644 index 0000000..d8e2731 --- /dev/null +++ b/node/src/engine/commandSelector.ts @@ -0,0 +1,43 @@ +/** + * Weighted command selection. + * + * Uses normalized cumulative weights, matching Java's inline CommandSelector + * (BenchmarkEngine.java:520-545) and the Python engine. Selection intentionally + * uses the platform RNG (`Math.random`), not the Java LCG: only *key* generation + * must be cross-engine deterministic. Each worker owns its own selector, so + * there is no shared state. + */ + +import type { Command } from '../command/command.js'; + +export class CommandSelector { + private readonly commands: Command[]; + private readonly cumulativeWeights: number[]; + + constructor(commands: Command[]) { + if (commands.length === 0) throw new Error('CommandSelector requires at least one command'); + this.commands = commands; + this.cumulativeWeights = CommandSelector.buildCumulativeWeights(commands); + } + + select(): Command { + const r = Math.random(); + for (let i = 0; i < this.cumulativeWeights.length; i++) { + if (r <= this.cumulativeWeights[i]!) return this.commands[i]!; + } + return this.commands[this.commands.length - 1]!; + } + + private static buildCumulativeWeights(commands: Command[]): number[] { + let totalWeight = commands.reduce((sum, command) => sum + command.weight, 0); + if (totalWeight === 0) totalWeight = 1.0; + + const cumulative: number[] = []; + let running = 0; + for (const command of commands) { + running += command.weight / totalWeight; + cumulative.push(running); + } + return cumulative; + } +} diff --git a/node/src/engine/javaRandom.ts b/node/src/engine/javaRandom.ts new file mode 100644 index 0000000..ce5efba --- /dev/null +++ b/node/src/engine/javaRandom.ts @@ -0,0 +1,68 @@ +/** + * Faithful port of `java.util.Random` (48-bit LCG). + * + * This guarantees identical `uniform_rand` key sequences across the Java + * (reference), Ruby, C#, Python and Node engines. Java is the canonical source, + * so this port reproduces `nextInt(bound)` exactly -- including the 32-bit + * signed-overflow rejection in the general case, which is what avoids modulo + * bias. (The Ruby port omits that rejection because Ruby integers are + * arbitrary-precision; this port emulates the int32 wraparound so it matches + * the Java reference rather than the Ruby approximation.) + * + * BigInt is mandatory for the state, not a stylistic choice: `seed * MULTIPLIER` + * reaches ~2^83, far past the 2^53 that a JS `number` can represent exactly, so + * a `number`-based port silently diverges from Java after the first step. + * + * @see https://docs.oracle.com/javase/8/docs/api/java/util/Random.html + */ + +const MULTIPLIER = 0x5deece66dn; +const ADDEND = 0xbn; +const MASK = (1n << 48n) - 1n; + +/** Interpret the low 32 bits of `value` as a signed 32-bit integer. */ +export function toInt32(value: number): number { + return value | 0; +} + +export class JavaRandom { + private seed: bigint; + + constructor(seed: number | bigint) { + this.seed = JavaRandom.initialScramble(seed); + } + + setSeed(seed: number | bigint): void { + this.seed = JavaRandom.initialScramble(seed); + } + + /** Return a random int in [0, bound) matching Java's nextInt(int). */ + nextInt(bound: number): number { + if (!Number.isInteger(bound) || bound <= 0) { + throw new Error(`bound must be a positive integer (got ${bound})`); + } + + // Power-of-two fast path (matches Java exactly). + if ((bound & -bound) === bound) { + return Number((BigInt(bound) * BigInt(this.next(31))) >> 31n); + } + + // General case: rejection sampling to avoid modulo bias. The rejection + // condition relies on 32-bit signed overflow, which `| 0` emulates. + for (;;) { + const bits = this.next(31); + const val = bits % bound; + if (toInt32(bits - val + (bound - 1)) >= 0) return val; + } + } + + private static initialScramble(seed: number | bigint): bigint { + return (BigInt(seed) ^ MULTIPLIER) & MASK; + } + + /** Java's `protected int next(int bits)`. Exposed for parity tests. */ + next(bits: number): number { + this.seed = (this.seed * MULTIPLIER + ADDEND) & MASK; + return Number(this.seed >> BigInt(48 - bits)); + } +} diff --git a/node/src/engine/keyGenerator.ts b/node/src/engine/keyGenerator.ts new file mode 100644 index 0000000..c44a87c --- /dev/null +++ b/node/src/engine/keyGenerator.ts @@ -0,0 +1,93 @@ +/** + * Key generator producing sequences identical to the other engines. + * + * - `sequential_int`: keys 0, 1, 2, ... N-1, wrapping around. + * - `uniform_rand`: Java-LCG random keys (see ./javaRandom.ts). + * + * Key formatting matches Java's `String.format("%0Nd", index)`: the numeric part + * is zero-padded to `max(keySizeBytes - prefix.length, 1)` digits. With the + * reference configs (`key_prefix: "bench:"`, `key_size_bytes: 16`) that yields + * `bench:` + 10 digits, e.g. `bench:0000000042`. + * + * Cross-worker semantics follow the Java reference: + * - `sequential_int` uses a counter SHARED across all workers in a phase, so the + * workers collectively emit 0, 1, 2, ... (this is what populates the whole + * keyspace during a WARMUP/populate phase). Pass a shared Counter. + * - `uniform_rand` uses a per-worker RNG seeded `baseSeed + workerIndex`. + */ + +import type { KeyspaceConfig } from '../config/keyspaceConfig.js'; +import { JavaRandom } from './javaRandom.js'; + +/** + * A monotonic 0-based counter, shared across a phase's workers. + * + * Safe to share across concurrent workers: `nextValue` performs its + * read-increment with no `await` in between, so it is atomic on the single + * JS event-loop thread. + */ +export class Counter { + private value: number; + + constructor(start = 0) { + this.value = start; + } + + nextValue(): number { + return this.value++; + } + + reset(): void { + this.value = 0; + } +} + +export class KeyGenerator { + private readonly config: KeyspaceConfig; + private readonly keyPrefix: string; + private readonly keysCount: number; + private readonly paddingWidth: number; + private readonly seed: number; + private readonly sequentialCounter: Counter; + private readonly random: JavaRandom; + + constructor(config: KeyspaceConfig, seedOverride?: number, sequentialCounter?: Counter) { + this.config = config; + this.keyPrefix = config.keyPrefix; + this.keysCount = config.keysCount; + this.paddingWidth = Math.max(config.keySizeBytes - config.keyPrefix.length, 1); + this.seed = seedOverride ?? config.seedValue(); + this.sequentialCounter = sequentialCounter ?? new Counter(); + this.random = new JavaRandom(this.seed); + } + + static create(config: KeyspaceConfig): KeyGenerator { + return new KeyGenerator(config); + } + + /** Per-worker generator with a unique seed and an optional shared counter. */ + static createWithSeed( + config: KeyspaceConfig, + seed: number, + sequentialCounter?: Counter, + ): KeyGenerator { + return new KeyGenerator(config, seed, sequentialCounter); + } + + nextKey(): string { + const rawIndex = this.config.isSequentialInt() + ? this.sequentialCounter.nextValue() + : this.random.nextInt(this.keysCount); + + return this.formatKey(rawIndex % this.keysCount); + } + + reset(): void { + this.sequentialCounter.reset(); + this.random.setSeed(this.seed); + } + + private formatKey(keyIndex: number): string { + return this.keyPrefix + String(keyIndex).padStart(this.paddingWidth, '0'); + } +} diff --git a/node/src/engine/rateLimiter.ts b/node/src/engine/rateLimiter.ts new file mode 100644 index 0000000..d0deef9 --- /dev/null +++ b/node/src/engine/rateLimiter.ts @@ -0,0 +1,66 @@ +/** + * Leaky-bucket rate limiter. + * + * Enforces a constant rate with no burst (evenly-spaced operations), matching + * the Java reference's interval math exactly: + * `intervalNanos = 1_000_000_000 / ratePerSecond` (RateLimiter.java:28). Despite + * what docs/ARCHITECTURE.md says about a "token bucket", every engine actually + * implements this leaky bucket, so this follows the code rather than the doc. + * + * A single limiter is shared across all of a phase's workers. Because JS is + * single-threaded, the check-and-advance of `nextAllowedNanos` has no `await` + * between read and write, so it is atomic -- no CAS loop is needed (Java needs + * one only because its issuers are real threads). + * + * Node-specific wrinkle: `setTimeout` clamps to ~1ms, so at high rates (a 100k + * rps limit is a 10us interval) a timer-based wait would undershoot the target + * badly. Sub-millisecond waits therefore yield via `setImmediate`, which returns + * to the event loop -- letting other connections progress -- without sleeping a + * whole millisecond. + */ + +const NANOS_PER_SECOND = 1_000_000_000n; +const NANOS_PER_MILLI = 1_000_000n; + +/** Yield to the event loop without a clamped timer delay. */ +function yieldToEventLoop(): Promise { + return new Promise((resolve) => setImmediate(resolve)); +} + +function sleepMillis(millis: number): Promise { + return new Promise((resolve) => setTimeout(resolve, millis)); +} + +export class RateLimiter { + readonly ratePerSecond: number; + private readonly intervalNanos: bigint; + private nextAllowedNanos: bigint; + + private constructor(ratePerSecond: number) { + this.ratePerSecond = ratePerSecond; + this.intervalNanos = NANOS_PER_SECOND / BigInt(ratePerSecond); + // The first operation is allowed immediately. + this.nextAllowedNanos = process.hrtime.bigint(); + } + + /** Return a limiter, or null for unlimited (rate <= 0). */ + static create(ratePerSecond: number): RateLimiter | null { + return ratePerSecond > 0 ? new RateLimiter(ratePerSecond) : null; + } + + async acquire(): Promise { + for (;;) { + const now = process.hrtime.bigint(); + if (now >= this.nextAllowedNanos) { + this.nextAllowedNanos += this.intervalNanos; + return; + } + const waitNanos = this.nextAllowedNanos - now; + if (waitNanos >= NANOS_PER_MILLI) { + await sleepMillis(Number(waitNanos / NANOS_PER_MILLI)); + } else { + await yieldToEventLoop(); + } + } + } +} diff --git a/node/src/metrics/collector.ts b/node/src/metrics/collector.ts new file mode 100644 index 0000000..ecca794 --- /dev/null +++ b/node/src/metrics/collector.ts @@ -0,0 +1,111 @@ +/** + * Latency metrics collection. + * + * Single-event-loop design: no locks are needed because `record` runs to + * completion without awaiting, so concurrent workers never interleave inside it. + * Latencies are clamped to 600s before recording, and errors are counted but not + * recorded into the histogram -- matching the other engines. + */ + +import type { CommandResult } from '../command/command.js'; +import { HIGHEST_TRACKABLE_VALUE, newHistogram, type Histogram } from './hdrHistogram.js'; + +export class CommandMetrics { + readonly commandName: string; + requests = 0; + errors = 0; + /** + * Created eagerly (like the Java reference) so the NDJSON `hdr` block and + * summary are always present, even for a command that only ever errors -- an + * empty histogram reports count 0 and zero percentiles. + */ + readonly histogram: Histogram = newHistogram(); + + constructor(commandName: string) { + this.commandName = commandName; + } + + record(result: CommandResult): void { + this.requests += 1; + if (result.success) { + this.histogram.recordValue(Math.min(result.latencyMicros, HIGHEST_TRACKABLE_VALUE)); + } else { + this.errors += 1; + } + } + + count(): number { + return this.histogram.totalCount; + } + + /** + * Lowest recorded latency, matching Java's `Histogram.getMinValue()`. + * + * Deliberately NOT `minNonZeroValue`: that skips a legitimately recorded 0us + * sample, and on an empty histogram it returns Number.MAX_SAFE_INTEGER, which + * would land in the NDJSON as a nonsense min. `getValueAtPercentile(0)` returns + * 0 when empty and Java's `getMinValue()` otherwise -- verified equal for + * 1us..599s (and it is what the Ruby encoder uses). + */ + min(): number { + return this.histogram.getValueAtPercentile(0); + } + + /** + * Highest recorded latency, matching Java's `Histogram.getMaxValue()`. + * + * Deliberately NOT `maxValue`: hdr-histogram-js returns the raw recorded + * sample there, while Java returns the *bucket's* highest equivalent value. At + * 3 significant figures those diverge above ~1000us -- recording 50000us gives + * 50000 in JS but 50015 in Java -- which would make summary.max quietly + * incomparable across engines. `getValueAtPercentile(100)` is Java's value. + */ + max(): number { + return this.histogram.getValueAtPercentile(100); + } + + percentile(pct: number): number { + return this.histogram.getValueAtPercentile(pct); + } +} + +export class MetricsCollector { + readonly commandMetrics = new Map(); + totalRequests = 0; + totalErrors = 0; + private startTimeMs: number | null = null; + private endTimeMs: number | null = null; + + start(): void { + this.startTimeMs = Date.now(); + } + + stop(): void { + this.endTimeMs = Date.now(); + } + + record(result: CommandResult): void { + this.totalRequests += 1; + if (!result.success) this.totalErrors += 1; + + let metrics = this.commandMetrics.get(result.commandName); + if (metrics === undefined) { + metrics = new CommandMetrics(result.commandName); + this.commandMetrics.set(result.commandName, metrics); + } + metrics.record(result); + } + + startTime(): number { + return this.startTimeMs ?? 0; + } + + endTime(): number { + return this.endTimeMs ?? 0; + } + + durationMillis(): number { + if (this.startTimeMs === null || this.endTimeMs === null) return 0; + return this.endTimeMs - this.startTimeMs; + } +} diff --git a/node/src/metrics/hdrHistogram.ts b/node/src/metrics/hdrHistogram.ts new file mode 100644 index 0000000..f6c748b --- /dev/null +++ b/node/src/metrics/hdrHistogram.ts @@ -0,0 +1,48 @@ +/** + * HdrHistogram helpers. + * + * Uses `hdr-histogram-js`, the TypeScript port of HdrHistogram. Its + * `encodeIntoCompressedBase64()` emits the base64-encoded V2 *compressed* + * payload -- the same format Java's `encodeIntoCompressedByteBuffer` + + * `Base64.getEncoder()` produces (NdjsonMetricsWriter.java:164-180) and the same + * the Ruby/Python engines emit -- so payloads are mutually decodable across + * engines for cross-language analysis. (Byte-identity is not guaranteed since + * zlib compression levels may differ, but decodability -- what merge/analysis + * needs -- is.) + * + * The returned string is ALREADY base64 (it starts `HIST`). Never base64 it + * again: double-encoding produces a payload Java and Ruby cannot decode. + * + * Histograms use range (1, 600_000_000, 3): 1 microsecond to 600 seconds at 3 + * significant figures, matching every other engine (Java + * `SynchronizedHistogram(600_000_000, 3)`, C# `LongConcurrentHistogram(1, + * 600_000_000, 3)`, Ruby `HDRHistogram.new(1, 600_000_000, 3)`). + */ + +import * as hdr from 'hdr-histogram-js'; + +export const LOWEST_TRACKABLE_VALUE = 1; +export const HIGHEST_TRACKABLE_VALUE = 600_000_000; // 600 seconds in microseconds +export const SIGNIFICANT_FIGURES = 3; + +export type Histogram = hdr.Histogram; + +export function newHistogram(): Histogram { + return hdr.build({ + bitBucketSize: 64, + autoResize: false, + lowestDiscernibleValue: LOWEST_TRACKABLE_VALUE, + highestTrackableValue: HIGHEST_TRACKABLE_VALUE, + numberOfSignificantValueDigits: SIGNIFICANT_FIGURES, + }); +} + +/** Return the base64 V2-compressed encoding, ready for `payload_b64`. */ +export function encodeBase64(histogram: Histogram): string { + return hdr.encodeIntoCompressedBase64(histogram); +} + +/** Inverse of `encodeBase64`, used by the parity tests. */ +export function decodeBase64(payload: string): Histogram { + return hdr.decodeFromCompressedBase64(payload); +} diff --git a/node/src/metrics/ndjsonWriter.ts b/node/src/metrics/ndjsonWriter.ts new file mode 100644 index 0000000..9aa7d60 --- /dev/null +++ b/node/src/metrics/ndjsonWriter.ts @@ -0,0 +1,122 @@ +/** + * Writes benchmark metrics as NDJSON (newline-delimited JSON), one line per + * phase, so the orchestrator can detect phase completion by watching for new + * lines. + * + * The record shape is byte-for-byte the contract in docs/ARCHITECTURE.md + * "Metrics Output Format" plus the `metadata` block the Java writer emits + * (NdjsonMetricsWriter.java:92-111). The matrix runner treats "exited 0 but wrote + * no record" as a cell failure (run_benchmark_matrix.py:1063-1069), so appending + * really must happen. + */ + +import { appendFileSync, mkdirSync } from 'node:fs'; +import { dirname } from 'node:path'; + +import type { CommandMetrics, MetricsCollector } from './collector.js'; +import { encodeBase64, SIGNIFICANT_FIGURES } from './hdrHistogram.js'; + +export interface Metadata { + commitId?: string | null; + driverId?: string | null; + primaryDriverVersion?: string | null; + secondaryDriverId?: string | null; + secondaryDriverVersion?: string | null; +} + +export class NdjsonWriter { + private readonly outputPath: string; + private metadata: Metadata = {}; + + constructor(outputPath: string) { + this.outputPath = outputPath; + } + + setMetadata(metadata: Metadata): void { + this.metadata = metadata; + } + + writePhaseResults(options: { + phaseId: string; + status: string; + connections: number; + collector: MetricsCollector; + }): void { + const record = this.buildPhaseRecord(options); + const parent = dirname(this.outputPath); + if (parent && parent !== '.') mkdirSync(parent, { recursive: true }); + appendFileSync(this.outputPath, `${JSON.stringify(record)}\n`, 'utf8'); + } + + private buildPhaseRecord(options: { + phaseId: string; + status: string; + connections: number; + collector: MetricsCollector; + }): Record { + const { phaseId, status, connections, collector } = options; + const record: Record = {}; + + const { commitId, driverId, primaryDriverVersion, secondaryDriverId, secondaryDriverVersion } = + this.metadata; + if (commitId != null || driverId != null) { + const metadata: Record = {}; + if (commitId != null) metadata['commit_id'] = commitId; + metadata['timestamp'] = new Date().toISOString(); + if (driverId != null) metadata['driver_id'] = driverId; + if (primaryDriverVersion != null) metadata['primary_driver_version'] = primaryDriverVersion; + if (secondaryDriverId != null) metadata['secondary_driver_id'] = secondaryDriverId; + if (secondaryDriverVersion != null) { + metadata['secondary_driver_version'] = secondaryDriverVersion; + } + record['metadata'] = metadata; + } + + record['phase'] = { + id: phaseId, + status, + start_timestamp: new Date(collector.startTime()).toISOString(), + finish_timestamp: new Date(collector.endTime()).toISOString(), + duration_ms: collector.durationMillis(), + connections, + }; + + record['totals'] = { + requests: collector.totalRequests, + errors: collector.totalErrors, + }; + + const metrics: Record = {}; + for (const [commandName, commandMetrics] of collector.commandMetrics) { + metrics[commandName] = NdjsonWriter.buildCommandRecord(commandMetrics); + } + record['metrics'] = metrics; + + return record; + } + + private static buildCommandRecord(m: CommandMetrics): Record { + return { + requests: m.requests, + errors: m.errors, + latency: { + unit: 'us', + count: m.count(), + summary: { + min: m.min(), + p50: m.percentile(50), + p95: m.percentile(95), + p99: m.percentile(99), + p999: m.percentile(99.9), + max: m.max(), + }, + hdr: { + format: 'hdr', + sigfig: SIGNIFICANT_FIGURES, + // Already base64 -- do NOT encode again (see metrics/hdrHistogram.ts). + payload_b64: encodeBase64(m.histogram), + }, + }, + }; + } +} diff --git a/node/src/version.ts b/node/src/version.ts new file mode 100644 index 0000000..966a9ef --- /dev/null +++ b/node/src/version.ts @@ -0,0 +1,2 @@ +/** Engine version, reported by `--version` and `--info`. */ +export const VERSION = '1.0.0'; diff --git a/node/test/integration/clients.test.ts b/node/test/integration/clients.test.ts new file mode 100644 index 0000000..24548d4 --- /dev/null +++ b/node/test/integration/clients.test.ts @@ -0,0 +1,197 @@ +/** + * Live-server tests for the real drivers. + * + * Skipped unless VALKEY_HOST/VALKEY_PORT point at a running server, matching the + * Java/Ruby/C# integration suites (`make node-integration-test` starts one). + */ + +import assert from 'node:assert/strict'; +import { mkdtempSync, readFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { describe, it } from 'node:test'; + +import { BenchmarkClientFactory } from '../../src/client/factory.js'; +import { ConfigLoader } from '../../src/config/loader.js'; +import { BenchmarkEngine, type Logger } from '../../src/engine/benchmark.js'; +import { decodeBase64 } from '../../src/metrics/hdrHistogram.js'; + +const HOST = process.env['VALKEY_HOST'] ?? 'localhost'; +const PORT = Number(process.env['VALKEY_PORT'] ?? 6379); +const REAL_DRIVERS = ['valkey-glide-node', 'ioredis', 'iovalkey'] as const; + +const silentLogger: Logger = { info: () => {}, warn: () => {}, error: () => {} }; + +/** True when VALKEY_HOST is set, i.e. a server is expected to be reachable. */ +const enabled = process.env['VALKEY_HOST'] !== undefined; +const skip = enabled ? false : 'set VALKEY_HOST to run the live-server tests'; + +function driverConfig(driverId: string, extra: Record = {}) { + return ConfigLoader.parseDriverConfig({ + schema_version: '1.0', + driver_id: driverId, + mode: 'standalone', + specific_driver_config: {}, + ...extra, + }); +} + +for (const driverId of REAL_DRIVERS) { + describe(`${driverId} against a live server`, { skip }, () => { + it('connects, pings, sets, gets and closes', async () => { + const client = await BenchmarkClientFactory.createAndConnect(HOST, PORT, driverConfig(driverId)); + try { + const ping = await client.ping(); + assert.equal(ping.error, undefined); + assert.equal(ping.value, 'PONG'); + assert.ok(ping.latencyMicros >= 0); + + const key = `node-it:${driverId}:${process.pid}`; + const payload = Buffer.from('x'.repeat(64), 'latin1'); + + const set = await client.set(key, payload); + assert.equal(set.error, undefined); + assert.equal(set.value, 'OK'); + + const get = await client.get(key); + assert.equal(get.error, undefined); + // Every driver must return the same decoded shape, or the engines are + // charged for different work: a string of the payload's length. + assert.equal(typeof get.value, 'string'); + assert.equal(get.value, payload.toString('latin1')); + } finally { + await client.close(); + } + }); + + it('returns null for a missing key rather than erroring', async () => { + const client = await BenchmarkClientFactory.createAndConnect(HOST, PORT, driverConfig(driverId)); + try { + const get = await client.get(`node-it:absent:${process.pid}:${Math.random()}`); + assert.equal(get.error, undefined); + assert.equal(get.value, null); + } finally { + await client.close(); + } + }); + + it('reports a real driver version, not "unknown"', async () => { + const client = await BenchmarkClientFactory.create(driverId); + assert.match(client.driverVersion(), /^\d+\.\d+\.\d+/); + }); + + it('applies command_timeout_ms without breaking normal commands', async () => { + const client = await BenchmarkClientFactory.createAndConnect( + HOST, + PORT, + driverConfig(driverId, { command_timeout_ms: 5000 }), + ); + try { + assert.equal((await client.ping()).error, undefined); + } finally { + await client.close(); + } + }); + + it('runs a short workload and writes decodable metrics', async () => { + const metricsPath = join(mkdtempSync(join(tmpdir(), 'resp-bench-node-')), 'metrics.ndjson'); + await new BenchmarkEngine({ + host: HOST, + port: PORT, + driverConfig: driverConfig(driverId), + workloadConfig: ConfigLoader.parseWorkloadConfig({ + benchmark_profile: { name: `${driverId} smoke` }, + phases: [ + { + id: 'STEADY', + connections: 2, + warmup_requests: 1, + completion: { type: 'requests', requests: 200 }, + keyspace: { keys_count: 100, key_prefix: 'node-it:', key_size_bytes: 16 }, + commands: [ + { command: 'set', weight: 0.5, data_size_bytes: 64 }, + { command: 'get', weight: 0.5 }, + ], + }, + ], + }), + metricsPath, + commitId: 'integration-test', + logger: silentLogger, + }).run(); + + const record = JSON.parse(readFileSync(metricsPath, 'utf8').trimEnd().split('\n')[0]!); + assert.equal(record.phase.status, 'COMPLETED'); + assert.equal(record.totals.requests, 200); + assert.equal(record.totals.errors, 0, 'a healthy server should produce no errors'); + assert.equal(record.metadata.driver_id, driverId); + assert.match(record.metadata.primary_driver_version, /^\d+\.\d+\.\d+/); + for (const name of Object.keys(record.metrics)) { + const latency = record.metrics[name].latency; + assert.ok(latency.hdr.payload_b64.startsWith('HIST')); + assert.equal(decodeBase64(latency.hdr.payload_b64).totalCount, latency.count); + assert.ok(latency.summary.p50 >= 0); + assert.ok(latency.summary.max >= latency.summary.p50); + } + }); + + it('runs a pipelined workload', async () => { + const metricsPath = join(mkdtempSync(join(tmpdir(), 'resp-bench-node-')), 'metrics.ndjson'); + await new BenchmarkEngine({ + host: HOST, + port: PORT, + driverConfig: driverConfig(driverId), + workloadConfig: ConfigLoader.parseWorkloadConfig({ + phases: [ + { + id: 'PIPELINED', + connections: 2, + pipeline_depth: 8, + warmup_requests: 1, + completion: { type: 'requests', requests: 200 }, + keyspace: { keys_count: 100, key_prefix: 'node-it:', key_size_bytes: 16 }, + commands: [{ command: 'get', weight: 1.0 }], + }, + ], + }), + metricsPath, + logger: silentLogger, + }).run(); + + const record = JSON.parse(readFileSync(metricsPath, 'utf8').trimEnd().split('\n')[0]!); + assert.equal(record.phase.status, 'COMPLETED'); + // 2 connections x depth 8 = 16 in-flight slots on one shared budget of 200. + assert.equal(record.totals.requests, 200); + assert.equal(record.totals.errors, 0); + }); + }); +} + +describe('unreachable server', { skip }, () => { + it('fails fast instead of recording a phase of pure errors', async () => { + // Port 1 is reserved and never listening. A dead server must abort the run + // with a clear error, not silently produce 100% error metrics. + await assert.rejects( + () => + new BenchmarkEngine({ + host: '127.0.0.1', + port: 1, + driverConfig: driverConfig('ioredis'), + workloadConfig: ConfigLoader.parseWorkloadConfig({ + phases: [ + { + id: 'STEADY', + connections: 1, + warmup_requests: 1, + completion: { type: 'requests', requests: 10 }, + keyspace: { keys_count: 10 }, + commands: [{ command: 'get', weight: 1.0 }], + }, + ], + }), + metricsPath: join(mkdtempSync(join(tmpdir(), 'resp-bench-node-')), 'metrics.ndjson'), + logger: silentLogger, + }).run(), + ); + }); +}); diff --git a/node/test/integration/recordingWorkload.test.ts b/node/test/integration/recordingWorkload.test.ts new file mode 100644 index 0000000..00a17ca --- /dev/null +++ b/node/test/integration/recordingWorkload.test.ts @@ -0,0 +1,297 @@ +/** + * Full-engine tests against the `recording` driver — no server required. + * + * These are the tests that catch engine-level regressions the unit tests cannot: + * the shared request budget, warmup fail-fast, pipelining, rate limiting, and the + * NDJSON a real run produces. + */ + +import assert from 'node:assert/strict'; +import { mkdtempSync, readFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { describe, it } from 'node:test'; + +import { ConfigLoader } from '../../src/config/loader.js'; +import { BenchmarkEngine, type Logger } from '../../src/engine/benchmark.js'; +import { decodeBase64 } from '../../src/metrics/hdrHistogram.js'; + +const silentLogger: Logger = { info: () => {}, warn: () => {}, error: () => {} }; + +function tempMetricsPath(): string { + return join(mkdtempSync(join(tmpdir(), 'resp-bench-node-')), 'metrics.ndjson'); +} + +function readRecords(path: string): Array> { + return readFileSync(path, 'utf8') + .trimEnd() + .split('\n') + .map((line) => JSON.parse(line)); +} + +function driverConfig(specific: Record = {}) { + return ConfigLoader.parseDriverConfig({ + schema_version: '1.0', + driver_id: 'recording', + mode: 'standalone', + specific_driver_config: specific, + }); +} + +function workload(phases: unknown[]) { + return ConfigLoader.parseWorkloadConfig({ + schema_version: '1.0', + benchmark_profile: { name: 'Node engine test' }, + phases, + }); +} + +const STEADY_PHASE = { + id: 'STEADY', + description: 'short steady phase', + connections: 4, + warmup_requests: 1, + completion: { type: 'requests', requests: 200 }, + keyspace: { keys_count: 100, key_prefix: 'e2e:', key_size_bytes: 16 }, + commands: [ + { command: 'set', weight: 0.5, data_size_bytes: 32 }, + { command: 'get', weight: 0.5 }, + ], +}; + +async function run(options: { + phases: unknown[]; + specific?: Record; + commitId?: string; +}): Promise<{ path: string; records: Array> }> { + const path = tempMetricsPath(); + await new BenchmarkEngine({ + host: 'localhost', + port: 6379, + driverConfig: driverConfig(options.specific), + workloadConfig: workload(options.phases), + metricsPath: path, + commitId: options.commitId ?? 'test-commit', + logger: silentLogger, + }).run(); + return { path, records: readRecords(path) }; +} + +describe('recording-driver workload', () => { + it('runs a phase end to end and writes valid NDJSON', async () => { + const { records } = await run({ phases: [STEADY_PHASE] }); + + assert.equal(records.length, 1); + const record = records[0]!; + assert.equal(record['phase'].id, 'STEADY'); + assert.equal(record['phase'].status, 'COMPLETED'); + assert.equal(record['phase'].connections, 4); + assert.equal(record['totals'].requests, 200); + assert.equal(record['totals'].errors, 0); + assert.equal(record['metadata'].commit_id, 'test-commit'); + assert.equal(record['metadata'].driver_id, 'recording'); + assert.equal(record['metadata'].primary_driver_version, '1.0.0'); + + // Both commands were exercised and their counts add up to the total. + const commands = Object.keys(record['metrics']).sort(); + assert.deepEqual(commands, ['GET', 'SET']); + const sum = commands.reduce((acc, name) => acc + record['metrics'][name].requests, 0); + assert.equal(sum, 200); + + for (const name of commands) { + const payload = record['metrics'][name].latency.hdr.payload_b64; + assert.ok(payload.startsWith('HIST')); + assert.equal(decodeBase64(payload).totalCount, record['metrics'][name].latency.count); + } + }); + + it('honours the shared request budget exactly, not a per-worker split', async () => { + // 7 requests across 4 connections does not divide evenly. A pre-split budget + // would round to 4 or 8; a shared budget lands on exactly 7. + const { records } = await run({ + phases: [{ ...STEADY_PHASE, connections: 4, completion: { type: 'requests', requests: 7 } }], + }); + assert.equal(records[0]!['totals'].requests, 7); + }); + + it('honours the budget when connections exceed the request count', async () => { + const { records } = await run({ + phases: [{ ...STEADY_PHASE, connections: 8, completion: { type: 'requests', requests: 3 } }], + }); + assert.equal(records[0]!['totals'].requests, 3); + }); + + it('runs each phase in order, appending one record per phase', async () => { + const { records } = await run({ + phases: [ + { + ...STEADY_PHASE, + id: 'WARMUP', + connections: 2, + completion: { type: 'requests', requests: 20 }, + commands: [{ command: 'set', weight: 1.0, data_size_bytes: 16 }], + }, + { ...STEADY_PHASE, id: 'STEADY', completion: { type: 'requests', requests: 30 } }, + ], + }); + + assert.deepEqual( + records.map((r) => r['phase'].id), + ['WARMUP', 'STEADY'], + ); + assert.equal(records[0]!['totals'].requests, 20); + assert.equal(records[1]!['totals'].requests, 30); + assert.deepEqual(Object.keys(records[0]!['metrics']), ['SET']); + }); + + it('stops a duration-based phase at the deadline', async () => { + const started = Date.now(); + const { records } = await run({ + phases: [{ ...STEADY_PHASE, connections: 2, completion: { type: 'duration', seconds: 1 } }], + }); + const elapsed = Date.now() - started; + + assert.equal(records[0]!['phase'].status, 'COMPLETED'); + assert.ok(records[0]!['totals'].requests > 0, 'a duration phase should do some work'); + assert.ok(elapsed >= 900, `finished suspiciously early: ${elapsed}ms`); + assert.ok(elapsed < 6000, `overran the 1s deadline: ${elapsed}ms`); + }); + + it('keeps pipelined runs on the same shared budget', async () => { + const { records } = await run({ + phases: [ + { + ...STEADY_PHASE, + connections: 3, + pipeline_depth: 4, + completion: { type: 'requests', requests: 50 }, + }, + ], + }); + // 3 connections x depth 4 = 12 in-flight slots all drawing on one budget of + // 50; an over-issuing pipeline would show more than 50 requests. + assert.equal(records[0]!['totals'].requests, 50); + }); + + it('surfaces injected errors as errors, not as latency samples', async () => { + const { records } = await run({ + phases: [{ ...STEADY_PHASE, connections: 2, completion: { type: 'requests', requests: 100 } }], + specific: { error_rate: 1.0, error_message: 'Simulated failure' }, + }); + + const record = records[0]!; + assert.equal(record['totals'].requests, 100); + assert.equal(record['totals'].errors, 100); + for (const name of Object.keys(record['metrics'])) { + const metrics = record['metrics'][name]; + assert.equal(metrics.errors, metrics.requests); + // Failed requests are counted but never recorded into the histogram. + assert.equal(metrics.latency.count, 0); + assert.ok(metrics.latency.hdr.payload_b64.startsWith('HIST')); + } + }); + + it('records a partial error rate on both sides of the split', async () => { + const { records } = await run({ + phases: [{ ...STEADY_PHASE, connections: 2, completion: { type: 'requests', requests: 400 } }], + specific: { error_rate: 0.5 }, + }); + const { requests, errors } = records[0]!['totals']; + assert.equal(requests, 400); + assert.ok(errors > 100 && errors < 300, `error count ${errors} not near half of 400`); + }); + + it('does not count warmup requests toward the phase totals', async () => { + // Warmup runs 3 PINGs on each of 2 connections. If those leaked into the + // measured metrics we would see 6 extra requests and a PING command key. + const { records } = await run({ + phases: [ + { + ...STEADY_PHASE, + connections: 2, + warmup_requests: 3, + completion: { type: 'requests', requests: 10 }, + }, + ], + }); + assert.equal(records[0]!['totals'].requests, 10); + assert.equal('PING' in records[0]!['metrics'], false); + }); + + it('suppresses injected errors during warmup so the phase still runs', async () => { + // Warmup's fail-fast exists to catch an unreachable server. An error_rate + // workload is deliberately measuring errors, so warmup must not abort on + // them -- Java does the same via setWarmupMode. + const { records } = await run({ + phases: [ + { + ...STEADY_PHASE, + connections: 2, + warmup_requests: 2, + completion: { type: 'requests', requests: 20 }, + }, + ], + specific: { error_rate: 1.0, error_message: 'Simulated failure' }, + }); + assert.equal(records[0]!['phase'].status, 'COMPLETED'); + assert.equal(records[0]!['totals'].requests, 20); + assert.equal(records[0]!['totals'].errors, 20); + }); + + it('applies an rps_limit to the whole phase', async () => { + const started = Date.now(); + const { records } = await run({ + phases: [ + { + ...STEADY_PHASE, + connections: 4, + rps_limit: 100, + completion: { type: 'requests', requests: 50 }, + }, + ], + }); + const elapsed = Date.now() - started; + + assert.equal(records[0]!['totals'].requests, 50); + // 50 requests at 100/s cannot finish faster than ~0.49s however many + // connections are issuing them. + assert.ok(elapsed >= 400, `rps_limit was not enforced: ${elapsed}ms for 50 requests at 100/s`); + }); + + it('gates connection setup with a cps_limit', async () => { + const started = Date.now(); + await run({ + phases: [ + { + ...STEADY_PHASE, + connections: 5, + cps_limit: 20, + completion: { type: 'requests', requests: 5 }, + }, + ], + }); + const elapsed = Date.now() - started; + // 5 connections at 20/s means the last one opens ~200ms in. + assert.ok(elapsed >= 150, `cps_limit was not enforced: ${elapsed}ms to open 5 connections`); + }); + + it('generates keys inside the configured keyspace', async () => { + const { records } = await run({ + phases: [ + { + ...STEADY_PHASE, + connections: 2, + completion: { type: 'requests', requests: 40 }, + keyspace: { + keys_count: 10, + key_prefix: 'e2e:', + key_size_bytes: 8, + generation_alg: 'uniform_rand', + seed: 12345, + }, + }, + ], + }); + assert.equal(records[0]!['totals'].requests, 40); + }); +}); diff --git a/node/test/unit/collector.test.ts b/node/test/unit/collector.test.ts new file mode 100644 index 0000000..da926e9 --- /dev/null +++ b/node/test/unit/collector.test.ts @@ -0,0 +1,97 @@ +import assert from 'node:assert/strict'; +import { describe, it } from 'node:test'; + +import { CommandMetrics, MetricsCollector } from '../../src/metrics/collector.js'; +import { HIGHEST_TRACKABLE_VALUE } from '../../src/metrics/hdrHistogram.js'; + +describe('CommandMetrics', () => { + it('counts requests and records only successes into the histogram', () => { + const metrics = new CommandMetrics('GET'); + metrics.record({ commandName: 'GET', latencyMicros: 100, success: true }); + metrics.record({ commandName: 'GET', latencyMicros: 200, success: true }); + metrics.record({ commandName: 'GET', latencyMicros: 300, success: false }); + + assert.equal(metrics.requests, 3); + assert.equal(metrics.errors, 1); + // The failed request's latency must not skew the distribution. + assert.equal(metrics.count(), 2); + assert.equal(metrics.max(), 200); + }); + + it('reports zeros for an empty histogram rather than sentinels', () => { + // minNonZeroValue would return Number.MAX_SAFE_INTEGER here and land a + // nonsense min in the NDJSON. + const metrics = new CommandMetrics('GET'); + assert.equal(metrics.count(), 0); + assert.equal(metrics.min(), 0); + assert.equal(metrics.max(), 0); + assert.equal(metrics.percentile(50), 0); + assert.equal(metrics.percentile(99.9), 0); + }); + + it('reports a legitimately recorded 0us sample as min 0', () => { + const metrics = new CommandMetrics('PING'); + metrics.record({ commandName: 'PING', latencyMicros: 0, success: true }); + metrics.record({ commandName: 'PING', latencyMicros: 5, success: true }); + assert.equal(metrics.min(), 0); + }); + + it('matches Java getMinValue/getMaxValue bucket quantization', () => { + // Verified against org.HdrHistogram.Histogram(1, 600_000_000, 3): at 3 + // significant figures Java reports the bucket's equivalent bounds, not the + // raw sample. hdr-histogram-js' maxValue/minNonZeroValue do NOT do this, so + // these anchors keep summary.min/max comparable across engines. + for (const [recorded, expectedMin, expectedMax] of [ + [1, 1, 1], + [100, 100, 100], + [1234, 1234, 1234], + [50_000, 49_984, 50_015], + [599_000_000, 598_736_896, 599_261_183], + ] as const) { + const metrics = new CommandMetrics('GET'); + metrics.record({ commandName: 'GET', latencyMicros: recorded, success: true }); + assert.equal(metrics.min(), expectedMin, `min for ${recorded}`); + assert.equal(metrics.max(), expectedMax, `max for ${recorded}`); + } + }); + + it('clamps a latency above the trackable range instead of throwing', () => { + const metrics = new CommandMetrics('GET'); + metrics.record({ + commandName: 'GET', + latencyMicros: HIGHEST_TRACKABLE_VALUE * 2, + success: true, + }); + assert.equal(metrics.count(), 1); + }); +}); + +describe('MetricsCollector', () => { + it('aggregates totals across commands', () => { + const collector = new MetricsCollector(); + collector.record({ commandName: 'GET', latencyMicros: 10, success: true }); + collector.record({ commandName: 'SET', latencyMicros: 20, success: true }); + collector.record({ commandName: 'SET', latencyMicros: 30, success: false }); + + assert.equal(collector.totalRequests, 3); + assert.equal(collector.totalErrors, 1); + assert.deepEqual([...collector.commandMetrics.keys()], ['GET', 'SET']); + assert.equal(collector.commandMetrics.get('SET')!.requests, 2); + }); + + it('preserves first-seen command order for stable NDJSON output', () => { + const collector = new MetricsCollector(); + collector.record({ commandName: 'SET', latencyMicros: 1, success: true }); + collector.record({ commandName: 'GET', latencyMicros: 1, success: true }); + assert.deepEqual([...collector.commandMetrics.keys()], ['SET', 'GET']); + }); + + it('reports zero duration until both start and stop have run', () => { + const collector = new MetricsCollector(); + assert.equal(collector.durationMillis(), 0); + collector.start(); + assert.equal(collector.durationMillis(), 0); + collector.stop(); + assert.ok(collector.durationMillis() >= 0); + }); +}); diff --git a/node/test/unit/commandSelector.test.ts b/node/test/unit/commandSelector.test.ts new file mode 100644 index 0000000..f29ae9e --- /dev/null +++ b/node/test/unit/commandSelector.test.ts @@ -0,0 +1,59 @@ +import assert from 'node:assert/strict'; +import { describe, it } from 'node:test'; + +import { CommandFactory } from '../../src/command/factory.js'; +import { CommandConfig } from '../../src/config/commandConfig.js'; +import { CommandSelector } from '../../src/engine/commandSelector.js'; + +function selectorFor(weights: Array<[string, number]>): CommandSelector { + return new CommandSelector( + CommandFactory.createAll( + weights.map(([command, weight]) => new CommandConfig({ command, weight })), + ), + ); +} + +function distribution(selector: CommandSelector, draws: number): Map { + const counts = new Map(); + for (let i = 0; i < draws; i++) { + const name = selector.select().name; + counts.set(name, (counts.get(name) ?? 0) + 1); + } + return counts; +} + +describe('CommandSelector', () => { + it('respects an 80/20 weighting within tolerance', () => { + const draws = 20_000; + const counts = distribution(selectorFor([['get', 0.8], ['set', 0.2]]), draws); + const getShare = (counts.get('GET') ?? 0) / draws; + assert.ok(Math.abs(getShare - 0.8) < 0.02, `GET share ${getShare.toFixed(3)} not near 0.8`); + }); + + it('normalizes weights that do not sum to 1', () => { + const draws = 20_000; + const counts = distribution(selectorFor([['get', 0.25], ['set', 0.25]]), draws); + const getShare = (counts.get('GET') ?? 0) / draws; + assert.ok(Math.abs(getShare - 0.5) < 0.02, `GET share ${getShare.toFixed(3)} not near 0.5`); + }); + + it('always returns the sole command', () => { + const selector = selectorFor([['ping', 1.0]]); + for (let i = 0; i < 100; i++) assert.equal(selector.select().name, 'PING'); + }); + + it('never returns a zero-weight command', () => { + const counts = distribution(selectorFor([['get', 1.0], ['set', 0]]), 5000); + assert.equal(counts.get('SET') ?? 0, 0); + }); + + it('falls back to a command when every weight is zero', () => { + // Guards against dividing by a zero total and returning undefined. + const selector = selectorFor([['get', 0], ['set', 0]]); + for (let i = 0; i < 100; i++) assert.ok(['GET', 'SET'].includes(selector.select().name)); + }); + + it('rejects an empty command list', () => { + assert.throws(() => new CommandSelector([]), /at least one command/); + }); +}); diff --git a/node/test/unit/configLoader.test.ts b/node/test/unit/configLoader.test.ts new file mode 100644 index 0000000..13971eb --- /dev/null +++ b/node/test/unit/configLoader.test.ts @@ -0,0 +1,269 @@ +import assert from 'node:assert/strict'; +import { mkdtempSync, writeFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { describe, it } from 'node:test'; + +import { DEFAULT_DATA_SIZE_BYTES } from '../../src/config/commandConfig.js'; +import { + DEFAULT_KEY_PREFIX, + DEFAULT_KEY_SIZE_BYTES, +} from '../../src/config/keyspaceConfig.js'; +import { ConfigError, ConfigLoader } from '../../src/config/loader.js'; + +function writeTempJson(name: string, data: unknown): string { + const dir = mkdtempSync(join(tmpdir(), 'resp-bench-node-')); + const path = join(dir, name); + writeFileSync(path, JSON.stringify(data), 'utf8'); + return path; +} + +const MINIMAL_PHASE = { + id: 'STEADY', + connections: 2, + completion: { type: 'requests', requests: 100 }, + keyspace: { keys_count: 50 }, + commands: [{ command: 'get', weight: 1.0 }], +}; + +describe('ConfigLoader — driver config', () => { + it('parses every schema field', () => { + const config = ConfigLoader.parseDriverConfig({ + schema_version: '1.0', + description: 'test driver', + driver_id: 'ioredis', + mode: 'cluster', + command_timeout_ms: 10_000, + tls: { enabled: true, ca_path: '/tmp/ca.pem' }, + auth: { username: 'u', password: 'p' }, + specific_driver_config: { secondary_driver_id: 'other' }, + }); + + assert.equal(config.schemaVersion, '1.0'); + assert.equal(config.description, 'test driver'); + assert.equal(config.driverId, 'ioredis'); + assert.equal(config.mode, 'cluster'); + assert.equal(config.commandTimeoutMs, 10_000); + assert.equal(config.tlsEnabled(), true); + assert.equal(config.hasAuth(), true); + assert.equal(config.isCluster(), true); + assert.equal(config.isStandalone(), false); + assert.equal(config.secondaryDriverId(), 'other'); + }); + + it('applies defaults for the optional fields', () => { + const config = ConfigLoader.parseDriverConfig({ driver_id: 'ioredis' }); + assert.equal(config.schemaVersion, '1.0'); + assert.equal(config.mode, 'standalone'); + assert.equal(config.commandTimeoutMs, null); + assert.equal(config.tlsEnabled(), false); + assert.equal(config.hasAuth(), false); + assert.deepEqual(config.specificDriverConfig, {}); + assert.equal(config.secondaryDriverId(), null); + }); + + it('treats an empty username and password as no auth', () => { + const config = ConfigLoader.parseDriverConfig({ + driver_id: 'ioredis', + auth: { username: '', password: '' }, + }); + assert.equal(config.hasAuth(), false); + }); + + it('rejects a missing driver_id', () => { + assert.throws(() => ConfigLoader.parseDriverConfig({ mode: 'standalone' }), ConfigError); + }); + + it('rejects an unknown mode', () => { + assert.throws( + () => ConfigLoader.parseDriverConfig({ driver_id: 'ioredis', mode: 'galaxy' }), + /must be standalone, cluster or sentinel/, + ); + }); + + it('loads the real repo driver configs', () => { + const path = writeTempJson('driver.json', { + schema_version: '1.0', + description: 'valkey-glide-node client - default configuration', + driver_id: 'valkey-glide-node', + mode: 'standalone', + specific_driver_config: {}, + }); + assert.equal(ConfigLoader.loadDriverConfig(path).driverId, 'valkey-glide-node'); + }); + + it('reports a readable error for a missing file', () => { + assert.throws(() => ConfigLoader.loadDriverConfig('/nope/missing.json'), /cannot read driver config/); + }); + + it('reports a readable error for malformed JSON', () => { + const dir = mkdtempSync(join(tmpdir(), 'resp-bench-node-')); + const path = join(dir, 'bad.json'); + writeFileSync(path, '{ not json', 'utf8'); + assert.throws(() => ConfigLoader.loadDriverConfig(path), /is not valid JSON/); + }); +}); + +describe('ConfigLoader — workload config', () => { + it('parses a full workload', () => { + const workload = ConfigLoader.parseWorkloadConfig({ + schema_version: '1.0', + benchmark_profile: { name: 'Reference', description: 'd', version: '1.0.0' }, + phases: [ + { + id: 'WARMUP', + description: 'populate', + connections: 1, + cps_limit: -1, + rps_limit: -1, + pipeline_depth: 4, + warmup_requests: 3, + completion: { type: 'requests', requests: 1_000_000 }, + keyspace: { + keys_count: 1_000_000, + key_size_bytes: 16, + key_prefix: 'bench:', + generation_alg: 'uniform_rand', + seed: 12345, + }, + commands: [ + { command: 'get', weight: 0.8 }, + { command: 'set', weight: 0.2, data_size_bytes: 512 }, + ], + }, + ], + }); + + assert.equal(workload.name(), 'Reference'); + assert.equal(workload.phases.length, 1); + const phase = workload.phases[0]!; + assert.equal(phase.id, 'WARMUP'); + assert.equal(phase.connections, 1); + assert.equal(phase.pipelineDepth, 4); + assert.equal(phase.effectivePipelineDepth(), 4); + assert.equal(phase.warmupRequests, 3); + assert.equal(phase.hasCpsLimit(), false); + assert.equal(phase.hasRpsLimit(), false); + assert.equal(phase.completion.isRequestBased(), true); + assert.equal(phase.completion.totalRequests(), 1_000_000); + assert.equal(phase.keyspace.isUniformRand(), true); + assert.equal(phase.keyspace.seedValue(), 12345); + assert.equal(phase.commands[1]!.dataSizeBytes, 512); + }); + + it('applies the cross-engine defaults', () => { + const workload = ConfigLoader.parseWorkloadConfig({ phases: [MINIMAL_PHASE] }); + const phase = workload.phases[0]!; + assert.equal(phase.cpsLimit, -1); + assert.equal(phase.rpsLimit, -1); + assert.equal(phase.pipelineDepth, 1); + assert.equal(phase.warmupRequests, 1); + assert.equal(phase.description, null); + assert.equal(phase.keyspace.keySizeBytes, DEFAULT_KEY_SIZE_BYTES); + assert.equal(phase.keyspace.keyPrefix, DEFAULT_KEY_PREFIX); + assert.equal(phase.keyspace.isSequentialInt(), true); + assert.equal(phase.keyspace.seedValue(), 0); + assert.equal(phase.commands[0]!.dataSizeBytes, DEFAULT_DATA_SIZE_BYTES); + assert.equal(phase.commands[0]!.weight, 1.0); + assert.equal(workload.name(), 'unnamed'); + }); + + it('treats an explicit null as absent, not as null', () => { + // The Ruby/Python engines coerce nulls to defaults; a null leaking through + // would surface as NaN padding or a null prefix deep in the worker loop. + const workload = ConfigLoader.parseWorkloadConfig({ + phases: [ + { + ...MINIMAL_PHASE, + cps_limit: null, + rps_limit: null, + pipeline_depth: null, + warmup_requests: null, + keyspace: { keys_count: 50, key_size_bytes: null, key_prefix: null, generation_alg: null }, + commands: [{ command: 'set', weight: null, data_size_bytes: null }], + }, + ], + }); + const phase = workload.phases[0]!; + assert.equal(phase.cpsLimit, -1); + assert.equal(phase.pipelineDepth, 1); + assert.equal(phase.warmupRequests, 1); + assert.equal(phase.keyspace.keySizeBytes, DEFAULT_KEY_SIZE_BYTES); + assert.equal(phase.keyspace.keyPrefix, DEFAULT_KEY_PREFIX); + assert.equal(phase.keyspace.generationAlg, 'sequential_int'); + assert.equal(phase.commands[0]!.weight, 1.0); + assert.equal(phase.commands[0]!.dataSizeBytes, DEFAULT_DATA_SIZE_BYTES); + }); + + it('lower-cases command names', () => { + const workload = ConfigLoader.parseWorkloadConfig({ + phases: [{ ...MINIMAL_PHASE, commands: [{ command: 'GET', weight: 1 }] }], + }); + assert.equal(workload.phases[0]!.commands[0]!.command, 'get'); + }); + + it('recognises limits when set', () => { + const workload = ConfigLoader.parseWorkloadConfig({ + phases: [{ ...MINIMAL_PHASE, cps_limit: 10, rps_limit: 500 }], + }); + const phase = workload.phases[0]!; + assert.equal(phase.hasCpsLimit(), true); + assert.equal(phase.hasRpsLimit(), true); + }); + + it('rejects a workload with no phases', () => { + assert.throws(() => ConfigLoader.parseWorkloadConfig({ phases: [] }), /non-empty "phases"/); + assert.throws(() => ConfigLoader.parseWorkloadConfig({}), /non-empty "phases"/); + }); + + it('rejects a phase with no commands', () => { + assert.throws( + () => ConfigLoader.parseWorkloadConfig({ phases: [{ ...MINIMAL_PHASE, commands: [] }] }), + /non-empty "commands"/, + ); + }); + + it('rejects non-positive connections', () => { + assert.throws( + () => ConfigLoader.parseWorkloadConfig({ phases: [{ ...MINIMAL_PHASE, connections: 0 }] }), + /"connections" must be positive/, + ); + }); + + it('rejects a completion type that carries no target', () => { + assert.throws( + () => + ConfigLoader.parseWorkloadConfig({ + phases: [{ ...MINIMAL_PHASE, completion: { type: 'duration' } }], + }), + /requires a positive "seconds"/, + ); + assert.throws( + () => + ConfigLoader.parseWorkloadConfig({ + phases: [{ ...MINIMAL_PHASE, completion: { type: 'requests' } }], + }), + /requires a positive "requests"/, + ); + }); + + it('rejects an unknown generation_alg', () => { + assert.throws( + () => + ConfigLoader.parseWorkloadConfig({ + phases: [{ ...MINIMAL_PHASE, keyspace: { keys_count: 10, generation_alg: 'zipf' } }], + }), + /sequential_int or uniform_rand/, + ); + }); + + it('rejects a weight outside 0..1, as Java does', () => { + assert.throws( + () => + ConfigLoader.parseWorkloadConfig({ + phases: [{ ...MINIMAL_PHASE, commands: [{ command: 'get', weight: 5 }] }], + }), + /"weight" must be between 0 and 1/, + ); + }); +}); diff --git a/node/test/unit/factory.test.ts b/node/test/unit/factory.test.ts new file mode 100644 index 0000000..2ab2835 --- /dev/null +++ b/node/test/unit/factory.test.ts @@ -0,0 +1,115 @@ +import assert from 'node:assert/strict'; +import { describe, it } from 'node:test'; + +import { BenchmarkClientFactory } from '../../src/client/factory.js'; +import { packageVersion } from '../../src/client/driverVersion.js'; +import { CommandFactory } from '../../src/command/factory.js'; +import { CommandConfig } from '../../src/config/commandConfig.js'; +import { SetCommand } from '../../src/command/impl/setCommand.js'; + +describe('BenchmarkClientFactory', () => { + it('registers the node driver ids', () => { + assert.deepEqual(BenchmarkClientFactory.supportedDrivers(), [ + 'valkey-glide-node', + 'ioredis', + 'iovalkey', + 'recording', + ]); + }); + + it('does not register a bare valkey-glide', () => { + // scripts/run_benchmark_matrix.py's DRIVER_ENGINE_MAP is global and already + // maps "valkey-glide" to the Java engine. Claiming it here would silently + // reroute Java's glide runs to Node. + assert.equal(BenchmarkClientFactory.supportedDrivers().includes('valkey-glide'), false); + }); + + it('describes every driver for --info', () => { + for (const { driverId, description } of BenchmarkClientFactory.describe()) { + assert.ok(driverId.length > 0); + assert.ok(description.length > 0, `${driverId} has no description`); + } + }); + + it('rejects an unknown driver with the supported list', async () => { + await assert.rejects( + () => BenchmarkClientFactory.create('memcached'), + /Unknown driver: memcached\. Supported: valkey-glide-node, ioredis, iovalkey, recording/, + ); + }); + + it('is case-insensitive on driver_id', async () => { + const client = await BenchmarkClientFactory.create('IoRedis'); + assert.ok(client); + }); + + it('loads the recording driver without a server', async () => { + const client = await BenchmarkClientFactory.create('recording'); + assert.equal(client.driverVersion(), '1.0.0'); + }); +}); + +describe('CommandFactory', () => { + it('supports the cross-engine command set', () => { + assert.deepEqual(CommandFactory.supportedCommands(), ['get', 'set', 'ping']); + }); + + it('builds each command with its weight and name', () => { + const get = CommandFactory.create(new CommandConfig({ command: 'get', weight: 0.8 })); + assert.equal(get.name, 'GET'); + assert.equal(get.weight, 0.8); + assert.equal(get.usesKey, true); + + const ping = CommandFactory.create(new CommandConfig({ command: 'ping' })); + assert.equal(ping.name, 'PING'); + // PING must not consume a generated key: Java's PingCommand ignores the key + // generator, so advancing it here would shift the shared key sequence. + assert.equal(ping.usesKey, false); + }); + + it('accepts an upper-case command name', () => { + assert.equal(CommandFactory.create(new CommandConfig({ command: 'SET' })).name, 'SET'); + }); + + it('rejects an unknown command', () => { + assert.throws( + () => CommandFactory.create(new CommandConfig({ command: 'incr' })), + /Unknown command: incr\. Supported: get, set, ping/, + ); + }); + + it('createAll preserves order', () => { + const commands = CommandFactory.createAll([ + new CommandConfig({ command: 'get', weight: 0.8 }), + new CommandConfig({ command: 'set', weight: 0.2 }), + ]); + assert.deepEqual( + commands.map((c) => c.name), + ['GET', 'SET'], + ); + }); +}); + +describe('SetCommand payload', () => { + it('generates exactly data_size_bytes using the cross-engine pattern', () => { + // Ruby and Python use the same repeated "0123456789ABCDEF" filler. + for (const size of [1, 16, 32, 256, 512, 1000]) { + assert.equal(SetCommand.generateValue(size).length, size); + } + assert.equal(SetCommand.generateValue(20).toString('latin1'), '0123456789ABCDEF0123'); + }); +}); + +describe('packageVersion', () => { + it('reads a version despite a restricted exports map', () => { + // @valkey/valkey-glide does not export ./package.json, so a plain + // require('/package.json') throws ERR_PACKAGE_PATH_NOT_EXPORTED. + assert.match(packageVersion('@valkey/valkey-glide'), /^\d+\.\d+\.\d+/); + assert.match(packageVersion('ioredis'), /^\d+\.\d+\.\d+/); + assert.match(packageVersion('iovalkey'), /^\d+\.\d+\.\d+/); + }); + + it('returns "unknown" for a package that is not installed', () => { + assert.equal(packageVersion('definitely-not-installed-xyz'), 'unknown'); + }); +}); diff --git a/node/test/unit/hdrHistogram.test.ts b/node/test/unit/hdrHistogram.test.ts new file mode 100644 index 0000000..f39bef9 --- /dev/null +++ b/node/test/unit/hdrHistogram.test.ts @@ -0,0 +1,63 @@ +import assert from 'node:assert/strict'; +import { describe, it } from 'node:test'; + +import { + decodeBase64, + encodeBase64, + HIGHEST_TRACKABLE_VALUE, + LOWEST_TRACKABLE_VALUE, + newHistogram, + SIGNIFICANT_FIGURES, +} from '../../src/metrics/hdrHistogram.js'; + +describe('HDR histogram', () => { + it('uses the cross-engine range and precision', () => { + // Java SynchronizedHistogram(600_000_000, 3), C# LongConcurrentHistogram(1, + // 600_000_000, 3), Ruby HDRHistogram.new(1, 600_000_000, 3). + assert.equal(LOWEST_TRACKABLE_VALUE, 1); + assert.equal(HIGHEST_TRACKABLE_VALUE, 600_000_000); + assert.equal(SIGNIFICANT_FIGURES, 3); + const histogram = newHistogram(); + assert.equal(histogram.highestTrackableValue, 600_000_000); + assert.equal(histogram.numberOfSignificantValueDigits, 3); + }); + + it('emits a HIST-prefixed payload, not a double-encoded one', () => { + // Java's writer base64s the compressed bytes, which always start with the + // V2 cookie 0x1c849314 -> "HIST". A payload that does not start with HIST + // means it was base64-encoded twice and Java/Ruby cannot decode it. + const histogram = newHistogram(); + histogram.recordValue(1234); + const payload = encodeBase64(histogram); + assert.ok(payload.startsWith('HIST'), `payload should start with HIST, got ${payload.slice(0, 12)}`); + assert.doesNotMatch(payload, /^SElTVA/, 'payload is base64 of "HIST" — encoded twice'); + }); + + it('round-trips percentiles and total count', () => { + const histogram = newHistogram(); + for (let value = 1; value <= 1000; value++) histogram.recordValue(value); + + const decoded = decodeBase64(encodeBase64(histogram)); + assert.equal(decoded.totalCount, histogram.totalCount); + for (const percentile of [0, 50, 95, 99, 99.9, 100]) { + assert.equal( + decoded.getValueAtPercentile(percentile), + histogram.getValueAtPercentile(percentile), + `p${percentile} differs after round trip`, + ); + } + }); + + it('encodes an empty histogram without throwing', () => { + // An all-errors command still needs an hdr block in the NDJSON. + const payload = encodeBase64(newHistogram()); + assert.ok(payload.startsWith('HIST')); + assert.equal(decodeBase64(payload).totalCount, 0); + }); + + it('records the top of the range', () => { + const histogram = newHistogram(); + histogram.recordValue(HIGHEST_TRACKABLE_VALUE); + assert.equal(histogram.totalCount, 1); + }); +}); diff --git a/node/test/unit/javaRandom.test.ts b/node/test/unit/javaRandom.test.ts new file mode 100644 index 0000000..e1f7002 --- /dev/null +++ b/node/test/unit/javaRandom.test.ts @@ -0,0 +1,84 @@ +/** + * JavaRandom parity tests. + * + * The LCG is anchored to a well-known java.util.Random value, which proves + * byte-for-byte compatibility with the Java reference without needing a JVM. The + * two sequences below were additionally cross-checked against the Python + * engine's JavaRandom, so a failure here means Node has diverged from both. + */ + +import assert from 'node:assert/strict'; +import { describe, it } from 'node:test'; + +import { JavaRandom, toInt32 } from '../../src/engine/javaRandom.js'; + +describe('JavaRandom', () => { + it('matches the documented java.util.Random(0).nextInt() value', () => { + // java.util.Random(0).nextInt() -- i.e. next(32) as a signed int -- is the + // well-documented value -1155484576. This anchors the LCG to real Java. + assert.equal(toInt32(new JavaRandom(0).next(32)), -1155484576); + }); + + it('reproduces the cross-engine sequence for seed 12345, bound 1000', () => { + const rng = new JavaRandom(12345); + const actual = Array.from({ length: 10 }, () => rng.nextInt(1000)); + assert.deepEqual(actual, [251, 80, 241, 828, 55, 84, 375, 802, 501, 389]); + }); + + it('reproduces the cross-engine sequence for a power-of-two bound', () => { + // Exercises the power-of-two fast path, a separate branch in Java. + const rng = new JavaRandom(12345); + const actual = Array.from({ length: 8 }, () => rng.nextInt(256)); + assert.deepEqual(actual, [92, 131, 238, 234, 213, 9, 83, 31]); + }); + + it('is deterministic for a given seed', () => { + const rngA = new JavaRandom(12345); + const rngB = new JavaRandom(12345); + assert.deepEqual( + Array.from({ length: 10 }, () => rngA.nextInt(1000)), + Array.from({ length: 10 }, () => rngB.nextInt(1000)), + ); + }); + + it('produces different sequences for different seeds', () => { + const a = new JavaRandom(12345); + const b = new JavaRandom(54321); + assert.notDeepEqual( + Array.from({ length: 10 }, () => a.nextInt(1000)), + Array.from({ length: 10 }, () => b.nextInt(1000)), + ); + }); + + it('setSeed resets the stream', () => { + const rng = new JavaRandom(12345); + const first = Array.from({ length: 5 }, () => rng.nextInt(1000)); + rng.setSeed(12345); + const second = Array.from({ length: 5 }, () => rng.nextInt(1000)); + assert.deepEqual(first, second); + }); + + it('rejects a non-positive bound', () => { + const rng = new JavaRandom(12345); + assert.throws(() => rng.nextInt(0), /bound must be a positive integer/); + assert.throws(() => rng.nextInt(-1), /bound must be a positive integer/); + }); + + it('stays within the bound', () => { + const rng = new JavaRandom(12345); + for (let i = 0; i < 1000; i++) { + const value = rng.nextInt(100); + assert.ok(value >= 0 && value < 100, `${value} out of range`); + } + }); + + it('stays within power-of-two bounds', () => { + const rng = new JavaRandom(12345); + for (const bound of [2, 4, 8, 16, 256, 1024]) { + for (let i = 0; i < 200; i++) { + const value = rng.nextInt(bound); + assert.ok(value >= 0 && value < bound, `${value} out of range for ${bound}`); + } + } + }); +}); diff --git a/node/test/unit/keyGenerator.test.ts b/node/test/unit/keyGenerator.test.ts new file mode 100644 index 0000000..2f6195e --- /dev/null +++ b/node/test/unit/keyGenerator.test.ts @@ -0,0 +1,133 @@ +import assert from 'node:assert/strict'; +import { describe, it } from 'node:test'; + +import { KeyspaceConfig } from '../../src/config/keyspaceConfig.js'; +import { Counter, KeyGenerator } from '../../src/engine/keyGenerator.js'; + +describe('KeyGenerator', () => { + describe('sequential_int', () => { + it('emits 0..N-1 then wraps', () => { + const gen = KeyGenerator.create( + new KeyspaceConfig({ keysCount: 3, keyPrefix: 'test:', keySizeBytes: 6 }), + ); + assert.deepEqual( + Array.from({ length: 4 }, () => gen.nextKey()), + ['test:0', 'test:1', 'test:2', 'test:0'], + ); + }); + + it('shares one counter across workers, as Java does', () => { + // The whole point of the shared counter: several connections collectively + // populate the keyspace instead of each replaying 0, 1, 2, ... + const config = new KeyspaceConfig({ keysCount: 100, keyPrefix: 'k:', keySizeBytes: 4 }); + const counter = new Counter(); + const workerA = KeyGenerator.createWithSeed(config, 0, counter); + const workerB = KeyGenerator.createWithSeed(config, 1, counter); + assert.deepEqual( + [workerA.nextKey(), workerB.nextKey(), workerA.nextKey(), workerB.nextKey()], + ['k:00', 'k:01', 'k:02', 'k:03'], + ); + }); + + it('gives each generator its own counter when none is shared', () => { + const config = new KeyspaceConfig({ keysCount: 100, keyPrefix: 'k:', keySizeBytes: 4 }); + assert.equal(KeyGenerator.create(config).nextKey(), 'k:00'); + assert.equal(KeyGenerator.create(config).nextKey(), 'k:00'); + }); + }); + + describe('uniform_rand', () => { + it('is reproducible for the same seed', () => { + const config = new KeyspaceConfig({ + keysCount: 1000, + keyPrefix: 'test:', + generationAlg: 'uniform_rand', + seed: 12345, + }); + const a = KeyGenerator.create(config); + const b = KeyGenerator.create(config); + for (let i = 0; i < 100; i++) assert.equal(a.nextKey(), b.nextKey()); + }); + + it('derives a per-worker seed of base + index, as Java does', () => { + const config = new KeyspaceConfig({ + keysCount: 1000, + keyPrefix: 'test:', + generationAlg: 'uniform_rand', + seed: 12345, + }); + const worker0 = KeyGenerator.createWithSeed(config, 12345); + const worker1 = KeyGenerator.createWithSeed(config, 12346); + assert.notEqual(worker0.nextKey(), worker1.nextKey()); + }); + + it('matches the JavaRandom sequence for seed 12345', () => { + // Anchored to the same values as javaRandom.test.ts, so a key-formatting + // change cannot silently break cross-engine key parity. + const gen = KeyGenerator.create( + new KeyspaceConfig({ + keysCount: 1000, + keyPrefix: 'bench:', + keySizeBytes: 16, + generationAlg: 'uniform_rand', + seed: 12345, + }), + ); + assert.deepEqual( + Array.from({ length: 4 }, () => gen.nextKey()), + ['bench:0000000251', 'bench:0000000080', 'bench:0000000241', 'bench:0000000828'], + ); + }); + + it('reset restores the stream', () => { + const gen = KeyGenerator.create( + new KeyspaceConfig({ + keysCount: 1000, + keyPrefix: 'test:', + generationAlg: 'uniform_rand', + seed: 999, + }), + ); + const first = Array.from({ length: 5 }, () => gen.nextKey()); + gen.reset(); + assert.deepEqual( + Array.from({ length: 5 }, () => gen.nextKey()), + first, + ); + }); + }); + + describe('key formatting', () => { + it('zero-pads to key_size_bytes minus the prefix, as Java does', () => { + // Reference configs use key_prefix "bench:" (6) and key_size_bytes 16, + // so the numeric part is 10 digits: bench:0000000042 + const gen = KeyGenerator.create( + new KeyspaceConfig({ keysCount: 1_000_000, keyPrefix: 'bench:', keySizeBytes: 16 }), + ); + const key = gen.nextKey(); + assert.equal(key, 'bench:0000000000'); + assert.equal(key.length, 16); + }); + + it('keeps at least one digit when the prefix fills key_size_bytes', () => { + const gen = KeyGenerator.create( + new KeyspaceConfig({ keysCount: 10, keyPrefix: 'averylongprefix:', keySizeBytes: 4 }), + ); + assert.equal(gen.nextKey(), 'averylongprefix:0'); + }); + + it('does not truncate an index wider than the padding', () => { + const gen = KeyGenerator.create( + new KeyspaceConfig({ keysCount: 1000, keyPrefix: 'k:', keySizeBytes: 4 }), + ); + const keys = Array.from({ length: 101 }, () => gen.nextKey()); + assert.equal(keys[0], 'k:00'); + assert.equal(keys[100], 'k:100'); + }); + + it('applies the documented defaults', () => { + const gen = KeyGenerator.create(new KeyspaceConfig({ keysCount: 10 })); + assert.equal(gen.nextKey(), 'bench:0000000000'); + }); + }); +}); diff --git a/node/test/unit/ndjsonWriter.test.ts b/node/test/unit/ndjsonWriter.test.ts new file mode 100644 index 0000000..7bb3e22 --- /dev/null +++ b/node/test/unit/ndjsonWriter.test.ts @@ -0,0 +1,193 @@ +import assert from 'node:assert/strict'; +import { mkdtempSync, readFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { describe, it } from 'node:test'; + +import { MetricsCollector } from '../../src/metrics/collector.js'; +import { decodeBase64 } from '../../src/metrics/hdrHistogram.js'; +import { NdjsonWriter } from '../../src/metrics/ndjsonWriter.js'; + +function tempPath(name = 'metrics.ndjson'): string { + return join(mkdtempSync(join(tmpdir(), 'resp-bench-node-')), name); +} + +function collectorWith(samples: Array<{ name: string; latency: number; ok?: boolean }>) { + const collector = new MetricsCollector(); + collector.start(); + for (const { name, latency, ok = true } of samples) { + collector.record({ commandName: name, latencyMicros: latency, success: ok }); + } + collector.stop(); + return collector; +} + +function readRecords(path: string): Array> { + return readFileSync(path, 'utf8') + .trimEnd() + .split('\n') + .map((line) => JSON.parse(line)); +} + +describe('NdjsonWriter', () => { + it('writes the documented record shape', () => { + const path = tempPath(); + const writer = new NdjsonWriter(path); + writer.setMetadata({ + commitId: 'abc123', + driverId: 'ioredis', + primaryDriverVersion: '5.11.1', + }); + writer.writePhaseResults({ + phaseId: 'STEADY', + status: 'COMPLETED', + connections: 4, + collector: collectorWith([ + { name: 'GET', latency: 100 }, + { name: 'GET', latency: 200 }, + { name: 'SET', latency: 300 }, + ]), + }); + + const [record] = readRecords(path); + assert.ok(record); + + assert.deepEqual(Object.keys(record).sort(), ['metadata', 'metrics', 'phase', 'totals']); + assert.equal(record['metadata'].commit_id, 'abc123'); + assert.equal(record['metadata'].driver_id, 'ioredis'); + assert.equal(record['metadata'].primary_driver_version, '5.11.1'); + assert.ok(record['metadata'].timestamp); + // Absent secondary driver fields must be omitted, not null. + assert.equal('secondary_driver_id' in record['metadata'], false); + + assert.deepEqual(Object.keys(record['phase']).sort(), [ + 'connections', + 'duration_ms', + 'finish_timestamp', + 'id', + 'start_timestamp', + 'status', + ]); + assert.equal(record['phase'].id, 'STEADY'); + assert.equal(record['phase'].status, 'COMPLETED'); + assert.equal(record['phase'].connections, 4); + assert.match(record['phase'].start_timestamp, /^\d{4}-\d{2}-\d{2}T.*Z$/); + + assert.deepEqual(record['totals'], { requests: 3, errors: 0 }); + + assert.deepEqual(Object.keys(record['metrics']).sort(), ['GET', 'SET']); + const get = record['metrics'].GET; + assert.equal(get.requests, 2); + assert.equal(get.errors, 0); + assert.equal(get.latency.unit, 'us'); + assert.equal(get.latency.count, 2); + assert.deepEqual(Object.keys(get.latency.summary).sort(), [ + 'max', + 'min', + 'p50', + 'p95', + 'p99', + 'p999', + ]); + assert.equal(get.latency.hdr.format, 'hdr'); + assert.equal(get.latency.hdr.sigfig, 3); + assert.ok(get.latency.hdr.payload_b64.startsWith('HIST')); + }); + + it('writes one compact line per phase and appends across phases', () => { + const path = tempPath(); + const writer = new NdjsonWriter(path); + writer.setMetadata({ driverId: 'ioredis' }); + for (const phaseId of ['WARMUP', 'STEADY']) { + writer.writePhaseResults({ + phaseId, + status: 'COMPLETED', + connections: 1, + collector: collectorWith([{ name: 'GET', latency: 10 }]), + }); + } + + const raw = readFileSync(path, 'utf8'); + assert.equal(raw.endsWith('\n'), true); + const lines = raw.trimEnd().split('\n'); + assert.equal(lines.length, 2); + // NDJSON requires no embedded newlines -- no pretty printing. + for (const line of lines) assert.doesNotMatch(line, /\n/); + assert.deepEqual( + lines.map((line) => JSON.parse(line).phase.id), + ['WARMUP', 'STEADY'], + ); + }); + + it('creates the parent directory when it does not exist', () => { + const path = join(mkdtempSync(join(tmpdir(), 'resp-bench-node-')), 'nested', 'deep', 'm.ndjson'); + const writer = new NdjsonWriter(path); + writer.setMetadata({ driverId: 'ioredis' }); + writer.writePhaseResults({ + phaseId: 'P', + status: 'COMPLETED', + connections: 1, + collector: collectorWith([{ name: 'GET', latency: 10 }]), + }); + assert.equal(readRecords(path).length, 1); + }); + + it('emits an hdr block for a command that only ever errored', () => { + // Matches Java: the histogram is created eagerly, so analysis tooling can + // always read metrics..latency.hdr without a null check. + const path = tempPath(); + const writer = new NdjsonWriter(path); + writer.setMetadata({ driverId: 'ioredis' }); + writer.writePhaseResults({ + phaseId: 'ERRORS', + status: 'COMPLETED', + connections: 1, + collector: collectorWith([ + { name: 'GET', latency: 50, ok: false }, + { name: 'GET', latency: 60, ok: false }, + ]), + }); + + const [record] = readRecords(path); + const get = record!['metrics'].GET; + assert.equal(get.requests, 2); + assert.equal(get.errors, 2); + assert.equal(get.latency.count, 0); + assert.deepEqual(get.latency.summary, { min: 0, p50: 0, p95: 0, p99: 0, p999: 0, max: 0 }); + assert.ok(get.latency.hdr.payload_b64.startsWith('HIST')); + assert.equal(decodeBase64(get.latency.hdr.payload_b64).totalCount, 0); + assert.deepEqual(record!['totals'], { requests: 2, errors: 2 }); + }); + + it('omits the metadata block entirely when nothing identifies the run', () => { + const path = tempPath(); + const writer = new NdjsonWriter(path); + writer.writePhaseResults({ + phaseId: 'P', + status: 'COMPLETED', + connections: 1, + collector: collectorWith([{ name: 'GET', latency: 10 }]), + }); + assert.equal('metadata' in readRecords(path)[0]!, false); + }); + + it('carries the secondary driver fields when present', () => { + const path = tempPath(); + const writer = new NdjsonWriter(path); + writer.setMetadata({ + driverId: 'composite', + primaryDriverVersion: '1.0.0', + secondaryDriverId: 'ioredis', + secondaryDriverVersion: '5.11.1', + }); + writer.writePhaseResults({ + phaseId: 'P', + status: 'COMPLETED', + connections: 1, + collector: collectorWith([{ name: 'GET', latency: 10 }]), + }); + const metadata = readRecords(path)[0]!['metadata']; + assert.equal(metadata.secondary_driver_id, 'ioredis'); + assert.equal(metadata.secondary_driver_version, '5.11.1'); + }); +}); diff --git a/node/test/unit/rateLimiter.test.ts b/node/test/unit/rateLimiter.test.ts new file mode 100644 index 0000000..e39f96a --- /dev/null +++ b/node/test/unit/rateLimiter.test.ts @@ -0,0 +1,82 @@ +import assert from 'node:assert/strict'; +import { describe, it } from 'node:test'; + +import { RateLimiter } from '../../src/engine/rateLimiter.js'; + +describe('RateLimiter', () => { + it('returns null for an unlimited rate', () => { + assert.equal(RateLimiter.create(0), null); + assert.equal(RateLimiter.create(-1), null); + }); + + it('returns a limiter for a positive rate', () => { + const limiter = RateLimiter.create(100); + assert.ok(limiter); + assert.equal(limiter.ratePerSecond, 100); + }); + + it('achieves the target rate within 5%', async () => { + // docs/ADDING_LANGUAGE.md's stated tolerance for the rate limiter. + const rate = 200; + const requests = 100; + const limiter = RateLimiter.create(rate)!; + + const start = process.hrtime.bigint(); + for (let i = 0; i < requests; i++) await limiter.acquire(); + const elapsedSeconds = Number(process.hrtime.bigint() - start) / 1e9; + + // The first acquire is free (the bucket starts open), so the limiter only + // paces the remaining requests. + const expectedSeconds = (requests - 1) / rate; + assert.ok( + elapsedSeconds >= expectedSeconds * 0.95, + `finished too fast: ${elapsedSeconds.toFixed(3)}s < ${(expectedSeconds * 0.95).toFixed(3)}s`, + ); + assert.ok( + elapsedSeconds <= expectedSeconds * 1.3, + `finished too slow: ${elapsedSeconds.toFixed(3)}s > ${(expectedSeconds * 1.3).toFixed(3)}s`, + ); + }); + + it('paces a rate whose interval is below setTimeout resolution', async () => { + // 5000/s is a 200us interval -- well under setTimeout's ~1ms floor, so this + // only passes if sub-millisecond waits yield via setImmediate instead. + const rate = 5000; + const requests = 500; + const limiter = RateLimiter.create(rate)!; + + const start = process.hrtime.bigint(); + for (let i = 0; i < requests; i++) await limiter.acquire(); + const elapsedSeconds = Number(process.hrtime.bigint() - start) / 1e9; + + const expectedSeconds = (requests - 1) / rate; + assert.ok( + elapsedSeconds <= expectedSeconds * 2, + `sub-ms pacing overshot badly: ${elapsedSeconds.toFixed(3)}s vs ${expectedSeconds.toFixed(3)}s ` + + '(a clamped setTimeout would take ~10x this)', + ); + }); + + it('shares one budget across concurrent callers', async () => { + const rate = 200; + const perWorker = 25; + const workers = 4; + const limiter = RateLimiter.create(rate)!; + + const start = process.hrtime.bigint(); + await Promise.all( + Array.from({ length: workers }, async () => { + for (let i = 0; i < perWorker; i++) await limiter.acquire(); + }), + ); + const elapsedSeconds = Number(process.hrtime.bigint() - start) / 1e9; + + // The limit is global: 100 requests at 200/s takes ~0.5s regardless of how + // many workers issue them. + const expectedSeconds = (workers * perWorker - 1) / rate; + assert.ok( + elapsedSeconds >= expectedSeconds * 0.95, + `concurrent callers bypassed the limit: ${elapsedSeconds.toFixed(3)}s`, + ); + }); +}); diff --git a/node/test/unit/recordingClient.test.ts b/node/test/unit/recordingClient.test.ts new file mode 100644 index 0000000..19af4f6 --- /dev/null +++ b/node/test/unit/recordingClient.test.ts @@ -0,0 +1,101 @@ +import assert from 'node:assert/strict'; +import { describe, it } from 'node:test'; + +import { RecordingClient } from '../../src/client/impl/recordingClient.js'; +import { ConfigLoader } from '../../src/config/loader.js'; + +function config(specific: Record = {}) { + return ConfigLoader.parseDriverConfig({ + driver_id: 'recording', + mode: 'standalone', + specific_driver_config: specific, + }); +} + +async function connected(specific: Record = {}): Promise { + const client = new RecordingClient(); + await client.connect('localhost', 6379, config(specific)); + return client; +} + +describe('RecordingClient', () => { + it('records the operations it was asked to perform', async () => { + const client = await connected(); + await client.ping(); + await client.set('bench:0000000001', Buffer.from('abc', 'latin1')); + await client.get('bench:0000000001'); + await client.close(); + + assert.deepEqual( + client.operations.map((op) => op.command), + ['CONNECT', 'PING', 'SET', 'GET', 'CLOSE'], + ); + // The keys the engine passed through are observable, so a key-generation + // regression shows up as a wrong key here rather than as silent drift. + assert.deepEqual( + client.operations.filter((op) => op.key !== null).map((op) => op.key), + ['bench:0000000001', 'bench:0000000001'], + ); + assert.ok(client.operations.every((op) => op.success)); + }); + + it('behaves like a key-value store for GET after SET', async () => { + const client = await connected(); + assert.equal((await client.get('missing')).value, null); + await client.set('k', Buffer.from('hello', 'latin1')); + assert.equal((await client.get('k')).value, 'hello'); + }); + + it('returns the cross-engine success values', async () => { + const client = await connected(); + assert.equal((await client.ping()).value, 'PONG'); + assert.equal((await client.set('k', Buffer.alloc(4))).value, 'OK'); + }); + + it('injects errors at error_rate 1.0 with the configured message', async () => { + const client = await connected({ error_rate: 1.0, error_message: 'boom' }); + for (const result of [ + await client.ping(), + await client.get('k'), + await client.set('k', Buffer.alloc(1)), + ]) { + assert.equal(result.error?.message, 'boom'); + assert.equal(result.value, null); + } + assert.ok(client.operations.slice(1).every((op) => !op.success)); + }); + + it('never injects errors while in warmup mode', async () => { + // Mirrors Java's setWarmupMode: the engine's warmup fail-fast must not be + // tripped by errors the workload deliberately injects. + const client = await connected({ error_rate: 1.0 }); + client.setWarmupMode(true); + assert.equal((await client.ping()).error, undefined); + client.setWarmupMode(false); + assert.notEqual((await client.ping()).error, undefined); + }); + + it('does not fail a SET when its own error is injected mid-store', async () => { + // A failed SET must not store, or a later GET would report data the server + // never accepted. + const client = await connected({ error_rate: 1.0 }); + await client.set('k', Buffer.from('nope', 'latin1')); + client.setWarmupMode(true); + assert.equal((await client.get('k')).value, null); + }); + + it('applies a configured operation delay', async () => { + const client = await connected({ operation_delay_micros: 5000 }); + const result = await client.ping(); + assert.ok(result.latencyMicros >= 3000, `latency ${result.latencyMicros}us too low for a 5ms delay`); + }); + + it('reports zero-ish latency with no configured delay', async () => { + const client = await connected(); + assert.ok((await client.ping()).latencyMicros < 5000); + }); + + it('reports a fixed driver version', async () => { + assert.equal(new RecordingClient().driverVersion(), '1.0.0'); + }); +}); diff --git a/node/tsconfig.json b/node/tsconfig.json new file mode 100644 index 0000000..8f1e804 --- /dev/null +++ b/node/tsconfig.json @@ -0,0 +1,23 @@ +{ + "compilerOptions": { + "target": "ES2022", + "lib": ["ES2022"], + "module": "NodeNext", + "moduleResolution": "NodeNext", + "rootDir": ".", + "outDir": "dist", + "sourceMap": true, + "declaration": false, + "incremental": true, + "tsBuildInfoFile": "dist/.tsbuildinfo", + + "strict": true, + "noUncheckedIndexedAccess": true, + "noImplicitOverride": true, + "noFallthroughCasesInSwitch": true, + "exactOptionalPropertyTypes": false, + "forceConsistentCasingInFileNames": true, + "skipLibCheck": true + }, + "include": ["src/**/*.ts", "test/**/*.ts"] +} diff --git a/scripts/generate_graphs.py b/scripts/generate_graphs.py index 177a7e3..4f14af8 100644 --- a/scripts/generate_graphs.py +++ b/scripts/generate_graphs.py @@ -71,6 +71,10 @@ # C# drivers "stackexchange-redis": "csharp", "valkey-glide-csharp": "csharp", + # Node.js drivers + "valkey-glide-node": "node", + "ioredis": "node", + "iovalkey": "node", # Python drivers (future) "redis-py": "python", "aioredis": "python", diff --git a/scripts/generate_interactive_graphs.py b/scripts/generate_interactive_graphs.py index 3523185..34af16b 100644 --- a/scripts/generate_interactive_graphs.py +++ b/scripts/generate_interactive_graphs.py @@ -680,6 +680,11 @@ def load_cpu_data(results_dir, rps_outlier_map=None): # C# (.NET) drivers — cyan/pink "stackexchange-redis": "#00ACC1", # cyan 600 "valkey-glide-csharp": "#D81B60", # pink 600 + + # Node.js drivers — yellows/browns + "valkey-glide-node": "#F9A825", # yellow 800 + "ioredis": "#6D4C41", # brown 600 + "iovalkey": "#546E7A", # blue grey 600 } DRIVER_FAMILIES = { @@ -694,15 +699,19 @@ def load_cpu_data(results_dir, rps_outlier_map=None): "redisson": "low-level", "stackexchange-redis": "csharp", "valkey-glide-csharp": "csharp", + "valkey-glide-node": "node", + "ioredis": "node", + "iovalkey": "node", } -FAMILY_ORDER = ["spring-data-valkey", "spring-data-redis", "low-level", "csharp"] +FAMILY_ORDER = ["spring-data-valkey", "spring-data-redis", "low-level", "csharp", "node"] FAMILY_LABELS = { "spring-data-valkey": "Spring Data Valkey", "spring-data-redis": "Spring Data Redis", "low-level": "Low-Level Java Drivers", "csharp": "C# (.NET) Drivers", + "node": "Node.js Drivers", } # Fallback color for unknown drivers diff --git a/scripts/run_benchmark_matrix.py b/scripts/run_benchmark_matrix.py index 4bfcf9d..f40ab6c 100644 --- a/scripts/run_benchmark_matrix.py +++ b/scripts/run_benchmark_matrix.py @@ -124,6 +124,10 @@ # C# drivers "stackexchange-redis": "csharp", "valkey-glide-csharp": "csharp", + # Node.js drivers + "valkey-glide-node": "node", + "ioredis": "node", + "iovalkey": "node", # Recording (default to java) "recording": "java", } From a8fb0c7fc0f35e5a77263fe8792c45367c742ad9 Mon Sep 17 00:00:00 2001 From: James Xin Date: Tue, 8 Sep 2026 16:56:48 -0700 Subject: [PATCH 2/5] Wire the Node engine into the sweep automation MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two gaps that made the engine unreachable from, or slow under, the matrix orchestrator and the AWS runner. Add configs/matrices/node-driver-comparison.json. Every shipped matrix was Java-only, so although DRIVER_ENGINE_MAP resolves the Node drivers correctly, no matrix actually exercised them — a default `bench-aws.sh` run would never touch the Node engine. Mirrors driver-comparison-defaults.json in shape: three drivers x five connection counts x five iterations. Make `node-build` idempotent via a stamp file. The orchestrator calls `make node-run` once per matrix cell, and `npm ci` deletes and reinstalls node_modules every time it runs: measured ~40s of pure reinstall per cell, so ~15 minutes wasted on a 75-cell sweep, plus 75 opportunities for a network blip to fail a cell mid-sweep on the AWS runner. `npm ci` now runs only when package.json/package-lock.json change; `npm run build` still runs every time because tsc is incremental and no-ops in under a second. Invocation cost drops from ~40s to ~1s. CI is unaffected — the workflow calls `npm ci && npm run build` directly, which is the right thing on a clean machine. Signed-off-by: James Xin --- Makefile | 21 +++++++++++++++++--- configs/matrices/node-driver-comparison.json | 20 +++++++++++++++++++ 2 files changed, 38 insertions(+), 3 deletions(-) create mode 100644 configs/matrices/node-driver-comparison.json diff --git a/Makefile b/Makefile index ee1d868..c83bb95 100644 --- a/Makefile +++ b/Makefile @@ -34,7 +34,7 @@ WORK_DIR=$(shell pwd)/work python-build python-test python-run python-clean \ ruby-build ruby-test ruby-run ruby-clean ruby-info \ csharp-build csharp-test csharp-run csharp-clean csharp-info \ - node-build node-test node-unit-test node-integration-test \ + node-build node-deps node-test node-unit-test node-integration-test \ node-run node-clean node-info \ config-editor-build config-editor-dev @@ -403,8 +403,23 @@ csharp-info: csharp-build # tsc emits into node/dist mirroring the source tree, so src/cli.ts -> dist/src/cli.js NODE_CLI=node/dist/src/cli.js -node-build: - cd node && npm ci && npm run build +# Stamp file so `npm ci` runs only when the manifests actually change. The matrix +# runner invokes `make node-run` once per cell, and `npm ci` deletes and +# reinstalls node_modules every time it runs — on a 75-cell sweep that is ~15 +# minutes of pure reinstall plus 75 chances for a network blip mid-sweep. The +# stamp lives inside node_modules, so wiping that directory correctly forces a +# reinstall. `npm run build` stays on every invocation: tsc is incremental and +# no-ops in well under a second. +NODE_DEPS_STAMP=node/node_modules/.resp-bench-deps-stamp + +$(NODE_DEPS_STAMP): node/package.json node/package-lock.json + cd node && npm ci + @touch $@ + +node-deps: $(NODE_DEPS_STAMP) + +node-build: node-deps + cd node && npm run build node-test: node-unit-test node-integration-test diff --git a/configs/matrices/node-driver-comparison.json b/configs/matrices/node-driver-comparison.json new file mode 100644 index 0000000..b9eefff --- /dev/null +++ b/configs/matrices/node-driver-comparison.json @@ -0,0 +1,20 @@ +{ + "description": "Compare Node.js drivers (GLIDE vs ioredis vs iovalkey) across client counts", + "x_axis": "connections", + "workload_template": "configs/workloads/reference/basic-standalone-single-client-10-secs.json", + "iterations": 5, + "dimensions": { + "connections": [ + 1, + 2, + 4, + 8, + 16 + ], + "driver_config": [ + "configs/drivers/default/valkey-glide-node.json", + "configs/drivers/default/ioredis.json", + "configs/drivers/default/iovalkey.json" + ] + } +} From d201227169b9d6204473e5d8bfd2c73c826f6057 Mon Sep 17 00:00:00 2001 From: James Xin Date: Mon, 14 Sep 2026 10:00:30 -0700 Subject: [PATCH 3/5] address comment: warmup allSettled, rate-limiter scope, exact dep pins MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Three review items from #27. Warmup used Promise.all, which rejects on the first failure while the remaining warmup loops keep running unawaited — so executePhase's finally ran closeClients() underneath them and they rejected against closed clients with nobody awaiting. Switched to Promise.allSettled and rethrow the first rejection once every loop has settled, preserving fail-fast with nothing left in flight. RateLimiter was constructed before warmup. Its constructor sets nextAllowedNanos to "now", so the whole warmup duration was banked as credit and the workload issued (warmup_duration / interval) requests back-to-back before pacing engaged — defeating the evenly-spaced, no-burst property the limiter exists for. Now constructed after warmup. Added a regression test sized so the burst would swallow the entire workload (~500ms warmup at a 20ms interval banks ~25 free requests; the phase issues 25), verified to fail at 676ms with the bug present and pass at ~980ms without it. Pinned all six dependencies exactly instead of using caret ranges, matching Java/C#/Ruby and PR #5's "pin build inputs". Pinned to the versions already resolved in the committed lockfile, so package-lock.json is unchanged and npm ci still succeeds. This matters more for a benchmark than for ordinary code: a caret range lets `npm install` quietly measure a different client build. Reviewers: jeremyprime, Aryex. Signed-off-by: James Xin --- node/package.json | 12 +++---- node/src/engine/benchmark.ts | 21 ++++++++++-- .../integration/recordingWorkload.test.ts | 33 +++++++++++++++++++ 3 files changed, 58 insertions(+), 8 deletions(-) diff --git a/node/package.json b/node/package.json index 35e1f7c..e2b8b39 100644 --- a/node/package.json +++ b/node/package.json @@ -18,13 +18,13 @@ "start": "node dist/src/cli.js" }, "dependencies": { - "@valkey/valkey-glide": "^2.5.2", - "hdr-histogram-js": "^3.0.1", - "ioredis": "^5.11.1", - "iovalkey": "^0.4.0" + "@valkey/valkey-glide": "2.5.2", + "hdr-histogram-js": "3.0.1", + "ioredis": "5.11.1", + "iovalkey": "0.4.0" }, "devDependencies": { - "@types/node": "^20.19.0", - "typescript": "^5.9.2" + "@types/node": "20.19.43", + "typescript": "5.9.3" } } diff --git a/node/src/engine/benchmark.ts b/node/src/engine/benchmark.ts index 67c5aeb..15a9d48 100644 --- a/node/src/engine/benchmark.ts +++ b/node/src/engine/benchmark.ts @@ -153,12 +153,18 @@ export class BenchmarkEngine { const collector = new MetricsCollector(); const clients = await this.createClients(phase); const commands = CommandFactory.createAll(phase.commands); - const rateLimiter = phase.hasRpsLimit() ? RateLimiter.create(phase.rpsLimit) : null; let status: string; try { if (phase.warmupRequests > 0) await this.warmup(clients, phase.warmupRequests); + // Created *after* warmup, deliberately. The limiter starts its clock at + // construction, so building it earlier would bank the whole warmup + // duration as credit and release a burst of + // (warmup_duration / interval) requests the moment the workload starts -- + // defeating the evenly-spaced, no-burst property the limiter exists for. + const rateLimiter = phase.hasRpsLimit() ? RateLimiter.create(phase.rpsLimit) : null; + collector.start(); status = await this.runWorkload(phase, clients, commands, rateLimiter, collector); collector.stop(); @@ -198,14 +204,20 @@ export class BenchmarkEngine { * * A dead or misconfigured server would otherwise produce a whole phase of * nothing but errors, which is far harder to diagnose than an upfront throw. + * + * Uses `allSettled`, not `all`: `all` rejects on the first failure while the + * remaining warmup loops keep running unawaited, so `executePhase`'s `finally` + * would close the clients underneath them. Settling every loop first means a + * warmup failure leaves nothing in flight. */ private async warmup(clients: BenchmarkClient[], warmupRequests: number): Promise { this.log.info(`Warmup: ${warmupRequests} PING(s) per client...`); // Warmup mode lets the recording driver suppress simulated errors, so an // error_rate workload is not aborted by the very errors it is measuring. for (const client of clients) client.setWarmupMode?.(true); + let outcomes: PromiseSettledResult[]; try { - await Promise.all( + outcomes = await Promise.allSettled( clients.map(async (client) => { for (let i = 0; i < warmupRequests; i++) { const result = await client.ping(); @@ -218,6 +230,11 @@ export class BenchmarkEngine { } finally { for (const client of clients) client.setWarmupMode?.(false); } + + const failure = outcomes.find((o): o is PromiseRejectedResult => o.status === 'rejected'); + if (failure !== undefined) { + throw failure.reason instanceof Error ? failure.reason : new Error(String(failure.reason)); + } this.log.info('Warmup completed'); } diff --git a/node/test/integration/recordingWorkload.test.ts b/node/test/integration/recordingWorkload.test.ts index 00a17ca..df67dbd 100644 --- a/node/test/integration/recordingWorkload.test.ts +++ b/node/test/integration/recordingWorkload.test.ts @@ -258,6 +258,39 @@ describe('recording-driver workload', () => { assert.ok(elapsed >= 400, `rps_limit was not enforced: ${elapsed}ms for 50 requests at 100/s`); }); + it('does not bank warmup time as rate-limiter credit', async () => { + // Regression: the limiter starts its clock at construction, so building it + // before warmup banked the whole warmup duration as credit and released a + // burst of (warmup_duration / interval) requests once the workload began. + // + // Sized so the burst would swallow the entire workload: warmup is 25 PINGs + // at 20ms = ~500ms of idle limiter time, and at 50 rps (a 20ms interval) + // that banks ~25 free requests -- every request this phase issues. A banked + // limiter therefore finishes in ~500ms (warmup only); a correctly-scoped one + // additionally paces 24 intervals, so ~980ms. + const started = Date.now(); + const { records } = await run({ + phases: [ + { + ...STEADY_PHASE, + connections: 4, + warmup_requests: 25, + rps_limit: 50, + completion: { type: 'requests', requests: 25 }, + }, + ], + specific: { operation_delay_micros: 20_000 }, + }); + const elapsed = Date.now() - started; + + assert.equal(records[0]!['totals'].requests, 25); + assert.ok( + elapsed >= 850, + `rate limiter released a warmup-banked burst: 25 requests at 50 rps after ` + + `a ~500ms warmup took only ${elapsed}ms (expected ~980ms)`, + ); + }); + it('gates connection setup with a cps_limit', async () => { const started = Date.now(); await run({ From c780f89654d98eafc0089c2a73a34b2e88cbf1cf Mon Sep 17 00:00:00 2001 From: James Xin Date: Mon, 14 Sep 2026 17:49:41 -0700 Subject: [PATCH 4/5] address comment: graph the Node results, and allow per-engine CI runs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two gaps in the CI wiring, found while trying to exercise benchmark-node. generate-graphs listed benchmark-node in `needs` and downloaded its artifacts, but had no Node graph step — only Java and Ruby. Node results were collected and then silently dropped. Added a Node step; DRIVER_LANGUAGE_MAP already maps the three Node driver ids to "node", so nothing else was needed. One step rather than three: the reference workload runs a single connection, and the Java/Ruby "10/100 Clients" steps all re-read that same 1-connection glob, so the extra copies would be duplicates of the same data. The workflow also took no inputs, so validating one engine meant running all of them — Java's 9 drivers plus Ruby's 2, each at 1M requests. Added an `engines` choice input (all/java/ruby/node, default all) gating each engine job and each graph step. generate-graphs needs `if: !cancelled()` because a skipped `needs` job would otherwise skip it too, which would produce no graphs at all on an engine-scoped run. Default is `all`, so the existing behaviour is unchanged. Signed-off-by: James Xin --- .github/workflows/benchmark.yml | 34 +++++++++++++++++++++++++++++++++ 1 file changed, 34 insertions(+) diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml index 13f3a85..a3dcbf8 100644 --- a/.github/workflows/benchmark.yml +++ b/.github/workflows/benchmark.yml @@ -2,9 +2,20 @@ name: Benchmark on: workflow_dispatch: + inputs: + engines: + description: 'Which engine benchmarks to run' + type: choice + default: all + options: + - all + - java + - ruby + - node jobs: benchmark-java: + if: ${{ inputs.engines == 'all' || inputs.engines == 'java' }} runs-on: ubuntu-latest strategy: @@ -85,6 +96,7 @@ jobs: retention-days: 30 benchmark-ruby: + if: ${{ inputs.engines == 'all' || inputs.engines == 'ruby' }} runs-on: ubuntu-latest strategy: @@ -153,6 +165,7 @@ jobs: retention-days: 30 benchmark-node: + if: ${{ inputs.engines == 'all' || inputs.engines == 'node' }} runs-on: ubuntu-latest strategy: @@ -227,6 +240,10 @@ jobs: generate-graphs: needs: [benchmark-java, benchmark-ruby, benchmark-node] + # A skipped `needs` job would skip this one by default, so an engine-scoped + # run would produce no graphs at all. Individual graph steps are gated on the + # same input instead. + if: ${{ !cancelled() }} runs-on: ubuntu-latest permissions: contents: write @@ -258,6 +275,7 @@ jobs: # TODO - rework with matrix orchistrator - name: Generate Java graphs - 1 Client + if: ${{ inputs.engines == 'all' || inputs.engines == 'java' }} run: | python scripts/generate_graphs.py \ --results "results/github-runner/reference/*-basic-standalone-single-client-1M-reqs.ndjson" \ @@ -268,6 +286,7 @@ jobs: --commit-id ${{ github.sha }} - name: Generate Java graphs - 10 Clients + if: ${{ inputs.engines == 'all' || inputs.engines == 'java' }} run: | python scripts/generate_graphs.py \ --results "results/github-runner/reference/*-basic-standalone-10-clients.ndjson" \ @@ -278,6 +297,7 @@ jobs: --commit-id ${{ github.sha }} - name: Generate Java graphs - 100 Clients + if: ${{ inputs.engines == 'all' || inputs.engines == 'java' }} run: | python scripts/generate_graphs.py \ --results "results/github-runner/reference/*-basic-standalone-100-clients.ndjson" \ @@ -289,6 +309,7 @@ jobs: # Ruby graphs — per concurrency level - name: Generate Ruby graphs - 1 Client + if: ${{ inputs.engines == 'all' || inputs.engines == 'ruby' }} run: | python scripts/generate_graphs.py \ --results "results/github-runner/reference/*-basic-standalone-single-client-1M-reqs.ndjson" \ @@ -299,6 +320,7 @@ jobs: --commit-id ${{ github.sha }} - name: Generate Ruby graphs - 10 Clients + if: ${{ inputs.engines == 'all' || inputs.engines == 'ruby' }} run: | python scripts/generate_graphs.py \ --results "results/github-runner/reference/*-basic-standalone-10-clients.ndjson" \ @@ -309,6 +331,7 @@ jobs: --commit-id ${{ github.sha }} - name: Generate Ruby graphs - 100 Clients + if: ${{ inputs.engines == 'all' || inputs.engines == 'ruby' }} run: | python scripts/generate_graphs.py \ --results "results/github-runner/reference/*-basic-standalone-100-clients.ndjson" \ @@ -318,6 +341,17 @@ jobs: --workload "Ruby - 100 Clients" \ --commit-id ${{ github.sha }} + - name: Generate Node.js graphs - 1 Client + if: ${{ inputs.engines == 'all' || inputs.engines == 'node' }} + run: | + python scripts/generate_graphs.py \ + --results "results/github-runner/reference/*-basic-standalone-single-client-1M-reqs.ndjson" \ + --output graphs/node/1-client/ \ + --phase STEADY \ + --language node \ + --workload "Node.js - 1 Client" \ + --commit-id ${{ github.sha }} + - name: Upload graphs uses: actions/upload-artifact@v4 with: From 46206d8e17f020b1470d189da34e80d6cb95ba18 Mon Sep 17 00:00:00 2001 From: James Xin Date: Mon, 14 Sep 2026 17:57:50 -0700 Subject: [PATCH 5/5] address comment: allow --language node in generate_graphs.py The Node graph step added in c780f89 failed in CI with: generate_graphs.py: error: argument --language: invalid choice: 'node' (choose from 'java', 'ruby', 'csharp', 'python') My omission: the earlier commit added the three Node driver ids to DRIVER_LANGUAGE_MAP but not to the argparse choices list, so --language node was rejected before the map was ever consulted. Added "node" to the choices and a comment noting the two lists must stay in sync. Verified by re-running the exact failing command against the artifacts from run 34914789751: 3 result records found, 9 graphs generated. Signed-off-by: James Xin --- scripts/generate_graphs.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/scripts/generate_graphs.py b/scripts/generate_graphs.py index 4f14af8..3f6a276 100644 --- a/scripts/generate_graphs.py +++ b/scripts/generate_graphs.py @@ -122,7 +122,10 @@ def parse_args(): ) parser.add_argument( "--language", - choices=["java", "ruby", "csharp", "python"], + # Keep in sync with the values in DRIVER_LANGUAGE_MAP — argparse rejects + # anything not listed here, so a language added to the map alone fails at + # the CLI rather than silently producing empty graphs. + choices=["java", "ruby", "csharp", "node", "python"], help="Filter results by language (only include drivers for this language)", ) parser.add_argument(