diff --git a/.github/benchmark-config/drivers.json b/.github/benchmark-config/drivers.json deleted file mode 100644 index 74c36f9..0000000 --- a/.github/benchmark-config/drivers.json +++ /dev/null @@ -1,21 +0,0 @@ -{ - "java": [ - "configs/drivers/default/jedis.json", - "configs/drivers/default/lettuce.json", - "configs/drivers/default/redisson.json", - "configs/drivers/default/spring-data-redis-jedis.json", - "configs/drivers/default/spring-data-redis-lettuce.json", - "configs/drivers/default/spring-data-valkey-glide-standalone.json", - "configs/drivers/default/spring-data-valkey-jedis-standalone.json", - "configs/drivers/default/spring-data-valkey-lettuce-standalone.json", - "configs/drivers/default/valkey-glide-standalone.json" - ], - "ruby": [ - "configs/drivers/default/redis-rb.json", - "configs/drivers/default/valkey-glide-ruby.json" - ], - "csharp": [ - "configs/drivers/default/stackexchange-redis.json", - "configs/drivers/default/valkey-glide-csharp.json" - ] -} diff --git a/.github/benchmark-config/workloads.json b/.github/benchmark-config/workloads.json deleted file mode 100644 index 3be6e36..0000000 --- a/.github/benchmark-config/workloads.json +++ /dev/null @@ -1,3 +0,0 @@ -[ - "configs/workloads/reference/basic-standalone-single-client-1M-reqs.json" -] diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml index 66147cd..9dfb357 100644 --- a/.github/workflows/benchmark.yml +++ b/.github/workflows/benchmark.yml @@ -2,9 +2,21 @@ name: Benchmark on: workflow_dispatch: + inputs: + engines: + description: 'Which engine benchmarks to run' + type: choice + default: all + options: + - all + - java + - ruby + - node + - php jobs: benchmark-java: + if: ${{ inputs.engines == 'all' || inputs.engines == 'java' }} runs-on: ubuntu-latest strategy: @@ -85,6 +97,7 @@ jobs: retention-days: 30 benchmark-ruby: + if: ${{ inputs.engines == 'all' || inputs.engines == 'ruby' }} runs-on: ubuntu-latest strategy: @@ -152,8 +165,202 @@ jobs: path: ${{ steps.names.outputs.result_file }} retention-days: 30 + test-php: + # Fast, server-free PHP engine tests. Uses the recording driver, so it needs + # neither the valkey_glide extension nor a running server. + if: ${{ inputs.engines == 'all' || inputs.engines == 'php' }} + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - name: Set up PHP + uses: shivammathur/setup-php@v2 + with: + php-version: '8.3' + extensions: json, pcntl + tools: composer + + - name: Install PHP dependencies + run: cd php && composer install --no-interaction --prefer-dist + + - name: Run PHP unit tests + run: cd php && vendor/bin/phpunit --testsuite unit + + - name: Run PHP integration tests (server-free) + run: cd php && vendor/bin/phpunit --testsuite integration + + benchmark-php: + if: ${{ inputs.engines == 'all' || inputs.engines == 'php' }} + runs-on: ubuntu-latest + + strategy: + fail-fast: false + matrix: + driver: + - configs/drivers/default/valkey-glide-php.json + workload: + - configs/workloads/reference/basic-standalone-single-client-1M-reqs.json + + steps: + - uses: actions/checkout@v4 + with: + submodules: recursive + + - name: Set up PHP + uses: shivammathur/setup-php@v2 + with: + php-version: '8.3' + extensions: json, pcntl, bcmath + tools: composer + + - name: Install build dependencies + run: | + sudo apt-get update + sudo apt-get install -y build-essential php-dev pkg-config \ + libssl-dev libffi-dev libprotobuf-c-dev protobuf-c-compiler unzip + + - name: Install Rust toolchain + uses: dtolnay/rust-toolchain@stable + + - name: Install cbindgen + run: cargo install cbindgen + + - name: Build & install valkey_glide extension + run: | + # Build the Valkey GLIDE PHP extension from the upstream repo. + git clone --recurse-submodules https://github.com/valkey-io/valkey-glide-php.git /tmp/vg-php + cd /tmp/vg-php + python3 utils/patch_proto_and_rust.py + (cd valkey-glide/ffi && cargo build --release) + phpize + ./configure --enable-valkey-glide + make build-modules-pre + make -j"$(nproc)" + sudo make install + echo "extension=valkey_glide" | sudo tee -a "$(php -i | grep '^Loaded Configuration File' | awk '{print $NF}')" + php -m | grep valkey_glide + + - name: Install PHP dependencies + run: cd php && composer install --no-interaction --prefer-dist --optimize-autoloader + + - name: Start Valkey server + run: | + make server-standalone-start + sleep 2 + work/valkey/bin/valkey-cli ping + work/valkey/bin/valkey-cli CONFIG GET save + + - name: Extract names for result file + id: names + run: | + DRIVER_NAME=$(basename ${{ matrix.driver }} .json) + WORKLOAD_NAME=$(basename ${{ matrix.workload }} .json) + echo "driver_name=$DRIVER_NAME" >> $GITHUB_OUTPUT + echo "workload_name=$WORKLOAD_NAME" >> $GITHUB_OUTPUT + echo "result_file=results/github-runner/reference/${DRIVER_NAME}-${WORKLOAD_NAME}.ndjson" >> $GITHUB_OUTPUT + + - name: Run benchmark + run: | + mkdir -p results/github-runner/reference + php php/bin/resp-bench \ + --server localhost:6379 \ + --driver ${{ matrix.driver }} \ + --workload ${{ matrix.workload }} \ + --metrics ${{ steps.names.outputs.result_file }} \ + --commit-id ${{ github.sha }} + + - name: Stop Valkey server + if: always() + run: | + make server-standalone-stop || true + + - name: Upload results + uses: actions/upload-artifact@v4 + with: + name: benchmark-php-${{ steps.names.outputs.driver_name }}-${{ steps.names.outputs.workload_name }} + path: ${{ steps.names.outputs.result_file }} + retention-days: 30 + + benchmark-node: + if: ${{ inputs.engines == 'all' || inputs.engines == 'node' }} + runs-on: ubuntu-latest + + strategy: + fail-fast: false + matrix: + driver: + - configs/drivers/default/valkey-glide-node.json + - configs/drivers/default/ioredis.json + - configs/drivers/default/iovalkey.json + workload: + - configs/workloads/reference/basic-standalone-single-client-1M-reqs.json + + steps: + - uses: actions/checkout@v4 + + # Node 22 LTS: ioredis 6 and node-redis 6 both require >= 20, and 18 is EOL. + - name: Set up Node.js + uses: actions/setup-node@v4 + with: + node-version: '22' + cache: npm + cache-dependency-path: node/package-lock.json + + - name: Install build dependencies + run: | + sudo apt-get update + sudo apt-get install -y build-essential + + - name: Start Valkey server + run: | + # Use Makefile target which builds from source and configures with persistence disabled + make server-standalone-start + # Wait for server to be ready + sleep 2 + # Verify server is up and persistence is disabled + work/valkey/bin/valkey-cli ping + work/valkey/bin/valkey-cli CONFIG GET save + + - name: Build Node.js benchmark + run: cd node && npm ci && npm run build + + - name: Extract names for result file + id: names + run: | + DRIVER_NAME=$(basename ${{ matrix.driver }} .json) + WORKLOAD_NAME=$(basename ${{ matrix.workload }} .json) + echo "driver_name=$DRIVER_NAME" >> $GITHUB_OUTPUT + echo "workload_name=$WORKLOAD_NAME" >> $GITHUB_OUTPUT + echo "result_file=results/github-runner/reference/${DRIVER_NAME}-${WORKLOAD_NAME}.ndjson" >> $GITHUB_OUTPUT + + - name: Run benchmark + run: | + mkdir -p results/github-runner/reference + node node/dist/src/cli.js \ + --server localhost:6379 \ + --driver ${{ matrix.driver }} \ + --workload ${{ matrix.workload }} \ + --metrics ${{ steps.names.outputs.result_file }} \ + --commit-id ${{ github.sha }} + + - name: Stop Valkey server + if: always() + run: | + make server-standalone-stop || true + + - name: Upload results + uses: actions/upload-artifact@v4 + with: + name: benchmark-node-${{ steps.names.outputs.driver_name }}-${{ steps.names.outputs.workload_name }} + path: ${{ steps.names.outputs.result_file }} + retention-days: 30 + generate-graphs: - needs: [benchmark-java, benchmark-ruby] + needs: [benchmark-java, benchmark-ruby, benchmark-php, benchmark-node] + # A skipped `needs` job would skip this one by default, so an engine-scoped + # run would produce no graphs at all. Individual graph steps are gated on the + # same input instead. + if: ${{ !cancelled() }} runs-on: ubuntu-latest permissions: contents: write @@ -169,7 +376,7 @@ jobs: - name: Install dependencies run: | - pip install matplotlib numpy + pip install -r scripts/requirements.txt - name: Download all artifacts uses: actions/download-artifact@v4 @@ -185,6 +392,7 @@ jobs: # TODO - rework with matrix orchistrator - name: Generate Java graphs - 1 Client + if: ${{ inputs.engines == 'all' || inputs.engines == 'java' }} run: | python scripts/generate_graphs.py \ --results "results/github-runner/reference/*-basic-standalone-single-client-1M-reqs.ndjson" \ @@ -195,6 +403,7 @@ jobs: --commit-id ${{ github.sha }} - name: Generate Java graphs - 10 Clients + if: ${{ inputs.engines == 'all' || inputs.engines == 'java' }} run: | python scripts/generate_graphs.py \ --results "results/github-runner/reference/*-basic-standalone-10-clients.ndjson" \ @@ -205,6 +414,7 @@ jobs: --commit-id ${{ github.sha }} - name: Generate Java graphs - 100 Clients + if: ${{ inputs.engines == 'all' || inputs.engines == 'java' }} run: | python scripts/generate_graphs.py \ --results "results/github-runner/reference/*-basic-standalone-100-clients.ndjson" \ @@ -216,6 +426,7 @@ jobs: # Ruby graphs โ€” per concurrency level - name: Generate Ruby graphs - 1 Client + if: ${{ inputs.engines == 'all' || inputs.engines == 'ruby' }} run: | python scripts/generate_graphs.py \ --results "results/github-runner/reference/*-basic-standalone-single-client-1M-reqs.ndjson" \ @@ -226,6 +437,7 @@ jobs: --commit-id ${{ github.sha }} - name: Generate Ruby graphs - 10 Clients + if: ${{ inputs.engines == 'all' || inputs.engines == 'ruby' }} run: | python scripts/generate_graphs.py \ --results "results/github-runner/reference/*-basic-standalone-10-clients.ndjson" \ @@ -236,6 +448,7 @@ jobs: --commit-id ${{ github.sha }} - name: Generate Ruby graphs - 100 Clients + if: ${{ inputs.engines == 'all' || inputs.engines == 'ruby' }} run: | python scripts/generate_graphs.py \ --results "results/github-runner/reference/*-basic-standalone-100-clients.ndjson" \ @@ -245,6 +458,17 @@ jobs: --workload "Ruby - 100 Clients" \ --commit-id ${{ github.sha }} + - name: Generate Node.js graphs - 1 Client + if: ${{ inputs.engines == 'all' || inputs.engines == 'node' }} + run: | + python scripts/generate_graphs.py \ + --results "results/github-runner/reference/*-basic-standalone-single-client-1M-reqs.ndjson" \ + --output graphs/node/1-client/ \ + --phase STEADY \ + --language node \ + --workload "Node.js - 1 Client" \ + --commit-id ${{ github.sha }} + - name: Upload graphs uses: actions/upload-artifact@v4 with: diff --git a/Makefile b/Makefile index eb1e22e..6658476 100644 --- a/Makefile +++ b/Makefile @@ -34,6 +34,9 @@ WORK_DIR=$(shell pwd)/work python-build python-test python-run python-clean \ ruby-build ruby-test ruby-run ruby-clean ruby-info \ csharp-build csharp-test csharp-run csharp-clean csharp-info \ + php-build php-test php-integration-test php-test-live php-run php-clean php-info \ + node-build node-deps node-test node-unit-test node-integration-test \ + node-run node-clean node-info \ config-editor-build config-editor-dev # ============================================================================ @@ -74,6 +77,13 @@ help: @echo " make csharp-clean Clean C# build artifacts" @echo " make csharp-info Show supported C# drivers and commands" @echo "" + @echo "Node.js Engine:" + @echo " make node-build Install deps and compile the Node.js engine" + @echo " make node-test Run Node.js tests (unit + integration)" + @echo " make node-run Run Node.js benchmark (requires DRIVER and WORKLOAD)" + @echo " make node-clean Clean Node.js build artifacts" + @echo " make node-info Show supported Node.js drivers and commands" + @echo "" @echo "Config Editor:" @echo " make config-editor-build Build config editor UI" @echo " make config-editor-dev Run config editor in development mode" @@ -387,6 +397,118 @@ csharp-clean: csharp-info: csharp-build dotnet run --project $(CSHARP_PROJECT) -c Release -- --info +# ============================================================================ +# ============================================================================ +# PHP Engine +# ============================================================================ + +# Prefer composer's autoloader; the CLI falls back to a minimal PSR-4 loader, +# so php-run/php-info work even before `composer install`. +PHP?=php + +php-build: + cd php && if command -v composer >/dev/null 2>&1; then \ + composer install --no-interaction --prefer-dist --optimize-autoloader; \ + else \ + echo "composer not found; using the bundled minimal autoloader (php/vendor/autoload.php)"; \ + fi + +php-test: + cd php && if [ -f vendor/bin/phpunit ]; then \ + vendor/bin/phpunit --testsuite unit; \ + elif [ -f /tmp/phpunit.phar ]; then \ + $(PHP) /tmp/phpunit.phar --testsuite unit; \ + else \ + echo "PHPUnit not installed. Run 'make php-build' (composer) or download phpunit.phar."; \ + exit 1; \ + fi + +php-integration-test: + cd php && if [ -f vendor/bin/phpunit ]; then \ + vendor/bin/phpunit --testsuite integration; \ + elif [ -f /tmp/phpunit.phar ]; then \ + $(PHP) /tmp/phpunit.phar --testsuite integration; \ + else \ + echo "PHPUnit not installed. Run 'make php-build' (composer) or download phpunit.phar."; \ + exit 1; \ + fi + +# Live integration tests for the valkey-glide-php driver. Requires the +# valkey_glide extension AND a reachable server (VALKEY_HOST/VALKEY_PORT, +# default localhost:6379). Tests skip cleanly if either is missing. +php-test-live: + cd php && if [ -f vendor/bin/phpunit ]; then \ + vendor/bin/phpunit --testsuite integration --filter 'LiveClientTest|PhpRedisLiveTest'; \ + elif [ -f /tmp/phpunit.phar ]; then \ + $(PHP) /tmp/phpunit.phar --testsuite integration --filter 'LiveClientTest|PhpRedisLiveTest'; \ + else \ + echo "PHPUnit not installed. Run 'make php-build' (composer) or download phpunit.phar."; \ + exit 1; \ + fi + +php-run: php-build + $(PHP) php/bin/resp-bench \ + --server $(SERVER) \ + --driver $(DRIVER) \ + --workload $(WORKLOAD) \ + --metrics $(METRICS_OUTPUT) + +php-clean: + cd php && rm -rf vendor composer.lock .phpunit.cache output + +php-info: + $(PHP) php/bin/resp-bench --info + +# ============================================================================ +# Node.js Engine +# ============================================================================ + +# tsc emits into node/dist mirroring the source tree, so src/cli.ts -> dist/src/cli.js +NODE_CLI=node/dist/src/cli.js + +# Stamp file so `npm ci` runs only when the manifests actually change. The matrix +# runner invokes `make node-run` once per cell, and `npm ci` deletes and +# reinstalls node_modules every time it runs โ€” on a 75-cell sweep that is ~15 +# minutes of pure reinstall plus 75 chances for a network blip mid-sweep. The +# stamp lives inside node_modules, so wiping that directory correctly forces a +# reinstall. `npm run build` stays on every invocation: tsc is incremental and +# no-ops in well under a second. +NODE_DEPS_STAMP=node/node_modules/.resp-bench-deps-stamp + +$(NODE_DEPS_STAMP): node/package.json node/package-lock.json + cd node && npm ci + @touch $@ + +node-deps: $(NODE_DEPS_STAMP) + +node-build: node-deps + cd node && npm run build + +node-test: node-unit-test node-integration-test + +node-unit-test: node-build + cd node && node --test dist/test/unit/ + +# The live-server tests skip themselves unless VALKEY_HOST is set, so the server +# has to be up before this runs. +node-integration-test: node-build server-standalone-start + sleep 1 + cd node && VALKEY_HOST=localhost VALKEY_PORT=6379 node --test dist/test/integration/ + $(MAKE) server-standalone-stop + +node-run: node-build + node $(NODE_CLI) \ + --server $(SERVER) \ + --driver $(DRIVER) \ + --workload $(WORKLOAD) \ + --metrics $(METRICS_OUTPUT) + +node-info: node-build + node $(NODE_CLI) --info + +node-clean: + cd node && rm -rf dist node_modules coverage + # ============================================================================ # Config Editor # ============================================================================ @@ -403,6 +525,10 @@ config-editor-dev: MATRIX?=configs/matrices/driver-comparison-high-tps.json GRAPHS_DIR?=graphs/interactive/ +# Results live in $(OUTPUT_DIR)//; the orchestrator points 'latest' at +# the most recent run. Override RUN_ID to graph a specific run. +RUN_ID?=latest +MATRIX_RESULTS_DIR=$(OUTPUT_DIR)/$(RUN_ID) benchmark-matrix: java-build python scripts/run_benchmark_matrix.py \ @@ -417,8 +543,10 @@ benchmark-matrix-dry-run: --dry-run benchmark-matrix-graphs: + @test -n "$(OUTPUT_DIR)" || { echo "ERROR: OUTPUT_DIR is required, e.g. make benchmark-matrix-graphs OUTPUT_DIR=results/my-run" >&2; exit 1; } + @test -d "$(MATRIX_RESULTS_DIR)" || { echo "ERROR: $(MATRIX_RESULTS_DIR) is not a directory โ€” no run has completed in $(OUTPUT_DIR), or 'latest' is stale. Run 'make benchmark-matrix OUTPUT_DIR=$(OUTPUT_DIR)' or pass RUN_ID=." >&2; exit 1; } python scripts/generate_interactive_graphs.py \ - $(OUTPUT_DIR) \ + $(MATRIX_RESULTS_DIR) \ --output $(GRAPHS_DIR) # ============================================================================ @@ -441,8 +569,8 @@ test-scripts-all: java-build # All Languages # ============================================================================ -build-all: java-build ruby-build csharp-build python-build +build-all: java-build ruby-build csharp-build node-build python-build -test-all: java-test ruby-test csharp-test python-test +test-all: java-test ruby-test csharp-test node-test python-test -clean-all: java-clean ruby-clean csharp-clean python-clean clean +clean-all: java-clean ruby-clean csharp-clean node-clean python-clean clean diff --git a/README.md b/README.md index 94cf01f..cf4d2ad 100644 --- a/README.md +++ b/README.md @@ -3,13 +3,23 @@ A multi-language benchmark suite for RESP protocol (Redis/Valkey) compatible databases and client libraries, with a matrix-based orchestration layer for multi-dimensional parameter sweeps and interactive graph generation. > ๐Ÿ“ See [docs/ARCHITECTURE.md](docs/ARCHITECTURE.md) for the full architecture diagram and component details. +> +> โ˜๏ธ Want numbers without setting up a host? See [infra/](infra/README.md) for +> fire-and-forget benchmarking on AWS: one command provisions an EC2 instance, +> runs a sweep, publishes the report to S3, and self-terminates. ## Quick Start ### Prerequisites - Python 3.8+, Java 21+, Maven +- Node.js 20+ (for the Node.js engine) - Make +- A server CLI (`valkey-cli`) for the matrix runner's readiness probe and per-cell + FLUSHALL โ€” the Makefile's `server-*` targets build one into + `work//bin/`, or any `valkey-cli`/`redis-cli` on `PATH` is used. + Set `RESP_BENCH_CLI` to point at a specific binary. Not needed for matrices that + only use the serverless `recording` driver. ### 1. Run a Benchmark Matrix @@ -25,16 +35,21 @@ make server-standalone-start python scripts/run_benchmark_matrix.py \ --matrix configs/matrices/driver-comparison-high-tps.json \ --output-dir results/my-run \ + --run-id first-try \ --server-host localhost +# Results land in results/my-run// (--run-id defaults to a UTC timestamp). +# Exit code: 0 = all cells ran, 1 = some cell failed, 2 = preflight failed. ``` ### 2. Generate Interactive Graphs ```bash python scripts/generate_interactive_graphs.py \ - results/my-run/ \ + results/my-run/first-try/ \ --output graphs/interactive/my-run/ \ --title "My Benchmark Run" +# Or, for whichever run finished most recently: +# make benchmark-matrix-graphs OUTPUT_DIR=results/my-run # Open graphs/interactive/my-run/scalability_and_delta.html in a browser ``` @@ -89,6 +104,8 @@ Thread-based system metrics collector that runs alongside benchmarks, collecting | Java | โœ… Ready | Jedis, Lettuce, Valkey-Glide, Redisson, Spring Data Valkey/Redis | | Ruby | โœ… Ready | redis-rb, valkey-glide-ruby | | C# | โœ… Ready | valkey-glide-csharp, StackExchange.Redis | +| PHP | โœ… Ready | valkey-glide-php, PHPRedis | +| Node.js | โœ… Ready | valkey-glide-node, ioredis, iovalkey | | Python | ๐Ÿšง Planned | redis-py, aioredis, valkey-glide | ## Project Structure @@ -119,6 +136,8 @@ resp-bench/ โ”œโ”€โ”€ java/ # Java benchmark engine โ”œโ”€โ”€ ruby/ # Ruby benchmark engine โ”œโ”€โ”€ csharp/ # C# (.NET 10) benchmark engine +โ”œโ”€โ”€ php/ # PHP benchmark engine +โ”œโ”€โ”€ node/ # Node.js (TypeScript) benchmark engine โ”œโ”€โ”€ docs/ โ”‚ โ”œโ”€โ”€ ARCHITECTURE.md # System architecture โ”‚ โ”œโ”€โ”€ BENCHMARK_MATRIX.md # Matrix orchestrator docs @@ -126,7 +145,9 @@ resp-bench/ โ”‚ โ”œโ”€โ”€ CONFIG_SPECIFICATION.md # Configuration format spec โ”‚ โ”œโ”€โ”€ BENCHMARKS_JAVA.md # Java benchmark details โ”‚ โ”œโ”€โ”€ BENCHMARKS_CSHARP.md # C# benchmark details -โ”‚ โ””โ”€โ”€ BENCHMARKS_RUBY.md # Ruby benchmark details +โ”‚ โ”œโ”€โ”€ BENCHMARKS_RUBY.md # Ruby benchmark details +โ”‚ โ”œโ”€โ”€ BENCHMARKS_PHP.md # PHP benchmark details +โ”‚ โ””โ”€โ”€ BENCHMARKS_NODE.md # Node.js benchmark details โ””โ”€โ”€ graphs/interactive/ # Generated HTML graphs ``` @@ -183,6 +204,8 @@ See [docs/CONFIG_SPECIFICATION.md](docs/CONFIG_SPECIFICATION.md) for full detail | `make java-test` | Run Java unit tests | | `make ruby-test` | Run Ruby tests | | `make csharp-test` | Run C# tests | +| `make php-test` | Run PHP unit tests | +| `make node-test` | Run Node.js tests (unit + integration) | ### Engines @@ -191,8 +214,11 @@ See [docs/CONFIG_SPECIFICATION.md](docs/CONFIG_SPECIFICATION.md) for full detail | `make java-run` | Run Java engine (DRIVER, WORKLOAD, SERVER) | | `make ruby-run` | Run Ruby engine (DRIVER, WORKLOAD, SERVER) | | `make csharp-run` | Run C# engine (DRIVER, WORKLOAD, SERVER) | +| `make php-run` | Run PHP engine (DRIVER, WORKLOAD, SERVER) | +| `make node-run` | Run Node.js engine (DRIVER, WORKLOAD, SERVER) | | `make java-build` | Build Java JAR | | `make csharp-build` | Build C# executable | +| `make node-build` | Install deps and compile the Node.js engine | ### Server Management diff --git a/config-editor/src/App.tsx b/config-editor/src/App.tsx index 7b8e89f..492df38 100644 --- a/config-editor/src/App.tsx +++ b/config-editor/src/App.tsx @@ -35,7 +35,12 @@ interface CommandConfig { data_size_bytes?: number } -const DRIVERS = ['jedis', 'lettuce', 'valkey-glide', 'redisson', 'spring-data-valkey', 'spring-data-redis'] +const DRIVERS = [ + // Java + 'jedis', 'lettuce', 'valkey-glide', 'redisson', 'spring-data-valkey', 'spring-data-redis', + // Node.js + 'valkey-glide-node', 'ioredis', 'iovalkey', +] const COMMANDS = ['set', 'get', 'ping'] const ALGORITHMS = ['sequential_int', 'uniform_rand'] diff --git a/configs/drivers/default/ioredis.json b/configs/drivers/default/ioredis.json new file mode 100644 index 0000000..8261f9d --- /dev/null +++ b/configs/drivers/default/ioredis.json @@ -0,0 +1,7 @@ +{ + "schema_version": "1.0", + "description": "ioredis client - default configuration", + "driver_id": "ioredis", + "mode": "standalone", + "specific_driver_config": {} +} diff --git a/configs/drivers/default/iovalkey.json b/configs/drivers/default/iovalkey.json new file mode 100644 index 0000000..e4793ec --- /dev/null +++ b/configs/drivers/default/iovalkey.json @@ -0,0 +1,7 @@ +{ + "schema_version": "1.0", + "description": "iovalkey client - default configuration", + "driver_id": "iovalkey", + "mode": "standalone", + "specific_driver_config": {} +} diff --git a/configs/drivers/default/phpredis.json b/configs/drivers/default/phpredis.json new file mode 100644 index 0000000..8ce61cc --- /dev/null +++ b/configs/drivers/default/phpredis.json @@ -0,0 +1,7 @@ +{ + "schema_version": "1.0", + "description": "PHPRedis (ext-redis) - default configuration", + "driver_id": "phpredis", + "mode": "standalone", + "specific_driver_config": {} +} diff --git a/configs/drivers/default/valkey-glide-node.json b/configs/drivers/default/valkey-glide-node.json new file mode 100644 index 0000000..bc25ae1 --- /dev/null +++ b/configs/drivers/default/valkey-glide-node.json @@ -0,0 +1,7 @@ +{ + "schema_version": "1.0", + "description": "Valkey GLIDE for Node.js client - default configuration", + "driver_id": "valkey-glide-node", + "mode": "standalone", + "specific_driver_config": {} +} diff --git a/configs/drivers/default/valkey-glide-php.json b/configs/drivers/default/valkey-glide-php.json new file mode 100644 index 0000000..5d79433 --- /dev/null +++ b/configs/drivers/default/valkey-glide-php.json @@ -0,0 +1,7 @@ +{ + "schema_version": "1.0", + "description": "Valkey GLIDE PHP - default configuration", + "driver_id": "valkey-glide-php", + "mode": "standalone", + "specific_driver_config": {} +} diff --git a/configs/drivers/example-ioredis-standalone.json b/configs/drivers/example-ioredis-standalone.json new file mode 100644 index 0000000..9b594f5 --- /dev/null +++ b/configs/drivers/example-ioredis-standalone.json @@ -0,0 +1,7 @@ +{ + "schema_version": "1.0", + "description": "ioredis client, standalone mode", + "driver_id": "ioredis", + "mode": "standalone", + "specific_driver_config": {} +} diff --git a/configs/drivers/example-iovalkey-standalone.json b/configs/drivers/example-iovalkey-standalone.json new file mode 100644 index 0000000..978fa32 --- /dev/null +++ b/configs/drivers/example-iovalkey-standalone.json @@ -0,0 +1,7 @@ +{ + "schema_version": "1.0", + "description": "iovalkey client, standalone mode", + "driver_id": "iovalkey", + "mode": "standalone", + "specific_driver_config": {} +} diff --git a/configs/drivers/example-phpredis-standalone.json b/configs/drivers/example-phpredis-standalone.json new file mode 100644 index 0000000..225300c --- /dev/null +++ b/configs/drivers/example-phpredis-standalone.json @@ -0,0 +1,7 @@ +{ + "schema_version": "1.0", + "description": "PHPRedis (ext-redis) client - standalone mode", + "driver_id": "phpredis", + "mode": "standalone", + "specific_driver_config": {} +} diff --git a/configs/drivers/example-valkey-glide-node-standalone.json b/configs/drivers/example-valkey-glide-node-standalone.json new file mode 100644 index 0000000..5c5bb3d --- /dev/null +++ b/configs/drivers/example-valkey-glide-node-standalone.json @@ -0,0 +1,7 @@ +{ + "schema_version": "1.0", + "description": "Valkey GLIDE for Node.js client, standalone mode", + "driver_id": "valkey-glide-node", + "mode": "standalone", + "specific_driver_config": {} +} diff --git a/configs/drivers/example-valkey-glide-php-standalone.json b/configs/drivers/example-valkey-glide-php-standalone.json new file mode 100644 index 0000000..1a1c121 --- /dev/null +++ b/configs/drivers/example-valkey-glide-php-standalone.json @@ -0,0 +1,7 @@ +{ + "schema_version": "1.0", + "description": "Valkey GLIDE PHP client - standalone mode", + "driver_id": "valkey-glide-php", + "mode": "standalone", + "specific_driver_config": {} +} diff --git a/configs/drivers/high-throughput/ioredis.json b/configs/drivers/high-throughput/ioredis.json new file mode 100644 index 0000000..d214e79 --- /dev/null +++ b/configs/drivers/high-throughput/ioredis.json @@ -0,0 +1,8 @@ +{ + "schema_version": "1.0", + "description": "ioredis client - high-throughput configuration", + "driver_id": "ioredis", + "mode": "standalone", + "specific_driver_config": {}, + "command_timeout_ms": 10000 +} diff --git a/configs/drivers/high-throughput/iovalkey.json b/configs/drivers/high-throughput/iovalkey.json new file mode 100644 index 0000000..6a1d347 --- /dev/null +++ b/configs/drivers/high-throughput/iovalkey.json @@ -0,0 +1,8 @@ +{ + "schema_version": "1.0", + "description": "iovalkey client - high-throughput configuration", + "driver_id": "iovalkey", + "mode": "standalone", + "specific_driver_config": {}, + "command_timeout_ms": 10000 +} diff --git a/configs/drivers/high-throughput/phpredis.json b/configs/drivers/high-throughput/phpredis.json new file mode 100644 index 0000000..bf52065 --- /dev/null +++ b/configs/drivers/high-throughput/phpredis.json @@ -0,0 +1,7 @@ +{ + "schema_version": "1.0", + "description": "PHPRedis (ext-redis) - high-throughput configuration", + "driver_id": "phpredis", + "mode": "standalone", + "specific_driver_config": {} +} diff --git a/configs/drivers/high-throughput/valkey-glide-node.json b/configs/drivers/high-throughput/valkey-glide-node.json new file mode 100644 index 0000000..abc7a6a --- /dev/null +++ b/configs/drivers/high-throughput/valkey-glide-node.json @@ -0,0 +1,8 @@ +{ + "schema_version": "1.0", + "description": "Valkey GLIDE for Node.js client - high-throughput configuration", + "driver_id": "valkey-glide-node", + "mode": "standalone", + "specific_driver_config": {}, + "command_timeout_ms": 10000 +} diff --git a/configs/drivers/high-throughput/valkey-glide-php.json b/configs/drivers/high-throughput/valkey-glide-php.json new file mode 100644 index 0000000..c800749 --- /dev/null +++ b/configs/drivers/high-throughput/valkey-glide-php.json @@ -0,0 +1,7 @@ +{ + "schema_version": "1.0", + "description": "Valkey GLIDE PHP - high-throughput configuration", + "driver_id": "valkey-glide-php", + "mode": "standalone", + "specific_driver_config": {} +} diff --git a/configs/matrices/node-driver-comparison.json b/configs/matrices/node-driver-comparison.json new file mode 100644 index 0000000..b9eefff --- /dev/null +++ b/configs/matrices/node-driver-comparison.json @@ -0,0 +1,20 @@ +{ + "description": "Compare Node.js drivers (GLIDE vs ioredis vs iovalkey) across client counts", + "x_axis": "connections", + "workload_template": "configs/workloads/reference/basic-standalone-single-client-10-secs.json", + "iterations": 5, + "dimensions": { + "connections": [ + 1, + 2, + 4, + 8, + 16 + ], + "driver_config": [ + "configs/drivers/default/valkey-glide-node.json", + "configs/drivers/default/ioredis.json", + "configs/drivers/default/iovalkey.json" + ] + } +} diff --git a/configs/matrices/php-driver-comparison.json b/configs/matrices/php-driver-comparison.json new file mode 100644 index 0000000..bcccd08 --- /dev/null +++ b/configs/matrices/php-driver-comparison.json @@ -0,0 +1,19 @@ +{ + "description": "Compare PHP clients (Valkey GLIDE PHP vs PHPRedis) across client counts", + "x_axis": "connections", + "workload_template": "configs/workloads/reference/basic-standalone-single-client-1M-reqs.json", + "iterations": 5, + "dimensions": { + "connections": [ + 1, + 2, + 4, + 8, + 16 + ], + "driver_config": [ + "configs/drivers/high-throughput/valkey-glide-php.json", + "configs/drivers/high-throughput/phpredis.json" + ] + } +} diff --git a/configs/matrices/valkey-glide-basic-multiclients.json b/configs/matrices/valkey-glide-basic-multiclients.json index 4993982..22ff78a 100644 --- a/configs/matrices/valkey-glide-basic-multiclients.json +++ b/configs/matrices/valkey-glide-basic-multiclients.json @@ -1,7 +1,7 @@ { "description": "Valkey-Glide basic example for small connection scaling", "x_axis": "connections", - "workload_template": "configs/workloads/reference/basic-standalone-single-client-10M-secs.json", + "workload_template": "configs/workloads/reference/basic-standalone-single-client-10-secs.json", "iterations": 5, "dimensions": { "connections": [ diff --git a/configs/schemas/driver-config.schema.json b/configs/schemas/driver-config.schema.json index e06cf4e..c20cccc 100644 --- a/configs/schemas/driver-config.schema.json +++ b/configs/schemas/driver-config.schema.json @@ -18,7 +18,7 @@ "driver_id": { "type": "string", "description": "Identifier for the client driver implementation", - "examples": ["jedis", "lettuce", "valkey-glide", "redisson", "stackexchange-redis", "valkey-glide-csharp", "redis-py", "aioredis"] + "examples": ["jedis", "lettuce", "valkey-glide", "redisson", "stackexchange-redis", "valkey-glide-csharp", "valkey-glide-node", "ioredis", "iovalkey", "redis-py", "aioredis"] }, "mode": { "type": "string", diff --git a/docs/ADDING_LANGUAGE.md b/docs/ADDING_LANGUAGE.md index c8adab4..5ddb8c8 100644 --- a/docs/ADDING_LANGUAGE.md +++ b/docs/ADDING_LANGUAGE.md @@ -7,7 +7,10 @@ This guide explains how to add support for a new programming language to resp-be Before adding a new language, ensure you understand: - [Architecture](ARCHITECTURE.md) - Overall system design - [Configuration Specification](CONFIG_SPECIFICATION.md) - Config format details -- Existing implementations (Java is the reference implementation) +- Existing implementations: **Java is the reference implementation**; Ruby, C# and + Node.js follow it. For an async/event-loop language, `node/` is the closest model + and its README documents the parity traps worth knowing up front (RNG width, HDR + encoding, percentile-vs-raw min/max, shared request budget). ## Step-by-Step Guide @@ -178,9 +181,10 @@ class MetricsCollector: def record(self, command: str, latency_us: int, success: bool) -> None: if command not in self.command_metrics: - # 1ยตs to 1 hour, 3 significant figures + # 1ยตs to 600s, 3 significant figures (must match the other engines: + # Java/C#/Ruby/Node all use a max of 600_000_000ยตs, not 1 hour) self.command_metrics[command] = CommandMetrics( - histogram=HdrHistogram(1, 3600000000, 3) + histogram=HdrHistogram(1, 600000000, 3) ) metrics = self.command_metrics[command] @@ -389,9 +393,28 @@ Before submitting a new language engine: - [ ] Config parsing handles all schema fields - [ ] Key generator produces identical sequences (test with seed=12345) +- [ ] Keys are zero-padded to `key_size_bytes` (Java: `"%0" + max(1, key_size_bytes - prefix.length) + "d"`) +- [ ] The `uniform_rand` PRNG matches `java.util.Random` **including** the int32 + overflow check in `nextInt`'s rejection branch. Watch the arithmetic width: + the 48-bit LCG multiply exceeds what a double-based number type holds exactly + (this bites JavaScript, where `BigInt` is required) +- [ ] `sequential_int` uses a counter **shared** across all workers; `uniform_rand` + uses a per-worker PRNG seeded `base_seed + worker_index` +- [ ] The request budget is **shared** across workers and claimed one request at a + time, not pre-divided per worker (Java: one `AtomicLong` per phase) +- [ ] PING does **not** consume a generated key (Java's `PingCommand` ignores the + key generator, so consuming one shifts every subsequent key) - [ ] Rate limiter achieves target rates within 5% tolerance - [ ] Metrics output matches NDJSON schema exactly -- [ ] HdrHistogram produces compatible base64 payloads +- [ ] HdrHistogram produces compatible base64 payloads. If your library's encode + already returns base64, use it **directly** โ€” encoding it again yields a + payload Java and Ruby cannot decode +- [ ] `summary.min`/`max` match Java's `getMinValue()`/`getMaxValue()`, which return + the *bucket's* equivalent bounds. Many ports expose a raw min/max property + instead; those diverge above ~1000ยตs at 3 significant figures. Prefer + `getValueAtPercentile(0)` / `getValueAtPercentile(100)` +- [ ] `warmup_requests`, `cps_limit`, `rps_limit`, `pipeline_depth`, + `command_timeout_ms`, `tls` and `auth` are all actually honoured, not just parsed - [ ] All unit tests pass - [ ] Integration tests pass against live server - [ ] Documentation complete diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md index 6463d7e..f1ac564 100644 --- a/docs/ARCHITECTURE.md +++ b/docs/ARCHITECTURE.md @@ -156,6 +156,18 @@ class BenchmarkClient(ABC): async def close(self): ... ``` +**Node.js (TypeScript):** +```typescript +interface BenchmarkClient { + connect(host: string, port: number, config: DriverConfig): Promise; + ping(): Promise>; + get(key: string): Promise>; + set(key: string, value: Buffer): Promise>; + close(): Promise; + driverVersion(): string; +} +``` + ## Metrics Output Format All engines produce identical NDJSON output: @@ -236,10 +248,15 @@ Different languages use appropriate concurrency primitives: | C# | Task-per-client with async/await (.NET 8+) | | Python | asyncio with async/await | | Go | goroutines and channels | -| Node.js | Promise/async-await | +| Node.js | One event loop, worker-per-connection (Promise/async-await) | The key requirement is that N connections can operate concurrently, each potentially with pipeline_depth in-flight requests. +**Node.js caveat:** the engine is single-threaded, so one CPU core bounds the whole +run. Past that point measurements reflect the engine rather than the client. See +[BENCHMARKS_NODE.md](BENCHMARKS_NODE.md) ยง "The Single-Core Ceiling" for the +measured plateau and how to tell when you have hit it. + ## Parallel Command Issuers (Java) At high connection counts (128+), a single command-issuing thread becomes a CPU bottleneck โ€” saturating one core on semaphore contention, key generation (`String.format()`), and round-robin scanning. To address this, the Java engine supports **parallel command issuer threads** that partition client connections across multiple threads. diff --git a/docs/BENCHMARKS_NODE.md b/docs/BENCHMARKS_NODE.md new file mode 100644 index 0000000..00e2a4b --- /dev/null +++ b/docs/BENCHMARKS_NODE.md @@ -0,0 +1,147 @@ +# Node.js Benchmarks + +Details specific to the Node.js engine (`node/`). See [ARCHITECTURE.md](ARCHITECTURE.md) +for the cross-engine design and [../node/README.md](../node/README.md) for build and +usage. + +## Drivers + +| `driver_id` | Package | Notes | +|---|---|---| +| `valkey-glide-node` | `@valkey/valkey-glide` | Valkey GLIDE for Node.js. Prebuilt native binaries per platform. | +| `ioredis` | `ioredis` | The most widely used Node.js Redis client. | +| `iovalkey` | `iovalkey` | The Valkey-maintained fork of ioredis; API-identical. | +| `recording` | โ€” | Synthetic-latency client for server-free testing. | + +The GLIDE id is `valkey-glide-node`, **not** `valkey-glide` โ€” the latter belongs to +the Java engine in the global `DRIVER_ENGINE_MAP`. + +## Concurrency Model + +One event loop, one client per connection, one worker per connection. An awaited +command parks its worker, not the loop, so other connections keep progressing โ€” +the analogue of Java's virtual-thread-per-client design. + +`pipeline_depth > 1` gives each connection that many independent +issue/await/record slots, so a settled request is replaced immediately rather than +waiting for a batch. + +The per-phase request budget is shared across workers and claimed one request at a +time (matching Java's `AtomicLong`), so a slow connection cannot cap the run. + +## The Single-Core Ceiling + +**Node runs the whole engine on one thread, so one CPU core is the hard ceiling.** +This is the single most important caveat when comparing Node numbers to Java or C#, +which spread issuing across threads. + +Measured locally (Apple M-series laptop, Valkey 8 co-located on the same host, +100% GET, 512B values, 10k keys, 5s phases). **Absolute numbers here are not +publishable results** โ€” client and server contend for the same cores. The *shape* +is the point: + +Throughput vs connections (`pipeline_depth=1`): + +| Connections | glide RPS | ioredis RPS | iovalkey RPS | +|---|---|---|---| +| 1 | 1,139 | 1,584 | 1,646 | +| 10 | 8,424 | 8,467 | 10,360 | +| 50 | 25,065 | 23,071 | 26,268 | +| 100 | 33,639 | 34,898 | 35,087 | +| 200 | 41,812 | 42,748 | 41,144 | + +Doubling 100 โ†’ 200 connections buys only ~20% more throughput while p50 latency +roughly doubles (2.6ms โ†’ 4.5ms). That plateau is the event loop saturating, not the +clients. + +Throughput vs `pipeline_depth` at 10 connections (ioredis), with process CPU: + +| `pipeline_depth` | RPS | p50 | p99 | CPU (of one core) | +|---|---|---|---|---| +| 1 | 10,124 | 929ยตs | 2,055ยตs | 29% | +| 4 | 32,194 | 1,157ยตs | 2,731ยตs | 52% | +| 16 | 72,631 | 1,955ยตs | 4,089ยตs | 99% | + +CPU rises in lockstep with throughput and pins at 99% of a single core, where +throughput stops scaling. **The engine, not the client, is the limit past that +point.** + +Practical guidance: + +- Prefer raising `pipeline_depth` over raising `connections` to reach high + throughput on Node โ€” it is far cheaper per unit of RPS. +- When comparing Node against Java/C#, check whether the Node process is CPU-bound. + If it is at ~100% of a core, you are measuring the engine, not the driver. +- Java addresses the same ceiling with parallel command-issuer threads (see + [ARCHITECTURE.md](ARCHITECTURE.md) ยง "Parallel Command Issuers"). The Node + equivalent would be `worker_threads` with a client partition per worker. That is + deliberately **not** implemented โ€” it is a follow-up, to be justified by + measurements rather than assumed. + +## Fairness Controls + +Node-specific hazards with no analogue in the other engines, each handled +explicitly: + +- **Auto-pipelining is forced off** (`enableAutoPipelining: false`). ioredis and + iovalkey can transparently batch commands issued in the same event-loop tick, + which would inflate throughput against every other engine while looking like a + driver win. +- **Reconnects are disabled** (`retryStrategy: () => null`). ioredis' default + retries forever, so a wrong host would hang a run rather than fail it, and a + mid-phase reconnect would fold connection setup into request latency. +- **Response decoding is uniform.** `get` returns a `string` in all three drivers, + so none is charged for a different amount of decoding. GLIDE returns strings by + default; we do not opt one driver into bytes. +- **SET payloads are allocated once** per command object, so GC churn from payload + construction is not attributed to the driver. +- **Sub-millisecond rate limits work.** `setTimeout` clamps to ~1ms, so the limiter + yields via `setImmediate` below that. A 100k rps limit is a 10ยตs interval; a + timer-based wait would undershoot by ~100ร—. +- **Memory is not comparable to the JVM.** The system monitor's RSS samples include + V8 heap growth, which grows and collects on a different schedule from the JVM's. + +## Cross-Engine Parity + +Verified against the Java reference rather than assumed: + +- **Key sequences are byte-identical to Java.** 79,000 keys diffed against Java's + real `KeyGenerator` across `sequential_int` and `uniform_rand`, 1โ€“16 workers, + prime `keys_count`, and prefix-width edge cases. `javaRandom.ts` ports + `java.util.Random` with `BigInt` (the 48-bit LCG multiply reaches ~2^83, past + what a JS `number` holds exactly) and is anchored to + `new Random(0).nextInt() == -1155484576`. +- **HDR payloads decode in Java.** `payload_b64` is + `encodeIntoCompressedBase64()` used directly โ€” it is already base64, so encoding + it again would produce something Java cannot read. Verified by decoding a + Node-produced payload with `org.HdrHistogram.Histogram`: identical count and + percentiles. Range `(1, 600_000_000, 3)`, as in every engine. +- **`summary.min`/`max` match Java's quantization.** Java's `getMinValue()`/ + `getMaxValue()` return the bucket's equivalent bounds, while hdr-histogram-js' + `minNonZeroValue`/`maxValue` return the raw sample โ€” they diverge above ~1000ยตs + (50000ยตs recorded reads back as 50015 in Java). The engine uses + `getValueAtPercentile(0)` and `getValueAtPercentile(100)`, which match Java exactly. +- **PING does not consume a key**, matching Java's `PingCommand`, so mixing PING + into a workload does not shift the key sequence other engines would produce. + +## Reproducing the Numbers Above + +```bash +make server-standalone-start + +# Populate the keyspace, then sweep connections. +make node-run \ + DRIVER=configs/drivers/default/ioredis.json \ + WORKLOAD=configs/workloads/reference/basic-standalone-single-client-1M-reqs.json \ + METRICS_OUTPUT=output/node-ioredis.ndjson + +make server-standalone-stop +``` + +For a full matrix across drivers and connection counts, use the orchestrator โ€” +Node drivers are registered in `DRIVER_ENGINE_MAP`, so it dispatches to +`make node-run` automatically: + +```bash +python scripts/run_benchmark_matrix.py --matrix configs/matrices/.json +``` diff --git a/docs/BENCHMARKS_PHP.md b/docs/BENCHMARKS_PHP.md new file mode 100644 index 0000000..40b6dba --- /dev/null +++ b/docs/BENCHMARKS_PHP.md @@ -0,0 +1,69 @@ +# PHP Client Benchmarks โ€” Full Details + +Performance benchmarking of PHP client libraries against RESP-compatible servers, +using the resp-bench PHP engine. + +> **Note**: Graphs are generated by CI once benchmark runs are published. Until +> then, this document describes the engine, drivers, and workload. See +> [../php/README.md](../php/README.md) for engine internals. + +## Drivers Tested + +| Driver | Package | Description | +|--------|---------|-------------| +| `valkey-glide-php` | [ext-valkey_glide](https://github.com/valkey-io/valkey-glide-php) | Valkey GLIDE PHP client โ€” a native PHP extension backed by a Rust core, exposing a PHPRedis-compatible API | + +The `recording` driver is used for server-free CI tests and is not part of the +performance comparison. + +## Engine Notes + +- **Language/runtime**: PHP 8.2+ (GLIDE extension tested on 8.2/8.3). +- **Concurrency**: process-per-connection via `pcntl` (capped at 256 workers). + Each worker owns one client connection, opened *after* forking, and keeps a + single request in flight โ€” matching the "one in-flight request per connection" + model used by the Java, Ruby, C#, and Node engines so results are comparable. +- **Metrics**: HdrHistogram range `(1, 600_000_000, 3)`; per-worker histograms + are merged losslessly in the parent process and emitted as Java-compatible V2 + compressed base64 payloads. +- **Rate limiting**: leaky-bucket, with phase-level `rps_limit` divided across + workers. + +## Prerequisites for live runs + +- The `valkey_glide` PHP extension installed and enabled (`extension=valkey_glide`). +- `ext-pcntl` (standard on Linux/macOS CLI PHP). + +See [../php/README.md](../php/README.md#installing-the-valkey-glide-php-extension) +for installation. + +## Running + +```bash +make server-standalone-start +make php-run \ + DRIVER=configs/drivers/example-valkey-glide-php-standalone.json \ + WORKLOAD=configs/workloads/reference/basic-standalone-single-client-1M-reqs.json \ + SERVER=localhost:6379 \ + METRICS_OUTPUT=output/php.ndjson +``` + +Or through the matrix orchestrator (the `valkey-glide-php` driver id maps to the +`php` engine): + +```bash +python scripts/run_benchmark_matrix.py \ + --matrix configs/matrices/.json \ + --output-dir results/php-run \ + --server-host localhost +``` + +## Workload Configuration + +PHP runs use the same reference workloads as the other engines (see +`configs/workloads/reference/`), so cross-language comparisons are apples-to-apples: +- 1M requests per steady phase +- 1M keys, 16-byte keys, `bench:` prefix +- 512-byte SET values +- GET/SET mix via uniform-random key selection +- No rate limiting for max-throughput runs diff --git a/docs/BENCHMARK_MATRIX.md b/docs/BENCHMARK_MATRIX.md index 6bddd7a..ca0bb16 100644 --- a/docs/BENCHMARK_MATRIX.md +++ b/docs/BENCHMARK_MATRIX.md @@ -13,15 +13,16 @@ python scripts/run_benchmark_matrix.py \ --output-dir results/valkey-glide-sweep \ --dry-run -# Run the matrix benchmark +# Run the matrix benchmark (results land in results/valkey-glide-sweep//) python scripts/run_benchmark_matrix.py \ --matrix configs/matrices/valkey-glide-thread-sweep.json \ --output-dir results/valkey-glide-sweep \ + --run-id first-try \ --server-host 10.0.0.5 -# Generate interactive graphs from results +# Generate interactive graphs from one run's results python scripts/generate_interactive_graphs.py \ - results/valkey-glide-sweep/ \ + results/valkey-glide-sweep/first-try/ \ --output graphs/interactive/valkey-glide-sweep/ ``` @@ -30,6 +31,11 @@ Or via Makefile: make benchmark-matrix-dry-run MATRIX=configs/matrices/valkey-glide-thread-sweep.json make benchmark-matrix MATRIX=configs/matrices/valkey-glide-thread-sweep.json \ OUTPUT_DIR=results/glide-sweep SERVER_HOST=10.0.0.5 + +# Graphs for the run that just finished (follows OUTPUT_DIR/latest) +make benchmark-matrix-graphs OUTPUT_DIR=results/glide-sweep +# ...or for a specific run +make benchmark-matrix-graphs OUTPUT_DIR=results/glide-sweep RUN_ID=20260321T140322Z ``` ## Matrix Config Format @@ -136,19 +142,77 @@ Non-matching drivers skip the dimension entirely, avoiding wasted benchmark time ## Output Format -The matrix runner produces a **flat directory**: +The matrix runner produces a **flat directory per run**, under `//`: ``` results/glide-sweep/ - spring-data-valkey-glide@cb=8,tw=8,pool_size=connections.ndjson - spring-data-valkey-glide@cb=16,tw=16,pool_size=connections.ndjson - *.cpu.ndjson # CPU samples per variant - _manifest.json # Maps labels โ†’ config metadata + 20260321T140322Z/ + spring-data-valkey-glide@cb=8,tw=8,pool_size=connections.ndjson + spring-data-valkey-glide@cb=16,tw=16,pool_size=connections.ndjson + *.cpu.ndjson # CPU samples per variant + _manifest.json # Maps labels โ†’ config metadata + per-cell outcomes + latest -> 20260321T140322Z # symlink to the most recent successful start ``` +The run id defaults to a UTC timestamp, so two runs into the same `--output-dir` +never merge into the same NDJSON files. Pass `--run-id` to name a run yourself; +if that run directory already holds results, the run is refused unless you pass +`--resume` (append deliberately) or `--overwrite` (discard them first). Point the +graph generator at the run directory, not at `--output-dir`. + +Once preflight passes, the orchestrator repoints `/latest` at the +current run, so tooling can find the newest results without knowing the run id +(`--resume` repoints it at the run being appended to). A failed preflight leaves +the link on the previous run, and a `latest` that is a real directory rather than +a symlink is never touched. + Each `.ndjson` file contains STEADY phase records for ALL connection counts (multiple iterations each). The NDJSON format is identical to what the benchmark engine produces โ€” no changes to the output schema. -The `_manifest.json` records the full configuration for each variant, enabling the graph generator to build rich legend labels. +The `_manifest.json` records the full configuration for each variant, enabling the graph generator to build rich legend labels. It also records what actually ran: + +```json +{ + "run_id": "20260321T140322Z", + "variants": { "...": {} }, + "summary": {"planned": 6, "attempted": 6, "succeeded": 5, "failed": 1}, + "cells": [ + { + "iteration": 1, "x_axis": "connections", "x_value": 4, + "label": "jedis", "driver_config": "configs/drivers/default/jedis.json", + "engine": "java", "metrics_output": "jedis.ndjson", + "started_at": "2026-03-21T14:03:22Z", "status": "ok", + "records_written": 1, "duration_seconds": 41.2 + } + ] +} +``` + +A cell counts as failed if the engine exits non-zero, if the pre-cell FLUSHALL +fails, or if the engine exits 0 but writes no new metrics record. + +## Server Preconditions + +Before the first benchmark runs, the orchestrator resolves a CLI binary and +PINGs the server with bounded retry, so an unreachable endpoint fails up front +instead of aborting mid-sweep. The CLI is resolved in this order: + +1. `$RESP_BENCH_CLI` +2. `work//bin/-cli` โ€” the binary the Makefile builds, where + `` is `$SERVER_PROJECT` (default `valkey`) +3. `-cli`, then `valkey-cli`, then `redis-cli` on `PATH` + +Auth and TLS settings from the driver config (`auth.username`, `auth.password`, +`tls.*`) are passed to the probe and to the per-cell FLUSHALL. Matrices built +only from serverless drivers (`driver_id: "recording"`) skip both the probe and +the flush entirely. + +## Exit Codes + +| Code | Meaning | +|------|---------| +| 0 | Every attempted cell succeeded | +| 1 | At least one cell failed (the sweep still ran to the end) | +| 2 | Preflight failed and nothing ran: server unreachable, no CLI binary, populated run directory, or a matrix with no cells | ## CLI Reference @@ -156,7 +220,10 @@ The `_manifest.json` records the full configuration for each variant, enabling t python scripts/run_benchmark_matrix.py --help --matrix, -m Path to matrix configuration JSON file (required) - --output-dir, -o Directory to write benchmark results (required) + --output-dir, -o Base directory for results; results land in // (required) + --run-id Name of this run's subdirectory (default: UTC timestamp) + --resume Allow appending into a run directory that already has results + --overwrite Delete existing results in the run directory first --server-host Server hostname (overrides matrix config) --port Server port (overrides matrix config) --iterations Override iterations from matrix config diff --git a/docs/CONFIG_SPECIFICATION.md b/docs/CONFIG_SPECIFICATION.md index 6a4c0b7..153ea73 100644 --- a/docs/CONFIG_SPECIFICATION.md +++ b/docs/CONFIG_SPECIFICATION.md @@ -54,6 +54,11 @@ resp-bench uses two JSON configuration files: - `stackexchange-redis` - StackExchange.Redis client - `valkey-glide-csharp` - Valkey GLIDE C# client +**Node.js:** +- `valkey-glide-node` - Valkey GLIDE Node.js client (not `valkey-glide`, which is Java's) +- `ioredis` - ioredis client +- `iovalkey` - iovalkey client (the Valkey-maintained ioredis fork) + **Python (planned):** - `redis-py` - redis-py synchronous client - `redis-py-async` - redis-py async client diff --git a/infra/README.md b/infra/README.md new file mode 100644 index 0000000..b504aa5 --- /dev/null +++ b/infra/README.md @@ -0,0 +1,194 @@ +# resp-bench infrastructure โ€” fire-and-forget benchmarking on AWS + +Launch a benchmark sweep from your laptop, close the lid, and read the report +in S3 later. One command stands up a stock Amazon Linux 2023 EC2 instance that +provisions its own toolchain, runs the sweep, publishes the results to S3, and +then **terminates itself**. The only thing left behind is a free stack shell (a +security group + an IAM role), which you remove with a single `delete-stack`. + +## Layout + +``` +infra/ +โ”œโ”€โ”€ provision.sh # cloud-agnostic toolchain install + engine build + cache warm-up +โ”œโ”€โ”€ aws/ +โ”‚ โ”œโ”€โ”€ benchmark.yaml # CloudFormation: EC2 + SG + IAM role/profile + UserData + CreationPolicy +โ”‚ โ”œโ”€โ”€ run-remote.sh # on-instance job runner (server โ†’ signal โ†’ sweep โ†’ report โ†’ S3 โ†’ self-terminate) +โ”‚ โ””โ”€โ”€ bench-aws.sh # thin driver: generate job id โ†’ deploy โ†’ print S3 prefix โ†’ exit +โ””โ”€โ”€ README.md +``` + +### Architecture split: cloud-agnostic recipe vs. AWS control plane + +The design deliberately separates *what gets installed inside the VM* from *how +the VM is launched, gated, and torn down*: + +| Concern | Lives in | Cloud-specific? | +|---|---|---| +| Toolchain install, server build, cache warm-up | `provision.sh` | **No** โ€” pure `install + build`, inputs via env vars, `dnf`/`apt` shim. A future GCP/Azure control plane reuses it unchanged. | +| Launch / readiness gate / teardown | `aws/benchmark.yaml` (CloudFormation) | Yes | +| Server start, `cfn-signal`, sweep, S3 upload, self-terminate | `aws/run-remote.sh` | Yes (IMDS, S3, cfn-signal) | +| Job id + `deploy` + print prefix | `aws/bench-aws.sh` | Yes | + +`provision.sh` contains **no** AWS/CloudFormation/S3/IMDS calls โ€” that is the +invariant that keeps it portable. All AWS plumbing lives under `aws/`. + +## Prerequisites + +- The AWS CLI (v2), authenticated to the target account. No Terraform, CDK, + Docker, or Packer needed โ€” CloudFormation is driven through the CLI. +- Everything runs in **us-east-1** (pinned in every script); the destination + bucket `valkey-glide-resp-bench` lives there. +- The instance provisions on a **stock AL2023 x86_64 AMI** resolved at deploy + time from the public SSM parameter + `/aws/service/ami-amazon-linux-latest/al2023-ami-kernel-default-x86_64` โ€” no + baked AMI, no hardcoded image id. + +## Usage + +Launch a smoke run (smallest shipped matrix) and walk away: + +``` +$ infra/aws/bench-aws.sh deploy --matrix valkey-glide-basic-multiclients +Launching run bench-20260825-143002-a1b9c3 (stack resp-bench-bench-20260825-143002-a1b9c3)... +... +Launched run bench-20260825-143002-a1b9c3. +Results will appear at: s3://valkey-glide-resp-bench/runs/bench-20260825-143002-a1b9c3/ +$ +``` + +`deploy` blocks only until provisioning is **genuinely healthy** โ€” a +`CreationPolicy` + `cfn-signal` gate makes stack creation wait for the server to +answer `PING` (and roll back if it never does), rather than reporting success +the moment the instance exists. Once that gate clears, the instance runs the +sweep, uploads, and self-terminates on its own; your machine is out of the loop. + +### Preview without touching AWS + +``` +$ infra/aws/bench-aws.sh deploy --dry-run --matrix valkey-glide-basic-multiclients +DRY RUN โ€” no AWS calls made. + +Resolved deploy command: + aws cloudformation deploy --region us-east-1 --stack-name ... --parameter-overrides ... + +Job id: bench-... +Stack name: resp-bench-bench-... +Results will appear at: s3://valkey-glide-resp-bench/runs/bench-.../ +``` + +### Cost guardrails + +- The default matrix is the **smallest shipped one** (a smoke run). Large sweeps + (e.g. `driver-comparison-high-tps` = 720 cells) require passing `--matrix` + explicitly, so nobody launches a huge bill by reflex. +- The default instance type (`m5.large`) is small and cheap. For a + fidelity-grade sweep pass `--instance-type m5.metal` (or `c5.metal`) โ€” and + note that comparable numbers require pinning the CPU model. +- Runs use **on-demand** instances (a spot reclamation at 4am would kill an + unattended sweep). +- A hard **runtime cap** (`--runtime-cap `, default 180) schedules + `shutdown -h +N` at boot so a hung run can't bill indefinitely. + +### Larger sweep example + +``` +infra/aws/bench-aws.sh deploy \ + --matrix driver-comparison-defaults \ + --instance-type m5.metal \ + --repo-tag v0.1.0-phase0 \ + --runtime-cap 360 +``` + +> **Repo tag / known dependency.** The automation assumes the pinned +> `--repo-tag` includes the Phase-0 correctness fixes (fail-loud runner, honest +> exit code, CLI-based flush). On raw `main` the sweep may fail at the flush +> step (the runner calls `redis-cli`, which isn't installed โ€” the server ships +> `valkey-cli`). That is expected and out of scope here: the run still uploads a +> diagnosable bundle and self-terminates. + +## What lands in S3 + +``` +runs// + report.html # interactive Plotly report (scalability + delta) + run-metadata.json # job id, status, matrix, repo tag + git sha, Valkey + # version, instance type / id / AZ / AMI, region, timing + results/ # raw sweep output: *.ndjson, *.system.ndjson, _manifest.json + graphs/ # generated graph assets + logs/ # cloud-init-output.log (provision + run log) +``` + +`job_id = [prefix-]bench-YYYYMMDD-HHMMSS-<6 random>`. The `results/_manifest.json` +is the per-cell status manifest; it is uploaded **even when the sweep fails** +(the run script wraps upload + self-terminate in a `trap ... EXIT`), so a failed +run leaves a diagnosable result in S3 rather than a hung box. + +## Cleaning up + +Two distinct things exist after a run โ€” handle them separately. + +### 1. The compute (handled automatically) + +The instance **terminates itself** when the run ends โ€” whether the sweep passed +or failed, and when the runtime cap fires. Compute and its EBS volume are gone +($0) with no action from you, and with **no `ec2:TerminateInstances` +permission** on the instance role (it self-terminates via +`InstanceInitiatedShutdownBehavior: terminate` + `shutdown -h now`). + +### 2. The free stack shell (you remove it) + +After the instance self-terminates, the CloudFormation stack still exists as a +free shell โ€” the security group and the IAM role. It costs nothing, but you +should remove it once you've read the results: + +``` +aws cloudformation delete-stack --region us-east-1 --stack-name resp-bench- +``` + +or the convenience wrapper: + +``` +infra/aws/bench-aws.sh delete resp-bench- +``` + +> There is intentionally **no watchdog Lambda** or auto-teardown. `delete-stack` +> is a deliberate, documented manual step. + +To list every leftover shell: + +``` +aws cloudformation list-stacks --region us-east-1 \ + --stack-status-filter CREATE_COMPLETE UPDATE_COMPLETE \ + --query "StackSummaries[?starts_with(StackName, 'resp-bench-')].StackName" +``` + +## Viewing the report + +The bucket blocks public access (account-wide), so use a **presigned URL**: + +``` +aws s3 presign --region us-east-1 \ + s3://valkey-glide-resp-bench/runs//report.html --expires-in 3600 +``` + +or: + +``` +infra/aws/bench-aws.sh check +``` + +which lists the run's objects and prints a 1-hour presigned URL for the report. + +## Testing `provision.sh` in isolation + +Because `provision.sh` is cloud-agnostic and parameterised by env vars, you can +run it on any Linux host (it detects `dnf` vs `apt`): + +``` +REPO_DIR=/path/to/resp-bench bash infra/provision.sh +``` + +It installs JDK 21 + Maven, Ruby + bundler, the .NET SDK, Python + pinned deps, +builds the Valkey server, and warms the Java/Ruby/C# build caches. `SKIP_SERVER_BUILD=1` +and `SKIP_WARM_CACHES=1` skip the slow steps for quick smoke checks. diff --git a/infra/aws/bench-aws.sh b/infra/aws/bench-aws.sh new file mode 100755 index 0000000..af58812 --- /dev/null +++ b/infra/aws/bench-aws.sh @@ -0,0 +1,182 @@ +#!/usr/bin/env bash +# +# bench-aws.sh โ€” thin fire-and-forget driver for the resp-bench AWS runner. +# +# Generates a job id, deploys the CloudFormation stack (which provisions an +# EC2 box, runs the sweep, uploads to S3, and self-terminates), prints the S3 +# prefix where results will appear, and exits. No polling loop. +# +# Region is PINNED to us-east-1 for every AWS call โ€” never inherited from the +# ambient shell (which may export AWS_REGION=us-west-2). The S3 bucket lives in +# us-east-1. +# +# Subcommands: +# deploy [flags] Launch a run (default). +# check [flags] List the run's S3 prefix + presign the report. +# delete [flags] Delete a leftover stack shell (SG + role). +# +# deploy flags: +# --matrix Matrix config name under configs/matrices/ +# (default: valkey-glide-basic-multiclients โ€” the +# smallest shipped matrix, used as a smoke run). +# REQUIRE this explicitly for large sweeps. +# --instance-type EC2 instance type (default: m5.large). +# --repo-tag Git ref to check out (default: main). +# --repo-url Fork URL (default: Bit-Quill/resp-bench). +# --runtime-cap Hard runtime cap in minutes (default: 180). +# --bucket Destination bucket (default: valkey-glide-resp-bench). +# --job-id-prefix

Prefix the job id (e.g. nightly, pr-123). +# --dry-run Print the resolved deploy command + S3 prefix; do +# not call AWS. + +set -euo pipefail + +# โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ +# Constants (region pinned) +# โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ +readonly REGION="us-east-1" +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +readonly TEMPLATE="${SCRIPT_DIR}/benchmark.yaml" + +# โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ +# Defaults +# โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ +MATRIX="valkey-glide-basic-multiclients" +INSTANCE_TYPE="m5.large" +REPO_TAG="main" +REPO_URL="https://github.com/Bit-Quill/resp-bench.git" +RUNTIME_CAP="180" +BUCKET="valkey-glide-resp-bench" +JOB_ID_PREFIX="" +DRY_RUN=0 + +die() { printf 'bench-aws: %s\n' "$*" >&2; exit 1; } + +usage() { sed -n '2,40p' "${BASH_SOURCE[0]}" | sed 's/^# \{0,1\}//'; } + +# โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ +# job_id generation: [prefix-]bench-YYYYMMDD-HHMMSS-<6 random> +# โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ +generate_job_id() { + local ts rand + ts="$(date -u +%Y%m%d-%H%M%S)" + rand="$(LC_ALL=C tr -dc 'a-z0-9' /dev/null | head -c 6 || true)" + [ -z "${rand}" ] && rand="$(printf '%06x' $((RANDOM * RANDOM % 16777216)))" + if [ -n "${JOB_ID_PREFIX}" ]; then + printf '%s-bench-%s-%s' "${JOB_ID_PREFIX}" "${ts}" "${rand}" + else + printf 'bench-%s-%s' "${ts}" "${rand}" + fi +} + +# โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ +# deploy +# โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ +cmd_deploy() { + while [ $# -gt 0 ]; do + case "$1" in + --matrix) MATRIX="$2"; shift 2 ;; + --instance-type) INSTANCE_TYPE="$2"; shift 2 ;; + --repo-tag) REPO_TAG="$2"; shift 2 ;; + --repo-url) REPO_URL="$2"; shift 2 ;; + --runtime-cap) RUNTIME_CAP="$2"; shift 2 ;; + --bucket) BUCKET="$2"; shift 2 ;; + --job-id-prefix) JOB_ID_PREFIX="$2"; shift 2 ;; + --dry-run) DRY_RUN=1; shift ;; + -h|--help) usage; exit 0 ;; + *) die "unknown deploy flag: $1" ;; + esac + done + + [ -f "${TEMPLATE}" ] || die "template not found: ${TEMPLATE}" + + local job_id stack_name s3_prefix + job_id="$(generate_job_id)" + stack_name="resp-bench-${job_id}" + s3_prefix="s3://${BUCKET}/runs/${job_id}/" + + # Build the deploy argv (kept as an array so the dry-run prints exactly what + # would run). + local -a deploy_cmd=( + aws cloudformation deploy + --region "${REGION}" + --stack-name "${stack_name}" + --template-file "${TEMPLATE}" + --capabilities CAPABILITY_IAM + --tags Project=resp-bench "RunId=${job_id}" + --parameter-overrides + "JobId=${job_id}" + "MatrixName=${MATRIX}" + "InstanceType=${INSTANCE_TYPE}" + "RepoTag=${REPO_TAG}" + "RepoUrl=${REPO_URL}" + "RuntimeCapMinutes=${RUNTIME_CAP}" + "S3Bucket=${BUCKET}" + ) + + if [ "${DRY_RUN}" -eq 1 ]; then + echo "DRY RUN โ€” no AWS calls made." + echo + echo "Resolved deploy command:" + printf ' %q' "${deploy_cmd[@]}" + echo + echo + echo "Job id: ${job_id}" + echo "Stack name: ${stack_name}" + echo "Results will appear at: ${s3_prefix}" + return 0 + fi + + echo "Launching run ${job_id} (stack ${stack_name})..." + "${deploy_cmd[@]}" + echo "Launched run ${job_id}." + echo "Results will appear at: ${s3_prefix}" +} + +# โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ +# check โ€” list the prefix and presign the report +# โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ +cmd_check() { + local job_id="${1:-}"; shift || true + [ -n "${job_id}" ] || die "usage: bench-aws.sh check [--bucket ]" + while [ $# -gt 0 ]; do + case "$1" in + --bucket) BUCKET="$2"; shift 2 ;; + *) die "unknown check flag: $1" ;; + esac + done + local prefix="s3://${BUCKET}/runs/${job_id}/" + echo "Listing ${prefix}" + aws s3 ls --region "${REGION}" "${prefix}" --recursive || true + echo + echo "Presigned report URL (valid 1h):" + aws s3 presign --region "${REGION}" "${prefix}report.html" --expires-in 3600 +} + +# โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ +# delete โ€” convenience wrapper over delete-stack +# โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ +cmd_delete() { + local stack="${1:-}" + [ -n "${stack}" ] || die "usage: bench-aws.sh delete " + echo "Deleting stack ${stack} in ${REGION}..." + aws cloudformation delete-stack --region "${REGION}" --stack-name "${stack}" + echo "Delete requested. Track with: aws cloudformation describe-stacks --region ${REGION} --stack-name ${stack}" +} + +# โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ +# Dispatch +# โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ +main() { + local sub="${1:-deploy}" + case "${sub}" in + deploy) shift || true; cmd_deploy "$@" ;; + check) shift; cmd_check "$@" ;; + delete) shift; cmd_delete "$@" ;; + -h|--help|help) usage ;; + --*) cmd_deploy "$@" ;; # allow `bench-aws.sh --dry-run ...` + *) die "unknown subcommand: ${sub} (expected deploy|check|delete)" ;; + esac +} + +main "$@" diff --git a/infra/aws/benchmark.yaml b/infra/aws/benchmark.yaml new file mode 100644 index 0000000..cdc1a01 --- /dev/null +++ b/infra/aws/benchmark.yaml @@ -0,0 +1,262 @@ +AWSTemplateFormatVersion: "2010-09-09" + +Description: >- + resp-bench fire-and-forget benchmark runner. Launches a single stock + Amazon Linux 2023 (x86_64) EC2 instance that provisions its own toolchain, + runs a benchmark sweep, publishes the report + raw results + run metadata to + S3 under runs//, and then TERMINATES ITSELF. A CreationPolicy + + cfn-signal gate makes stack creation block until provisioning is genuinely + healthy (and roll back otherwise). After the instance self-terminates the + only leftover is this stack's free shell (security group + IAM role); remove + it with `aws cloudformation delete-stack` (see infra/README.md). + +# โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ +# Parameters +# โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ +Parameters: + + JobId: + Type: String + Description: >- + Unique run id, generated by bench-aws.sh as + [prefix-]bench-YYYYMMDD-HHMMSS-<6 random>. Used as the S3 prefix + (runs//) and the RunId tag. + AllowedPattern: "^[a-zA-Z0-9._-]+$" + + MatrixName: + Type: String + Default: valkey-glide-basic-multiclients + Description: >- + Matrix config name (without path/extension) under configs/matrices/. + Defaults to the smallest shipped matrix as a smoke run; pass a larger + one explicitly for a real sweep. + + RepoUrl: + Type: String + Default: https://github.com/Bit-Quill/resp-bench.git + Description: Public git URL of the resp-bench fork to clone. + + RepoTag: + Type: String + Default: main + Description: >- + Git ref (tag/branch/sha) to check out. SHOULD be a tag that includes the + Phase-0 correctness fixes (fail-loud runner, honest exit code, CLI-based + flush). On raw main the sweep may fail at the flush step; that is expected + and out of scope for this template โ€” the run still uploads diagnostics and + self-terminates. + + InstanceType: + Type: String + Default: m5.large + Description: >- + EC2 instance type (on-demand). The default is a small, cheap smoke-run + size. For a fidelity-grade sweep use a dedicated/metal type (e.g. + m5.metal or c5.metal) and pin the CPU model for run-to-run comparability. + + RuntimeCapMinutes: + Type: Number + Default: 180 + MinValue: 5 + MaxValue: 1440 + Description: >- + Hard runtime cap. The instance schedules `shutdown -h +N` at boot so a + hung run cannot bill indefinitely. Conservative default of 3 hours. + + S3Bucket: + Type: String + Default: valkey-glide-resp-bench + Description: >- + Destination S3 bucket. Results land under s3:///runs//. + The instance role is scoped to s3:PutObject on runs/* of this bucket only. + + BaseAmiId: + Type: "AWS::SSM::Parameter::Value" + Default: /aws/service/ami-amazon-linux-latest/al2023-ami-kernel-default-x86_64 + Description: >- + Base AMI, resolved at deploy time from the public SSM parameter for the + latest AL2023 x86_64 image. Never a hardcoded AMI id. + + RootVolumeSizeGb: + Type: Number + Default: 30 + MinValue: 20 + MaxValue: 500 + Description: Root EBS volume size (GiB). Deleted on termination. + +# โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ +# Resources +# โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ +Resources: + + # Least-privilege instance profile: SSM core (for optional interactive + # debugging / patch compliance) + s3:PutObject scoped to this bucket's runs/* + # + cloudformation:SignalResource scoped to THIS stack (for the CreationPolicy + # gate). Notably NO ec2:TerminateInstances โ€” the box self-terminates via + # InstanceInitiatedShutdownBehavior below, so no terminate permission exists. + InstanceRole: + Type: AWS::IAM::Role + Properties: + AssumeRolePolicyDocument: + Version: "2012-10-17" + Statement: + - Effect: Allow + Principal: + Service: ec2.amazonaws.com + Action: sts:AssumeRole + ManagedPolicyArns: + - arn:aws:iam::aws:policy/AmazonSSMManagedInstanceCore + Policies: + - PolicyName: PutBenchmarkResults + PolicyDocument: + Version: "2012-10-17" + Statement: + - Effect: Allow + Action: s3:PutObject + Resource: !Sub "arn:aws:s3:::${S3Bucket}/runs/*" + - PolicyName: SignalOwnStack + PolicyDocument: + Version: "2012-10-17" + Statement: + - Effect: Allow + Action: cloudformation:SignalResource + Resource: !Ref AWS::StackId + Tags: + - Key: Project + Value: resp-bench + + InstanceProfile: + Type: AWS::IAM::InstanceProfile + Properties: + Roles: + - !Ref InstanceRole + + # No inbound rules at all: this is a single colocated box (client + server on + # the same host over localhost:6379), so nothing needs to reach it. We omit + # SecurityGroupIngress entirely (no public ingress) and omit + # SecurityGroupEgress so the group keeps its default allow-all egress โ€” which + # the instance needs to clone the repo, install toolchains, and PUT to S3. + # (An explicit egress block would force declaring a VpcId; omitting it lets + # the group be created in the account's default VPC.) + BenchmarkSecurityGroup: + Type: AWS::EC2::SecurityGroup + Properties: + GroupDescription: >- + resp-bench runner - no public ingress; default allow-all egress only. + The Valkey server is reached only over localhost within the instance. + Tags: + - Key: Project + Value: resp-bench + - Key: RunId + Value: !Ref JobId + + BenchmarkInstance: + Type: AWS::EC2::Instance + CreationPolicy: + ResourceSignal: + # Blocks stack creation until run-remote.sh signals SUCCESS (server up + # and healthy) or FAILURE (rolls back). Generous timeout to cover the + # ~10-20 min launch-time toolchain install. + Count: 1 + Timeout: PT45M + Properties: + ImageId: !Ref BaseAmiId + InstanceType: !Ref InstanceType + IamInstanceProfile: !Ref InstanceProfile + SecurityGroupIds: + - !GetAtt BenchmarkSecurityGroup.GroupId + # Self-terminate (not stop) when the run script calls `shutdown -h now`, + # or when the runtime-cap timer fires. Compute + EBS gone, $0, with NO + # ec2:TerminateInstances permission on the instance role. + InstanceInitiatedShutdownBehavior: terminate + BlockDeviceMappings: + - DeviceName: /dev/xvda + Ebs: + VolumeSize: !Ref RootVolumeSizeGb + VolumeType: gp3 + DeleteOnTermination: true + Tags: + - Key: Project + Value: resp-bench + - Key: RunId + Value: !Ref JobId + - Key: Name + Value: !Sub "resp-bench-${JobId}" + # UserData is deliberately thin: install git, clone the fork at the pinned + # ref, run the cloud-agnostic provision.sh, then hand off to the on-instance + # run script (which owns server start, the cfn-signal gate, the sweep, the + # S3 upload, and self-termination โ€” all under a trap). The real logic lives + # in reviewable repo scripts, not in this YAML. + UserData: + Fn::Base64: !Sub | + #!/bin/bash + set -euo pipefail + + REGION="${AWS::Region}" + STACK_NAME="${AWS::StackName}" + SIGNAL_RESOURCE="BenchmarkInstance" + + # IMDSv2: fetch a token and this instance's id (for signalling + + # run metadata). This AWS-specific plumbing lives in UserData/the + # run script, never in provision.sh. + TOKEN=$(curl -sS -X PUT "http://169.254.169.254/latest/api/token" -H "X-aws-ec2-metadata-token-ttl-seconds: 300" || true) + INSTANCE_ID=$(curl -sS -H "X-aws-ec2-metadata-token: $TOKEN" http://169.254.169.254/latest/meta-data/instance-id || true) + + # If provisioning fails before the run script sends its SUCCESS + # signal, fail the stack fast instead of waiting out the timeout. + # First preserve the provisioning log to S3: the stack rolls back and + # terminates the instance on FAILURE, taking its console output with + # it, so this is the only durable record of a provision-phase failure. + signal_failure() { + aws s3 cp /var/log/cloud-init-output.log \ + "s3://${S3Bucket}/runs/${JobId}/provision-failure.log" \ + --region "$REGION" || true + aws cloudformation signal-resource --region "$REGION" \ + --stack-name "$STACK_NAME" --logical-resource-id "$SIGNAL_RESOURCE" \ + --unique-id "$INSTANCE_ID" --status FAILURE || true + } + trap signal_failure ERR + + dnf install -y git + + REPO_DIR=/opt/resp-bench + git clone --depth 1 --branch "${RepoTag}" "${RepoUrl}" "$REPO_DIR" + + # Cloud-agnostic toolchain install + engine build + cache warm-up. + export REPO_DIR + bash "$REPO_DIR/infra/provision.sh" + + # Provisioning succeeded; hand off. The run script sends the + # CreationPolicy SUCCESS signal itself once the server is healthy. + trap - ERR + export JOB_ID="${JobId}" + export S3_BUCKET="${S3Bucket}" + export MATRIX_NAME="${MatrixName}" + export REPO_TAG="${RepoTag}" + export RUNTIME_CAP_MINUTES="${RuntimeCapMinutes}" + export AWS_REGION="$REGION" + export STACK_NAME + export CFN_SIGNAL_RESOURCE="$SIGNAL_RESOURCE" + export CFN_INSTANCE_ID="$INSTANCE_ID" + bash "$REPO_DIR/infra/aws/run-remote.sh" + +# โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ +# Outputs +# โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ +Outputs: + + JobId: + Description: The run id / S3 prefix segment. + Value: !Ref JobId + + S3Prefix: + Description: Where results will appear. + Value: !Sub "s3://${S3Bucket}/runs/${JobId}/" + + InstanceId: + Description: The (self-terminating) benchmark instance. + Value: !Ref BenchmarkInstance + + StackName: + Description: Delete this stack after reading results to remove the free shell (SG + role). + Value: !Ref AWS::StackName diff --git a/infra/aws/run-remote.sh b/infra/aws/run-remote.sh new file mode 100755 index 0000000..83b1b8f --- /dev/null +++ b/infra/aws/run-remote.sh @@ -0,0 +1,261 @@ +#!/usr/bin/env bash +# +# run-remote.sh โ€” the on-instance benchmark job runner (AWS side). +# +# Invoked by the CloudFormation UserData after provision.sh has installed the +# toolchain. This script owns the AWS-specific run lifecycle: +# +# 1. schedule a hard runtime cap (`shutdown -h +N`) so a hung run can't bill +# indefinitely; +# 2. start the Valkey server and confirm PING; +# 3. send the CreationPolicy SUCCESS signal (cfn-signal) โ€” this is what +# unblocks `aws cloudformation deploy`, gating on a genuinely healthy box; +# 4. run the matrix sweep into runs//results/; +# 5. generate the interactive HTML report; +# 6. assemble run-metadata.json; +# 7. upload the whole bundle to s3:///runs//; +# 8. `shutdown -h now` (โ†’ self-TERMINATE, per the instance's +# InstanceInitiatedShutdownBehavior). +# +# A `trap ... EXIT` guarantees that steps 6-8 (upload results + the per-cell +# manifest, then terminate) happen EVEN WHEN THE SWEEP FAILS โ€” a failed run +# leaves a diagnosable bundle in S3, not a hung, billing box. +# +# All AWS/IMDS/S3/cfn-signal calls live here (never in provision.sh). Region is +# pinned via AWS_REGION (us-east-1), never inherited from the ambient shell. +# +# Inputs (exported by UserData): +# JOB_ID, S3_BUCKET, MATRIX_NAME, REPO_TAG, RUNTIME_CAP_MINUTES, +# AWS_REGION, STACK_NAME, CFN_SIGNAL_RESOURCE, CFN_INSTANCE_ID, REPO_DIR + +set -uo pipefail + +log() { printf '[run-remote] %s\n' "$*"; } + +# โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ +# Inputs & derived paths +# โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ +REPO_DIR="${REPO_DIR:-/opt/resp-bench}" +JOB_ID="${JOB_ID:?JOB_ID is required}" +S3_BUCKET="${S3_BUCKET:-valkey-glide-resp-bench}" +MATRIX_NAME="${MATRIX_NAME:-valkey-glide-basic-multiclients}" +REPO_TAG="${REPO_TAG:-unknown}" +RUNTIME_CAP_MINUTES="${RUNTIME_CAP_MINUTES:-180}" +AWS_REGION="${AWS_REGION:-us-east-1}" +STACK_NAME="${STACK_NAME:-}" +CFN_SIGNAL_RESOURCE="${CFN_SIGNAL_RESOURCE:-BenchmarkInstance}" +CFN_INSTANCE_ID="${CFN_INSTANCE_ID:-}" + +MATRIX_PATH="configs/matrices/${MATRIX_NAME}.json" +STAGE_DIR="/opt/resp-bench-runs/${JOB_ID}" +RESULTS_DIR="${STAGE_DIR}/results" +GRAPHS_DIR="${STAGE_DIR}/graphs" +LOGS_DIR="${STAGE_DIR}/logs" +METADATA_FILE="${STAGE_DIR}/run-metadata.json" +REPORT_FILE="${STAGE_DIR}/report.html" +S3_PREFIX="s3://${S3_BUCKET}/runs/${JOB_ID}/" + +START_TS="$(date -u +%Y-%m-%dT%H:%M:%SZ)" +SIGNALLED=0 +UPLOADED=0 +SWEEP_RC="" + +mkdir -p "${RESULTS_DIR}" "${GRAPHS_DIR}" "${LOGS_DIR}" + +# โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ +# IMDSv2 helpers (AWS-specific โ€” correct place for them) +# โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ +imds() { + # imds -> value (empty on failure) + local token + token="$(curl -sS -X PUT "http://169.254.169.254/latest/api/token" \ + -H "X-aws-ec2-metadata-token-ttl-seconds: 300" 2>/dev/null || true)" + curl -sS -H "X-aws-ec2-metadata-token: ${token}" \ + "http://169.254.169.254/latest/meta-data/${1}" 2>/dev/null || true +} + +INSTANCE_ID="${CFN_INSTANCE_ID:-$(imds instance-id)}" +INSTANCE_TYPE="$(imds instance-type)" +AVAILABILITY_ZONE="$(imds placement/availability-zone)" +AMI_ID="$(imds ami-id)" + +# โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ +# cfn-signal โ€” unblocks the CreationPolicy in `deploy` +# โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ +cfn_signal() { + # cfn_signal SUCCESS|FAILURE + [ -z "${STACK_NAME}" ] && { log "no STACK_NAME; skipping cfn-signal $1"; return 0; } + log "cfn-signal $1" + aws cloudformation signal-resource \ + --region "${AWS_REGION}" \ + --stack-name "${STACK_NAME}" \ + --logical-resource-id "${CFN_SIGNAL_RESOURCE}" \ + --unique-id "${INSTANCE_ID}" \ + --status "$1" || log "WARNING: cfn-signal $1 call failed" +} + +# โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ +# Upload the bundle to S3 (idempotent; safe to call more than once) +# โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ +upload_bundle() { + [ "${UPLOADED}" -eq 1 ] && return 0 + # Best-effort capture of the boot/provision log for post-mortem. + cp /var/log/cloud-init-output.log "${LOGS_DIR}/cloud-init-output.log" 2>/dev/null || true + log "uploading bundle to ${S3_PREFIX}" + if aws s3 cp --region "${AWS_REGION}" --recursive "${STAGE_DIR}/" "${S3_PREFIX}"; then + UPLOADED=1 + log "upload complete" + else + log "WARNING: S3 upload failed" + fi +} + +# โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ +# EXIT trap โ€” the safety net: upload whatever exists, then self-terminate, +# no matter how we got here (success, sweep failure, or unexpected abort). +# โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ +# shellcheck disable=SC2329 # invoked indirectly via `trap ... EXIT` +on_exit() { + local rc=$? + log "EXIT trap (rc=${rc}); ensuring results are uploaded and box terminates" + # If we never reached the healthy-signal point, tell CFN we failed so the + # stack rolls back promptly instead of timing out. + if [ "${SIGNALLED}" -eq 0 ]; then + cfn_signal FAILURE + SIGNALLED=1 + fi + # Only synthesize "aborted" metadata if the normal path never uploaded a + # bundle (with its honest succeeded/failed status). Otherwise leave the good + # bundle untouched. + if [ "${UPLOADED}" -eq 0 ]; then + write_metadata "aborted" + upload_bundle + fi + log "self-terminating via 'shutdown -h now'" + shutdown -h now || sudo shutdown -h now || true +} +trap on_exit EXIT + +# โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ +# run-metadata.json assembler +# โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ +write_metadata() { + # write_metadata + local status="$1" end_ts git_sha valkey_version + end_ts="$(date -u +%Y-%m-%dT%H:%M:%SZ)" + git_sha="$(git -C "${REPO_DIR}" rev-parse HEAD 2>/dev/null || echo unknown)" + valkey_version="$("${REPO_DIR}/work/valkey/bin/valkey-server" --version 2>/dev/null | head -n1 || echo unknown)" + + if command -v jq >/dev/null 2>&1; then + jq -n \ + --arg job_id "${JOB_ID}" \ + --arg status "${status}" \ + --arg matrix "${MATRIX_NAME}" \ + --arg repo_tag "${REPO_TAG}" \ + --arg git_sha "${git_sha}" \ + --arg region "${AWS_REGION}" \ + --arg instance_id "${INSTANCE_ID}" \ + --arg instance_type "${INSTANCE_TYPE}" \ + --arg az "${AVAILABILITY_ZONE}" \ + --arg ami_id "${AMI_ID}" \ + --arg valkey_version "${valkey_version}" \ + --arg sweep_rc "${SWEEP_RC}" \ + --arg start "${START_TS}" \ + --arg end "${end_ts}" \ + '{ + job_id: $job_id, + status: $status, + matrix: $matrix, + repo_tag: $repo_tag, + git_sha: $git_sha, + sweep_exit_code: $sweep_rc, + server: { valkey_version: $valkey_version }, + aws: { region: $region, instance_id: $instance_id, instance_type: $instance_type, availability_zone: $az, ami_id: $ami_id }, + timing: { start: $start, end: $end } + }' > "${METADATA_FILE}" 2>/dev/null || true + else + # Fallback if jq is somehow unavailable. + printf '{"job_id":"%s","status":"%s","matrix":"%s","repo_tag":"%s","git_sha":"%s","sweep_exit_code":"%s","instance_type":"%s","availability_zone":"%s","ami_id":"%s","region":"%s","start":"%s","end":"%s"}\n' \ + "${JOB_ID}" "${status}" "${MATRIX_NAME}" "${REPO_TAG}" "${git_sha}" "${SWEEP_RC}" \ + "${INSTANCE_TYPE}" "${AVAILABILITY_ZONE}" "${AMI_ID}" "${AWS_REGION}" "${START_TS}" "${end_ts}" \ + > "${METADATA_FILE}" + fi +} + +# โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ• +# Main flow +# โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ• +cd "${REPO_DIR}" || exit 1 + +# Make the provisioned toolchain (e.g. dotnet on PATH) visible here. +if [ -f "${REPO_DIR}/.resp-bench-env" ]; then + # shellcheck source=/dev/null + . "${REPO_DIR}/.resp-bench-env" +fi + +# (1) Hard runtime cap โ€” self-terminate after N minutes no matter what. +log "scheduling runtime cap: shutdown -h +${RUNTIME_CAP_MINUTES}" +shutdown -h "+${RUNTIME_CAP_MINUTES}" "resp-bench runtime cap reached" \ + || sudo shutdown -h "+${RUNTIME_CAP_MINUTES}" || log "WARNING: could not schedule runtime cap" + +# (2) Start the server and confirm PING. +log "starting Valkey server" +make server-standalone-start +CLI="${REPO_DIR}/work/valkey/bin/valkey-cli" +log "probing server readiness (PING)" +PING_OK=0 +for _ in $(seq 1 30); do + if [ -x "${CLI}" ] && [ "$("${CLI}" -h 127.0.0.1 -p 6379 ping 2>/dev/null)" = "PONG" ]; then + PING_OK=1 + break + fi + sleep 1 +done +if [ "${PING_OK}" -ne 1 ]; then + log "ERROR: server did not become ready; signalling FAILURE and aborting" + cfn_signal FAILURE + SIGNALLED=1 + exit 1 # โ†’ EXIT trap uploads diagnostics and terminates +fi +log "server is up (PONG)" + +# (3) Provisioning + server are healthy โ€” unblock `deploy`. +cfn_signal SUCCESS +SIGNALLED=1 + +# (4) Run the sweep. Use the Makefile contract (stable), organizing output into +# the job-specific results dir ourselves (the runner on this ref has no +# --run-id). We do NOT abort on failure: the trap must still upload + the +# honest exit code is recorded in the metadata. +log "running sweep: matrix=${MATRIX_PATH} -> ${RESULTS_DIR}" +make benchmark-matrix \ + MATRIX="${MATRIX_PATH}" \ + OUTPUT_DIR="${RESULTS_DIR}" \ + SERVER_HOST=127.0.0.1 +SWEEP_RC=$? +log "sweep finished with exit code ${SWEEP_RC}" + +# (5) Generate the interactive HTML report (best effort). +log "generating report" +if make benchmark-matrix-graphs OUTPUT_DIR="${RESULTS_DIR}" GRAPHS_DIR="${GRAPHS_DIR}"; then + if [ -f "${GRAPHS_DIR}/scalability_and_delta.html" ]; then + cp "${GRAPHS_DIR}/scalability_and_delta.html" "${REPORT_FILE}" + fi +else + log "WARNING: report generation failed" +fi + +# (6) Metadata with the honest final status. +if [ "${SWEEP_RC}" -eq 0 ]; then + write_metadata "succeeded" +else + write_metadata "failed" +fi + +# (7) Upload the bundle. +upload_bundle + +log "run complete for ${JOB_ID}; results at ${S3_PREFIX}" +# (8) Normal completion: the EXIT trap performs the final upload check and +# `shutdown -h now`. Exit with the sweep's honest status. +exit "${SWEEP_RC}" diff --git a/infra/provision.sh b/infra/provision.sh new file mode 100755 index 0000000..1de54b7 --- /dev/null +++ b/infra/provision.sh @@ -0,0 +1,332 @@ +#!/usr/bin/env bash +# +# provision.sh โ€” cloud-agnostic toolchain installer for resp-bench. +# +# Installs every language toolchain the benchmark engines need, builds the +# Valkey server, and warms the per-engine build caches so that a subsequent +# sweep does no first-time compilation. +# +# DESIGN CONTRACT (keep this true โ€” it is what makes the recipe reusable): +# * NO cloud-specific calls live here. No AWS CLI, no CloudFormation, no S3, +# no IMDS/metadata reads, no cfn-signal. All of that belongs in the +# per-cloud wrapper under infra/aws/ (see infra/README.md). This script +# only knows how to turn a stock Linux box into a machine that can run the +# resp-bench matrix, so a future GCP/Azure control plane can reuse it +# verbatim. +# * Inputs arrive via environment variables (see below), never via cloud +# metadata. +# * Every block is idempotent: re-running the script is safe and cheap. +# * The per-language blocks are ADDITIVE and clearly delimited. Adding a new +# engine (Node.js, Go, PHP, ...) is a small, self-contained edit โ€” copy an +# existing "=== LANGUAGE: ... ===" block and adjust it. +# +# Environment variables (all optional; sensible defaults shown): +# REPO_DIR Path to the checked-out resp-bench repo. Default: the repo +# that contains this script (../ relative to infra/). +# DOTNET_ROOT Where to install the .NET SDK. Default: /usr/local/dotnet +# (falls back to $HOME/.dotnet if that is not writable). +# DOTNET_CHANNEL .NET SDK channel to install. Default: 10.0 +# SKIP_SERVER_BUILD If "1", skip compiling the Valkey server. +# SKIP_WARM_CACHES If "1", skip the throwaway engine builds. +# +# On success the script writes "$REPO_DIR/.resp-bench-env", a small snippet +# that exports the PATH/DOTNET_ROOT additions. Callers that run in a *separate* +# shell (e.g. the on-instance run script) should `source` that file so they see +# the installed toolchains. The script also drops the same snippet into +# /etc/profile.d/ when that directory is writable, for interactive logins. + +set -euo pipefail + +# โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ +# Bootstrap: locate the repo and set up helpers +# โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +REPO_DIR="${REPO_DIR:-$(cd "${SCRIPT_DIR}/.." && pwd)}" +DOTNET_CHANNEL="${DOTNET_CHANNEL:-10.0}" + +log() { printf '[provision] %s\n' "$*"; } +err() { printf '[provision] ERROR: %s\n' "$*" >&2; } + +# Run privileged commands only when we are not already root. +SUDO="" +if [ "$(id -u)" -ne 0 ]; then + if command -v sudo >/dev/null 2>&1; then + SUDO="sudo" + else + err "not running as root and 'sudo' is unavailable; package installs may fail" + fi +fi + +# โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ +# Package-manager shim โ€” detect dnf (AL2023/Fedora) vs apt (Debian/Ubuntu) +# so this script can be exercised on a non-AL2023 Linux box. +# โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ + +PKG="" +if command -v dnf >/dev/null 2>&1; then + PKG="dnf" +elif command -v apt-get >/dev/null 2>&1; then + PKG="apt" +else + err "no supported package manager found (need dnf or apt-get)" + exit 1 +fi +log "using package manager: ${PKG}" + +_apt_updated=0 +pkg_install() { + # pkg_install "--" + # Splits the argument list on "--"; installs the correct set for the + # detected package manager. Idempotent (the package managers no-op on + # already-installed packages). + local dnf_pkgs=() apt_pkgs=() seen_sep=0 + for arg in "$@"; do + if [ "${arg}" = "--" ]; then seen_sep=1; continue; fi + if [ "${seen_sep}" -eq 0 ]; then dnf_pkgs+=("${arg}"); else apt_pkgs+=("${arg}"); fi + done + + if [ "${PKG}" = "dnf" ]; then + ${SUDO} dnf install -y "${dnf_pkgs[@]}" + else + if [ "${_apt_updated}" -eq 0 ]; then + ${SUDO} apt-get update -y + _apt_updated=1 + fi + DEBIAN_FRONTEND=noninteractive ${SUDO} apt-get install -y "${apt_pkgs[@]}" + fi +} + +# โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ +# Environment snippet emitted for downstream shells (e.g. run-remote.sh) +# โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ + +ENV_FILE="${REPO_DIR}/.resp-bench-env" +: > "${ENV_FILE}" +env_add() { + # Append a line to the env snippet and apply it to the current shell too. + printf '%s\n' "$1" >> "${ENV_FILE}" + eval "$1" +} + +# โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ +# BASE: common build tooling shared by every engine + the server build +# โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ + +log "installing base build tooling" +if [ "${PKG}" = "dnf" ]; then + # gcc/make/etc. for compiling the Valkey server and native gems. + # libicu: required by the .NET runtime (dotnet aborts with an ICU error + # otherwise, which would break the C# engine's build/run). + pkg_install git tar gzip make gcc gcc-c++ openssl-devel pkgconf-pkg-config \ + findutils which jq ca-certificates procps-ng libicu -- \ + git tar gzip make build-essential libssl-dev pkg-config \ + findutils jq ca-certificates procps libicu-dev +else + pkg_install git libicu-dev -- git tar gzip make build-essential libssl-dev \ + pkg-config findutils jq ca-certificates procps libicu-dev +fi + +# โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ• +# LANGUAGE: Java (JDK 21 + Maven) โ€” engines: jedis, lettuce, valkey-glide, +# redisson, spring-data-valkey, spring-data-redis +# NOTE: JDK 21 is mandatory โ€” the Java engine uses virtual threads +# (Executors.newVirtualThreadPerTaskExecutor(), a Java 21 API). JDK 17 will +# hard-fail the build. See resp-bench Phase 0.5 (pom.xml pins java.version=21). +# โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ• + +log "LANGUAGE: Java โ€” installing JDK 21 + Maven" +if [ "${PKG}" = "dnf" ]; then + pkg_install java-21-amazon-corretto-devel maven -- openjdk-21-jdk maven +else + pkg_install openjdk-21-jdk maven -- openjdk-21-jdk maven +fi + +# JDK 21 may install alongside an older default JDK, so `mvn` can still pick up +# a pre-21 java โ€” and the pom's enforce-java-21 rule then hard-fails the build. +# Pin JAVA_HOME/PATH to the 21 install explicitly and persist it via the env +# snippet so the sweep-time rebuild (run-remote.sh) uses it too. +JAVA21_HOME="" +for d in /usr/lib/jvm/*corretto*21* /usr/lib/jvm/java-21* /usr/lib/jvm/jdk-21* \ + /usr/lib/jvm/*-21-*; do + if [ -x "${d}/bin/javac" ]; then JAVA21_HOME="${d}"; break; fi +done +[ -n "${JAVA21_HOME}" ] || { err "JDK 21 not found under /usr/lib/jvm after install"; exit 1; } +env_add "export JAVA_HOME=\"${JAVA21_HOME}\"" +env_add "export PATH=\"${JAVA21_HOME}/bin:\$PATH\"" +hash -r + +# Maven may not be packaged in some dnf repos; ensure it exists. +if ! command -v mvn >/dev/null 2>&1; then + pkg_install maven -- maven +fi +java -version 2>&1 | sed 's/^/[provision] /' || true +mvn -version 2>&1 | head -n2 | sed 's/^/[provision] /' || true + +# โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ• +# LANGUAGE: Ruby (3.2+ + bundler + dev headers) โ€” engines: redis-rb, +# valkey-glide-ruby. Native extensions (HDRHistogram, oj) need a compiler + +# ruby headers. The Gemfile pulls `valkey` from a git branch, so the first +# `bundle install` needs network access (warmed below). +# โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ• + +log "LANGUAGE: Ruby โ€” installing Ruby + bundler + dev headers" +if ! command -v ruby >/dev/null 2>&1; then + pkg_install ruby ruby-devel rubygems -- ruby ruby-dev +else + # Ensure dev headers are present even if ruby itself already is. + pkg_install ruby-devel -- ruby-dev || true +fi +if ! command -v bundle >/dev/null 2>&1 && ! ruby -e 'require "bundler"' >/dev/null 2>&1; then + ${SUDO} gem install --no-document bundler +fi +ruby -v 2>&1 | sed 's/^/[provision] /' || true + +# โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ• +# LANGUAGE: .NET SDK (net10.0) โ€” engines: stackexchange-redis, +# valkey-glide-csharp. net10.0 may not be in the distro feed, so install via +# Microsoft's official dotnet-install.sh, which is feed-independent. +# โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ• + +log "LANGUAGE: .NET โ€” installing SDK channel ${DOTNET_CHANNEL}" +# Choose an install dir we can actually write to. +DOTNET_ROOT="${DOTNET_ROOT:-/usr/local/dotnet}" +if ! mkdir -p "${DOTNET_ROOT}" 2>/dev/null; then + if [ -n "${SUDO}" ] && ${SUDO} mkdir -p "${DOTNET_ROOT}" 2>/dev/null; then + ${SUDO} chown "$(id -u):$(id -g)" "${DOTNET_ROOT}" + else + DOTNET_ROOT="${HOME}/.dotnet" + mkdir -p "${DOTNET_ROOT}" + fi +fi +if [ ! -x "${DOTNET_ROOT}/dotnet" ]; then + curl -fsSL https://dot.net/v1/dotnet-install.sh -o /tmp/dotnet-install.sh + bash /tmp/dotnet-install.sh --channel "${DOTNET_CHANNEL}" --install-dir "${DOTNET_ROOT}" + rm -f /tmp/dotnet-install.sh +else + log ".NET SDK already present at ${DOTNET_ROOT}" +fi +env_add "export DOTNET_ROOT=\"${DOTNET_ROOT}\"" +env_add "export PATH=\"${DOTNET_ROOT}:${DOTNET_ROOT}/tools:\$PATH\"" +# Opt out of first-run telemetry noise during unattended runs. +env_add "export DOTNET_CLI_TELEMETRY_OPTOUT=1" +env_add "export DOTNET_NOLOGO=1" +"${DOTNET_ROOT}/dotnet" --info 2>&1 | head -n3 | sed 's/^/[provision] /' || true + +# โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ• +# LANGUAGE: Node.js (20+) โ€” engines: valkey-glide-node, ioredis, iovalkey. +# node/package.json declares "engines": {"node": ">=20"}; distro feeds often +# ship 18 (EOL) or older, so install from NodeSource, which is version-pinned +# and works on both dnf and apt. valkey-glide ships prebuilt native binaries +# per platform, so no compiler is needed beyond build-essential (already +# installed above for the server build). +# โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ• + +NODE_MAJOR="${NODE_MAJOR:-22}" +log "LANGUAGE: Node.js โ€” installing Node ${NODE_MAJOR} + npm" +node_major_installed() { + command -v node >/dev/null 2>&1 && + [ "$(node -p 'process.versions.node.split(".")[0]' 2>/dev/null || echo 0)" -ge 20 ] +} +if ! node_major_installed; then + if [ "${PKG}" = "dnf" ]; then + curl -fsSL "https://rpm.nodesource.com/setup_${NODE_MAJOR}.x" | ${SUDO} bash - + pkg_install nodejs -- nodejs + else + curl -fsSL "https://deb.nodesource.com/setup_${NODE_MAJOR}.x" | ${SUDO} bash - + pkg_install nodejs -- nodejs + fi +else + log "Node.js $(node -v) already present (>= 20)" +fi +node -v 2>&1 | sed 's/^/[provision] /' || true +npm -v 2>&1 | sed 's/^/[provision] /' || true + +# โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ• +# LANGUAGE: Python (3.9+ + pip) โ€” used by the matrix orchestrator and graph +# generator (scripts/*.py). Engine deps are installed from the repo's pinned +# requirements files when present. +# โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ• + +log "LANGUAGE: Python 3.11 โ€” installing (scripts/requirements.txt pins numpy/" +log " matplotlib versions that require Python >= 3.11; AL2023 ships only 3.9)" +pkg_install python3.11 python3.11-pip -- python3.11 python3.11-venv python3-pip + +# The Makefile's benchmark targets invoke bare `python`, so make 3.11 the +# default `python`/`python3`. /usr/local/bin precedes /usr/bin on PATH, so this +# shadows the stock 3.9 without disturbing the OS's own python3. +PY311="$(command -v python3.11)" +[ -n "${PY311}" ] || err "python3.11 not found after install" +${SUDO} ln -sf "${PY311}" /usr/local/bin/python3 +${SUDO} ln -sf "${PY311}" /usr/local/bin/python +hash -r + +python3.11 -m ensurepip --upgrade >/dev/null 2>&1 || true +if [ -f "${REPO_DIR}/scripts/requirements.txt" ]; then + log "installing scripts/requirements.txt with Python 3.11" + python3.11 -m pip install --user -r "${REPO_DIR}/scripts/requirements.txt" +fi +python --version 2>&1 | sed 's/^/[provision] /' || true + +# โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ +# FUTURE ENGINES (leave room โ€” see resp-bench plan ยง5.5): +# Go (#14): install the Go toolchain + `go mod download` +# PHP (#15): pkg_install php php-cli composer -- php-cli composer +# Add each as its own "LANGUAGE:" block above, mirroring the pattern. +# โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ + +# โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ +# Make the env snippet available to interactive logins as well (best-effort). +# โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ +if [ -w /etc/profile.d ] || { [ -n "${SUDO}" ] && ${SUDO} test -w /etc; }; then + if [ -n "${SUDO}" ]; then + ${SUDO} cp "${ENV_FILE}" /etc/profile.d/resp-bench.sh 2>/dev/null || true + else + cp "${ENV_FILE}" /etc/profile.d/resp-bench.sh 2>/dev/null || true + fi +fi + +# โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ +# Build the Valkey server. Without this, `make server-standalone-start` +# compiles Valkey from source on first use, inside the timed run. +# โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ + +if [ "${SKIP_SERVER_BUILD:-0}" != "1" ]; then + log "building Valkey server (this compiles from source once)" + # The Makefile expresses the server binary as a file target at an absolute + # path; building that path compiles + installs the server into work//. + # SERVER_PROJECT matches the Makefile default (valkey); override via env if + # the Makefile default ever changes. + SERVER_PROJECT="${SERVER_PROJECT:-valkey}" + make -C "${REPO_DIR}" "${REPO_DIR}/work/${SERVER_PROJECT}/bin/${SERVER_PROJECT}-server" +else + log "SKIP_SERVER_BUILD=1 โ€” skipping Valkey server build" +fi + +# โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ +# Warm the per-engine build caches with one throwaway build each, so the sweep +# itself does no first-time compilation / dependency resolution. Sourcing the +# env file makes dotnet/etc. visible to the make sub-shells. +# โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ + +if [ "${SKIP_WARM_CACHES:-0}" != "1" ]; then + # shellcheck source=/dev/null + . "${ENV_FILE}" + # Warm-up is a cache optimization, not a correctness gate โ€” so each build is + # best-effort. A broken engine that the requested matrix does not use must not + # block provisioning; the sweep's own build_engines() step rebuilds exactly + # the engines the matrix needs and fails loudly (with diagnostics uploaded to + # S3) if one of those is broken. + log "warming Java build cache (mvn package)" + make -C "${REPO_DIR}" java-build || log "WARNING: Java warm-up build failed (see above)" + log "warming Ruby bundle (bundle install)" + make -C "${REPO_DIR}" ruby-build || log "WARNING: Ruby warm-up build failed (see above)" + log "warming C# build cache (dotnet build)" + make -C "${REPO_DIR}" csharp-build || log "WARNING: C# warm-up build failed (see above)" + log "warming Node.js deps (npm ci + tsc)" + make -C "${REPO_DIR}" node-build || log "WARNING: Node.js warm-up build failed (see above)" +else + log "SKIP_WARM_CACHES=1 โ€” skipping engine cache warm-up" +fi + +log "provisioning complete" diff --git a/java/README.md b/java/README.md index 0af7fc6..fef2d8d 100644 --- a/java/README.md +++ b/java/README.md @@ -15,8 +15,9 @@ Java implementation of the resp-bench benchmark suite. ## Requirements -- Java 17 or later -- Maven 3.6 or later +- Java 21 or later, to build and to run โ€” the engine uses virtual threads + (`Executors.newVirtualThreadPerTaskExecutor()`), and the jar is compiled for Java 21 +- Maven 3.6.3 or later ## Building diff --git a/java/pom.xml b/java/pom.xml index 986b800..6a8339f 100644 --- a/java/pom.xml +++ b/java/pom.xml @@ -13,9 +13,14 @@ Benchmark engine for Java Valkey/Redis client libraries - 17 - ${java.version} - ${java.version} + + 21 + ${java.version} UTF-8 @@ -170,6 +175,29 @@ + + + org.apache.maven.plugins + maven-enforcer-plugin + 3.5.0 + + + enforce-java-21 + + enforce + + + + + [${java.version},) + resp-bench requires JDK ${java.version} or newer: the Java engine uses virtual threads (Executors.newVirtualThreadPerTaskExecutor()). + + + + + + + io.github.git-commit-id @@ -204,8 +232,7 @@ maven-compiler-plugin 3.13.0 - ${java.version} - ${java.version} + ${java.version} info.picocli diff --git a/node/.gitignore b/node/.gitignore new file mode 100644 index 0000000..c19bb02 --- /dev/null +++ b/node/.gitignore @@ -0,0 +1,4 @@ +node_modules/ +dist/ +coverage/ +*.tsbuildinfo diff --git a/node/README.md b/node/README.md new file mode 100644 index 0000000..af7834f --- /dev/null +++ b/node/README.md @@ -0,0 +1,141 @@ +# resp-bench Node.js Engine + +Node.js implementation of the resp-bench benchmark suite, written in TypeScript. + +## Supported Drivers + +| `driver_id` | Package | Notes | +|---|---|---| +| `valkey-glide-node` | [`@valkey/valkey-glide`](https://www.npmjs.com/package/@valkey/valkey-glide) | Valkey GLIDE for Node.js. Ships prebuilt native binaries per platform. | +| `ioredis` | [`ioredis`](https://www.npmjs.com/package/ioredis) | The most widely used Node.js Redis client. | +| `iovalkey` | [`iovalkey`](https://www.npmjs.com/package/iovalkey) | The Valkey-maintained fork of ioredis; API-identical. | +| `recording` | โ€” | In-memory synthetic-latency client for server-free tests. | + +> **Note:** the GLIDE driver id is `valkey-glide-node`, not `valkey-glide`. The +> latter is already the Java engine's id in the global `DRIVER_ENGINE_MAP` in +> `scripts/run_benchmark_matrix.py`, and reusing it would reroute Java's runs here. + +## Requirements + +- Node.js **20 or newer** (`package.json` declares `engines.node >= 20`). CI uses 22 LTS. + +## Build + +```bash +npm ci && npm run build # or: make node-build +``` + +TypeScript compiles to `dist/`, mirroring the source tree, so the entry point is +`dist/src/cli.js`. + +## Usage + +```bash +node dist/src/cli.js \ + --server localhost:6379 \ + --driver ../configs/drivers/default/ioredis.json \ + --workload ../configs/workloads/reference/basic-standalone-single-client-1M-reqs.json \ + --metrics ../output/node.ndjson +``` + +Or through the Makefile, from the repository root: + +```bash +make node-run \ + DRIVER=configs/drivers/default/valkey-glide-node.json \ + WORKLOAD=configs/workloads/reference/basic-standalone-single-client-1M-reqs.json \ + METRICS_OUTPUT=output/node.ndjson + +make node-info # list supported drivers and commands +``` + +## Tests + +```bash +make node-test # unit + integration (starts/stops a server) +cd node && npm run test:unit # unit only, no server needed +``` + +The live-server tests skip themselves unless `VALKEY_HOST` is set: + +```bash +cd node && VALKEY_HOST=localhost VALKEY_PORT=6379 npm run test:integration +``` + +The `recording` driver lets the full engine โ€” phases, warmup, budget, pipelining, +rate limiting, NDJSON โ€” be exercised with no server at all. + +## Concurrency Model + +A single event loop with **one client per connection** and **one worker per +connection**, all started together. Node is single-threaded with async I/O, so an +awaited command parks that worker rather than the loop, and the other connections +keep making progress. This is the analogue of Java's +virtual-thread-per-client design. + +`pipeline_depth > 1` is supported: each connection runs that many independent +issue/await/record slots, so a settled request is replaced immediately. + +The per-phase request budget is **shared across all workers** and claimed one +request at a time, matching the Java reference's `AtomicLong`. It is deliberately +not pre-divided per worker โ€” a shared budget lets fast connections absorb a slow +one's slack, so wall-clock is not bounded by the slowest connection. + +## Cross-Engine Parity + +Verified against the Java reference engine, not assumed: + +- **Key sequences are byte-identical.** 79,000 keys were diffed against Java's + actual `KeyGenerator` across both algorithms, 1โ€“16 workers, prime `keys_count`, + and tight padding. `src/engine/javaRandom.ts` is a `java.util.Random` port + anchored to `new Random(0).nextInt() == -1155484576`; it uses `BigInt` because + the 48-bit LCG multiply overflows a JS `number`. +- **HDR payloads are mutually decodable.** `payload_b64` is + `encodeIntoCompressedBase64()` used directly (it is *already* base64 โ€” encoding + it again would produce something Java cannot read). Java's + `Histogram.decodeFromCompressedByteBuffer` reads our payloads with matching + count and percentiles. Range is `(1, 600_000_000, 3)`, as in every engine. +- **`summary.min`/`max` use `getValueAtPercentile(0/100)`**, not + `minNonZeroValue`/`maxValue`. Java reports the bucket's equivalent bounds, and + hdr-histogram-js' properties return the raw sample โ€” they diverge above ~1000ยตs + (recording 50000ยตs yields 50000 in JS but 50015 in Java). +- **PING does not consume a key**, matching Java's `PingCommand`, so mixing PING + into a workload does not shift the key sequence. + +## Node-Specific Fairness Notes + +Read these before comparing Node numbers to another engine: + +- **Auto-pipelining is explicitly disabled.** ioredis and iovalkey can + transparently batch commands issued in the same event-loop tick, which would + inflate throughput against every other engine. `enableAutoPipelining: false`. +- **Reconnects are disabled** (`retryStrategy: () => null`). ioredis otherwise + retries forever, so a wrong host would hang a run instead of failing it, and a + mid-phase reconnect would fold connection setup into request latency. +- **All drivers decode responses the same way.** `get` returns a `string` in all + three, so none is charged for a different amount of decoding. +- **SET payloads are built once** per command object, not per request, so the + driver is not charged for the engine's own allocation churn. +- **Sub-millisecond rate limits work.** `setTimeout` clamps to ~1 ms, so the + limiter yields via `setImmediate` for shorter intervals; a 100k rps limit is a + 10 ยตs interval and a timer-based wait would undershoot it by ~100ร—. +- **RSS is not comparable to the JVM's.** The system monitor's memory samples + include V8 heap growth and GC timing, which behave differently from the JVM's. +- **One event loop is one core.** At high connection counts the engine itself, not + the client, may become the ceiling. See `docs/BENCHMARKS_NODE.md`. + +## Layout + +``` +node/ +โ”œโ”€โ”€ src/ +โ”‚ โ”œโ”€โ”€ cli.ts # arg parsing, --info, error boundary +โ”‚ โ”œโ”€โ”€ client/ # driver interface, factory, per-driver impls +โ”‚ โ”œโ”€โ”€ command/ # GET / SET / PING +โ”‚ โ”œโ”€โ”€ config/ # JSON config parsing + validation +โ”‚ โ”œโ”€โ”€ engine/ # benchmark loops, key gen, RNG, rate limiter +โ”‚ โ””โ”€โ”€ metrics/ # HDR histogram, collector, NDJSON writer +โ””โ”€โ”€ test/ + โ”œโ”€โ”€ unit/ # no server, no optional deps + โ””โ”€โ”€ integration/ # recording driver + live-server tests +``` diff --git a/node/package-lock.json b/node/package-lock.json new file mode 100644 index 0000000..068c475 --- /dev/null +++ b/node/package-lock.json @@ -0,0 +1,420 @@ +{ + "name": "resp-bench-node", + "version": "1.0.0", + "lockfileVersion": 3, + "requires": true, + "packages": { + "": { + "name": "resp-bench-node", + "version": "1.0.0", + "license": "Apache-2.0", + "dependencies": { + "@valkey/valkey-glide": "^2.5.2", + "hdr-histogram-js": "^3.0.1", + "ioredis": "^5.11.1", + "iovalkey": "^0.4.0" + }, + "devDependencies": { + "@types/node": "^20.19.0", + "typescript": "^5.9.2" + }, + "engines": { + "node": ">=20" + } + }, + "node_modules/@assemblyscript/loader": { + "version": "0.19.23", + "resolved": "https://registry.npmjs.org/@assemblyscript/loader/-/loader-0.19.23.tgz", + "integrity": "sha512-ulkCYfFbYj01ie1MDOyxv2F6SpRN1TOj7fQxbP07D6HmeR+gr2JLSmINKjga2emB+b1L2KGrFKBTc+e00p54nw==", + "license": "Apache-2.0" + }, + "node_modules/@ioredis/commands": { + "version": "1.10.0", + "resolved": "https://registry.npmjs.org/@ioredis/commands/-/commands-1.10.0.tgz", + "integrity": "sha512-UmeW7z4LfctwoQ5wkhVzgq8tXkreED2xZGpX+Bg+zA+WJFZCT6c062AfCK/Dfk81xZnnwdhJCUMkitihRaoC2Q==", + "license": "MIT" + }, + "node_modules/@iovalkey/commands": { + "version": "0.1.0", + "resolved": "https://registry.npmjs.org/@iovalkey/commands/-/commands-0.1.0.tgz", + "integrity": "sha512-/B9W4qKSSITDii5nkBCHyPkIkAi+ealUtr1oqBJsLxjSRLka4pxun2VvMNSmcwgAMxgXtQfl0qRv7TE+udPJzg==", + "license": "MIT" + }, + "node_modules/@protobufjs/aspromise": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/@protobufjs/aspromise/-/aspromise-1.1.2.tgz", + "integrity": "sha512-j+gKExEuLmKwvz3OgROXtrJ2UG2x8Ch2YZUxahh+s1F2HZ+wAceUNLkvy6zKCPVRkU++ZWQrdxsUeQXmcg4uoQ==", + "license": "BSD-3-Clause" + }, + "node_modules/@protobufjs/base64": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/@protobufjs/base64/-/base64-1.1.2.tgz", + "integrity": "sha512-AZkcAA5vnN/v4PDqKyMR5lx7hZttPDgClv83E//FMNhR2TMcLUhfRUBHCmSl0oi9zMgDDqRUJkSxO3wm85+XLg==", + "license": "BSD-3-Clause" + }, + "node_modules/@protobufjs/codegen": { + "version": "2.0.5", + "resolved": "https://registry.npmjs.org/@protobufjs/codegen/-/codegen-2.0.5.tgz", + "integrity": "sha512-zgXFLzW3Ap33e6d0Wlj4MGIm6Ce8O89n/apUaGNB/jx+hw+ruWEp7EwGUshdLKVRCxZW12fp9r40E1mQrf/34g==", + "license": "BSD-3-Clause" + }, + "node_modules/@protobufjs/eventemitter": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/@protobufjs/eventemitter/-/eventemitter-1.1.1.tgz", + "integrity": "sha512-vW1GmwMZNnL+gMRaovlh9yZX74kc+TTU3FObkkurpMaRtBfLP3ldjS9KQWlwZgraRE0+dheEEoAxdzcJQ8eXZg==", + "license": "BSD-3-Clause" + }, + "node_modules/@protobufjs/fetch": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/@protobufjs/fetch/-/fetch-1.1.1.tgz", + "integrity": "sha512-GpptLrs57adMSuHi3VNj0mAF8dwh36LMaYF6XyJ6JMWlVsc+t42tm1HSEDmOs3A8fC9yyeisgLhsTVQokOZ0zw==", + "license": "BSD-3-Clause", + "dependencies": { + "@protobufjs/aspromise": "^1.1.1" + } + }, + "node_modules/@protobufjs/float": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/@protobufjs/float/-/float-1.0.2.tgz", + "integrity": "sha512-Ddb+kVXlXst9d+R9PfTIxh1EdNkgoRe5tOX6t01f1lYWOvJnSPDBlG241QLzcyPdoNTsblLUdujGSE4RzrTZGQ==", + "license": "BSD-3-Clause" + }, + "node_modules/@protobufjs/path": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/@protobufjs/path/-/path-1.1.2.tgz", + "integrity": "sha512-6JOcJ5Tm08dOHAbdR3GrvP+yUUfkjG5ePsHYczMFLq3ZmMkAD98cDgcT2iA1lJ9NVwFd4tH/iSSoe44YWkltEA==", + "license": "BSD-3-Clause" + }, + "node_modules/@protobufjs/pool": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/@protobufjs/pool/-/pool-1.1.0.tgz", + "integrity": "sha512-0kELaGSIDBKvcgS4zkjz1PeddatrjYcmMWOlAuAPwAeccUrPHdUqo/J6LiymHHEiJT5NrF1UVwxY14f+fy4WQw==", + "license": "BSD-3-Clause" + }, + "node_modules/@protobufjs/utf8": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/@protobufjs/utf8/-/utf8-1.1.2.tgz", + "integrity": "sha512-b1UQwcEZ4yCnMCD8DAL1VlbvBJE9/IX4FTIp7BG1xYpf29SLazLSrqUkj4w7Y5y7cCVP6E5tcqqcI0xemPkHug==", + "license": "BSD-3-Clause" + }, + "node_modules/@types/node": { + "version": "20.19.43", + "resolved": "https://registry.npmjs.org/@types/node/-/node-20.19.43.tgz", + "integrity": "sha512-6oYBAi5ikg4Pl+kGsoYtawUMBT2zZMCvPNF7pVLnHZfd1zf38DRiWn/gT01RYCdUqkv7Fhr+C9ot4/tb+2sVvA==", + "license": "MIT", + "dependencies": { + "undici-types": "~6.21.0" + } + }, + "node_modules/@valkey/valkey-glide": { + "version": "2.5.2", + "resolved": "https://registry.npmjs.org/@valkey/valkey-glide/-/valkey-glide-2.5.2.tgz", + "integrity": "sha512-8d3m2NyveQ2Ws4OUd7Zz3zsPP835Nqi4Rm3ZMoLdUjTfnEJejIMPJGR8b09foiPEG4GXV9j+QWyDRuM9VHOuVw==", + "license": "Apache-2.0", + "dependencies": { + "long": "5", + "protobufjs": "^7.6.3" + }, + "engines": { + "node": ">=16" + }, + "optionalDependencies": { + "@valkey/valkey-glide-darwin-arm64": "2.5.2", + "@valkey/valkey-glide-darwin-x64": "2.5.2", + "@valkey/valkey-glide-linux-arm64-gnu": "2.5.2", + "@valkey/valkey-glide-linux-arm64-musl": "2.5.2", + "@valkey/valkey-glide-linux-x64-gnu": "2.5.2", + "@valkey/valkey-glide-linux-x64-musl": "2.5.2" + } + }, + "node_modules/@valkey/valkey-glide-darwin-arm64": { + "version": "2.5.2", + "resolved": "https://registry.npmjs.org/@valkey/valkey-glide-darwin-arm64/-/valkey-glide-darwin-arm64-2.5.2.tgz", + "integrity": "sha512-6kmFsyBrMj8kRF72ZiKpLsT0WZrRQdM3P2p9vJskRaXbRiMGmLVoqj8vv1tYcu/xRr7bI6aEONOlwKbHojRLkQ==", + "cpu": [ + "arm64" + ], + "license": "Apache-2.0", + "optional": true, + "os": [ + "darwin" + ] + }, + "node_modules/@valkey/valkey-glide-darwin-x64": { + "version": "2.5.2", + "resolved": "https://registry.npmjs.org/@valkey/valkey-glide-darwin-x64/-/valkey-glide-darwin-x64-2.5.2.tgz", + "integrity": "sha512-2RgllAJlYvoKirMKDIseYfX7LNFOF06UohqRB4LM2U3Y9mjE/ssNrPy+netFYdqz0aWcgYajzCMr2jXeckmoEg==", + "cpu": [ + "x64" + ], + "license": "Apache-2.0", + "optional": true, + "os": [ + "darwin" + ] + }, + "node_modules/@valkey/valkey-glide-linux-arm64-gnu": { + "version": "2.5.2", + "resolved": "https://registry.npmjs.org/@valkey/valkey-glide-linux-arm64-gnu/-/valkey-glide-linux-arm64-gnu-2.5.2.tgz", + "integrity": "sha512-EVWT7VT9porB670AxjQ+b/g/Dx5Nm/GaMy9irCzdDdnnTOwPyHvK1kPFzMATNuyDgXd5L504i99u8iHL1GpDEw==", + "cpu": [ + "arm64" + ], + "license": "Apache-2.0", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@valkey/valkey-glide-linux-arm64-musl": { + "version": "2.5.2", + "resolved": "https://registry.npmjs.org/@valkey/valkey-glide-linux-arm64-musl/-/valkey-glide-linux-arm64-musl-2.5.2.tgz", + "integrity": "sha512-CFTrYPWIrh+k66eqH3/FvUlMZ4S7p328iFTK+qsHxyYH3CnhAmKMaOHcptFKz5n7ypT8L/XuOBl+NIQgUrJGQA==", + "cpu": [ + "arm64" + ], + "license": "Apache-2.0", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@valkey/valkey-glide-linux-x64-gnu": { + "version": "2.5.2", + "resolved": "https://registry.npmjs.org/@valkey/valkey-glide-linux-x64-gnu/-/valkey-glide-linux-x64-gnu-2.5.2.tgz", + "integrity": "sha512-osWsmJ+cg2iZjjNZ5IHQUMg/Xb/TQVyBvDxKA11vpLuEFMulQYfUfw+LWo0PwnG2mGJGKHhDnZ8KZAJpd03LWQ==", + "cpu": [ + "x64" + ], + "license": "Apache-2.0", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@valkey/valkey-glide-linux-x64-musl": { + "version": "2.5.2", + "resolved": "https://registry.npmjs.org/@valkey/valkey-glide-linux-x64-musl/-/valkey-glide-linux-x64-musl-2.5.2.tgz", + "integrity": "sha512-gJCma4s3j3AUQk3k2Kcxe3ImWpla4e9jr41f1w9VwZd6CkWuZQdk1PsLpdROvtaBxUiRiq+RZvvLKvGf0md1Kg==", + "cpu": [ + "x64" + ], + "license": "Apache-2.0", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/base64-js": { + "version": "1.5.1", + "resolved": "https://registry.npmjs.org/base64-js/-/base64-js-1.5.1.tgz", + "integrity": "sha512-AKpaYlHn8t4SVbOHCy+b5+KKgvR4vrsD8vbvrbiQJps7fKDTkjkDry6ji0rUJjC0kzbNePLwzxq8iypo41qeWA==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ], + "license": "MIT" + }, + "node_modules/cluster-key-slot": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/cluster-key-slot/-/cluster-key-slot-1.1.1.tgz", + "integrity": "sha512-rwHwUfXL40Chm1r08yrhU3qpUvdVlgkKNeyeGPOxnW8/SyVDvgRaed/Uz54AqWNaTCAThlj6QAs3TZcKI0xDEw==", + "license": "Apache-2.0", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/debug": { + "version": "4.4.3", + "resolved": "https://registry.npmjs.org/debug/-/debug-4.4.3.tgz", + "integrity": "sha512-RGwwWnwQvkVfavKVt22FGLw+xYSdzARwm0ru6DhTVA3umU5hZc28V3kO4stgYryrTlLpuvgI9GiijltAjNbcqA==", + "license": "MIT", + "dependencies": { + "ms": "^2.1.3" + }, + "engines": { + "node": ">=6.0" + }, + "peerDependenciesMeta": { + "supports-color": { + "optional": true + } + } + }, + "node_modules/denque": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/denque/-/denque-2.1.0.tgz", + "integrity": "sha512-HVQE3AAb/pxF8fQAoiqpvg9i3evqug3hoiwakOyZAwJm+6vZehbkYXZ0l4JxS+I3QxM97v5aaRNhj8v5oBhekw==", + "license": "Apache-2.0", + "engines": { + "node": ">=0.10" + } + }, + "node_modules/hdr-histogram-js": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/hdr-histogram-js/-/hdr-histogram-js-3.0.1.tgz", + "integrity": "sha512-l3GSdZL1Jr1C0kyb461tUjEdrRPZr8Qry7jByltf5JGrA0xvqOSrxRBfcrJqqV/AMEtqqhHhC6w8HW0gn76tRQ==", + "license": "BSD-2-Clause", + "dependencies": { + "@assemblyscript/loader": "^0.19.21", + "base64-js": "^1.2.0", + "pako": "^1.0.3" + }, + "engines": { + "node": ">=14" + } + }, + "node_modules/ioredis": { + "version": "5.11.1", + "resolved": "https://registry.npmjs.org/ioredis/-/ioredis-5.11.1.tgz", + "integrity": "sha512-ehuGcf94bQXhfagULNXrJdfnWO38v070jxSx/qE87Kjzmu2fU7ro5EFAb+OPituLqgfyuQaym5DlrNydW2sJ9A==", + "license": "MIT", + "dependencies": { + "@ioredis/commands": "1.10.0", + "cluster-key-slot": "1.1.1", + "debug": "4.4.3", + "denque": "2.1.0", + "redis-errors": "1.2.0", + "redis-parser": "3.0.0", + "standard-as-callback": "2.1.0" + }, + "engines": { + "node": ">=12.22.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/ioredis" + } + }, + "node_modules/iovalkey": { + "version": "0.4.0", + "resolved": "https://registry.npmjs.org/iovalkey/-/iovalkey-0.4.0.tgz", + "integrity": "sha512-OSUKxJ+s44CLdUTaicX4+pVrBN/zHKIYwr2oKhRDuczr19FwcSCXEZHzcMUYWZNh1mNSCV9bNJHW/T6cHVm9Aw==", + "license": "MIT", + "dependencies": { + "@iovalkey/commands": "^0.1.0", + "cluster-key-slot": "^1.1.0", + "debug": "^4.3.4", + "denque": "^2.1.0", + "lodash.defaults": "^4.2.0", + "lodash.isarguments": "^3.1.0", + "redis-errors": "^1.2.0", + "redis-parser": "^3.0.0", + "standard-as-callback": "^2.1.0" + }, + "engines": { + "node": ">=18.12.0" + } + }, + "node_modules/lodash.defaults": { + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/lodash.defaults/-/lodash.defaults-4.2.0.tgz", + "integrity": "sha512-qjxPLHd3r5DnsdGacqOMU6pb/avJzdh9tFX2ymgoZE27BmjXrNy/y4LoaiTeAb+O3gL8AfpJGtqfX/ae2leYYQ==", + "license": "MIT" + }, + "node_modules/lodash.isarguments": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/lodash.isarguments/-/lodash.isarguments-3.1.0.tgz", + "integrity": "sha512-chi4NHZlZqZD18a0imDHnZPrDeBbTtVN7GXMwuGdRH9qotxAjYs3aVLKc7zNOG9eddR5Ksd8rvFEBc9SsggPpg==", + "license": "MIT" + }, + "node_modules/long": { + "version": "5.3.2", + "resolved": "https://registry.npmjs.org/long/-/long-5.3.2.tgz", + "integrity": "sha512-mNAgZ1GmyNhD7AuqnTG3/VQ26o760+ZYBPKjPvugO8+nLbYfX6TVpJPseBvopbdY+qpZ/lKUnmEc1LeZYS3QAA==", + "license": "Apache-2.0" + }, + "node_modules/ms": { + "version": "2.1.3", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", + "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==", + "license": "MIT" + }, + "node_modules/pako": { + "version": "1.0.11", + "resolved": "https://registry.npmjs.org/pako/-/pako-1.0.11.tgz", + "integrity": "sha512-4hLB8Py4zZce5s4yd9XzopqwVv/yGNhV1Bl8NTmCq1763HeK2+EwVTv+leGeL13Dnh2wfbqowVPXCIO0z4taYw==", + "license": "(MIT AND Zlib)" + }, + "node_modules/protobufjs": { + "version": "7.6.6", + "resolved": "https://registry.npmjs.org/protobufjs/-/protobufjs-7.6.6.tgz", + "integrity": "sha512-dYDWdjSl5RNb7SgPxGQcRU+GtvP7s2fpkrY0r432PcOIaZ0/rBcxEZnQN67iJhFuQiVw754JDoPruPCNdGsbjg==", + "hasInstallScript": true, + "license": "BSD-3-Clause", + "dependencies": { + "@protobufjs/aspromise": "^1.1.2", + "@protobufjs/base64": "^1.1.2", + "@protobufjs/codegen": "^2.0.5", + "@protobufjs/eventemitter": "^1.1.1", + "@protobufjs/fetch": "^1.1.1", + "@protobufjs/float": "^1.0.2", + "@protobufjs/path": "^1.1.2", + "@protobufjs/pool": "^1.1.0", + "@protobufjs/utf8": "^1.1.1", + "@types/node": ">=13.7.0", + "long": "^5.3.2" + }, + "engines": { + "node": ">=12.0.0" + } + }, + "node_modules/redis-errors": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/redis-errors/-/redis-errors-1.2.0.tgz", + "integrity": "sha512-1qny3OExCf0UvUV/5wpYKf2YwPcOqXzkwKKSmKHiE6ZMQs5heeE/c8eXK+PNllPvmjgAbfnsbpkGZWy8cBpn9w==", + "license": "MIT", + "engines": { + "node": ">=4" + } + }, + "node_modules/redis-parser": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/redis-parser/-/redis-parser-3.0.0.tgz", + "integrity": "sha512-DJnGAeenTdpMEH6uAJRK/uiyEIH9WVsUmoLwzudwGJUwZPp80PDBWPHXSAGNPwNvIXAbe7MSUB1zQFugFml66A==", + "license": "MIT", + "dependencies": { + "redis-errors": "^1.0.0" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/standard-as-callback": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/standard-as-callback/-/standard-as-callback-2.1.0.tgz", + "integrity": "sha512-qoRRSyROncaz1z0mvYqIE4lCd9p2R90i6GxW3uZv5ucSu8tU7B5HXUP1gG8pVZsYNVaXjk8ClXHPttLyxAL48A==", + "license": "MIT" + }, + "node_modules/typescript": { + "version": "5.9.3", + "resolved": "https://registry.npmjs.org/typescript/-/typescript-5.9.3.tgz", + "integrity": "sha512-jl1vZzPDinLr9eUt3J/t7V6FgNEw9QjvBPdysz9KfQDD41fQrC2Y4vKQdiaUpFT4bXlb1RHhLpp8wtm6M5TgSw==", + "dev": true, + "license": "Apache-2.0", + "bin": { + "tsc": "bin/tsc", + "tsserver": "bin/tsserver" + }, + "engines": { + "node": ">=14.17" + } + }, + "node_modules/undici-types": { + "version": "6.21.0", + "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-6.21.0.tgz", + "integrity": "sha512-iwDZqg0QAGrg9Rav5H4n0M64c3mkR59cJ6wQp+7C4nI0gsmExaedaYLNO44eT4AtBBwjbTiGPMlt2Md0T9H9JQ==", + "license": "MIT" + } + } +} diff --git a/node/package.json b/node/package.json new file mode 100644 index 0000000..e2b8b39 --- /dev/null +++ b/node/package.json @@ -0,0 +1,30 @@ +{ + "name": "resp-bench-node", + "version": "1.0.0", + "description": "resp-bench Node.js benchmark engine", + "license": "Apache-2.0", + "private": true, + "type": "module", + "engines": { + "node": ">=20" + }, + "scripts": { + "build": "tsc", + "clean": "rm -rf dist", + "typecheck": "tsc --noEmit", + "test": "npm run build && node --test dist/test/", + "test:unit": "npm run build && node --test dist/test/unit/", + "test:integration": "npm run build && node --test dist/test/integration/", + "start": "node dist/src/cli.js" + }, + "dependencies": { + "@valkey/valkey-glide": "2.5.2", + "hdr-histogram-js": "3.0.1", + "ioredis": "5.11.1", + "iovalkey": "0.4.0" + }, + "devDependencies": { + "@types/node": "20.19.43", + "typescript": "5.9.3" + } +} diff --git a/node/src/cli.ts b/node/src/cli.ts new file mode 100644 index 0000000..9c6fef3 --- /dev/null +++ b/node/src/cli.ts @@ -0,0 +1,142 @@ +/** + * Command-line interface for the resp-bench Node.js engine. + * + * Implements the shared cross-engine CLI contract: `--server`, `--driver`, + * `--workload`, `--metrics`, plus `--info`, `--commit-id` (used by CI), and + * `--version`. Deliberately no `--concurrency` flag: the engine has a single + * execution model, and pipelining comes from the workload's `pipeline_depth`. + */ + +import { existsSync } from 'node:fs'; +import { parseArgs } from 'node:util'; + +import { BenchmarkClientFactory } from './client/factory.js'; +import { CommandFactory } from './command/factory.js'; +import { ConfigLoader } from './config/loader.js'; +import { BenchmarkEngine } from './engine/benchmark.js'; +import { VERSION } from './version.js'; + +const DEFAULT_SERVER = 'localhost:6379'; +const DEFAULT_PORT = 6379; + +const USAGE = `resp-bench Node.js engine v${VERSION} + +Usage: + node dist/src/cli.js --driver --workload --metrics [options] + +Options: + --server Server address (default: ${DEFAULT_SERVER}) + --driver Driver configuration JSON (required) + --workload Workload configuration JSON (required) + --metrics Metrics NDJSON output path (required) + --commit-id Git commit ID recorded in the metrics metadata + --info Show supported drivers and commands + --version Show the engine version + --help Show this message +`; + +function parseServer(server: string): { host: string; port: number } { + const separator = server.lastIndexOf(':'); + if (separator === -1) return { host: server || 'localhost', port: DEFAULT_PORT }; + const host = server.slice(0, separator) || 'localhost'; + const port = Number(server.slice(separator + 1)); + if (!Number.isInteger(port) || port <= 0) { + throw new Error(`invalid --server "${server}": port must be a positive integer`); + } + return { host, port }; +} + +function printInfo(): void { + const lines = [ + `resp-bench Node.js Engine v${VERSION}`, + '', + 'Supported Drivers:', + ...BenchmarkClientFactory.describe().map( + ({ driverId, description }) => ` - ${driverId.padEnd(20)} : ${description}`, + ), + '', + 'Supported Commands:', + ...CommandFactory.describe().map(({ name, description }) => ` - ${name.padEnd(10)} : ${description}`), + '', + 'Supported Key Generation Algorithms:', + ' - sequential_int : Sequential integers (0 to keys_count), shared across connections', + ' - uniform_rand : Uniform random, java.util.Random-compatible per connection', + '', + 'Supported Completion Types:', + ' - duration : Run for the specified seconds', + ' - requests : Run until the shared request budget is exhausted', + '', + 'Concurrency: event-loop task-per-connection (one client per connection)', + ]; + console.log(lines.join('\n')); +} + +export async function main(argv: string[] = process.argv.slice(2)): Promise { + let options; + try { + ({ values: options } = parseArgs({ + args: argv, + options: { + server: { type: 'string', default: DEFAULT_SERVER }, + driver: { type: 'string' }, + workload: { type: 'string' }, + metrics: { type: 'string' }, + 'commit-id': { type: 'string' }, + info: { type: 'boolean', default: false }, + version: { type: 'boolean', default: false }, + help: { type: 'boolean', default: false }, + }, + strict: true, + })); + } catch (error) { + console.error(`Error: ${(error as Error).message}`); + console.error(USAGE); + return 1; + } + + if (options.help) { + console.log(USAGE); + return 0; + } + if (options.version) { + console.log(`resp-bench Node.js Engine v${VERSION}`); + return 0; + } + if (options.info) { + printInfo(); + return 0; + } + + try { + const missing = (['driver', 'workload', 'metrics'] as const).filter((flag) => !options[flag]); + if (missing.length > 0) { + throw new Error(`missing required options: ${missing.map((f) => `--${f}`).join(', ')}`); + } + for (const flag of ['driver', 'workload'] as const) { + if (!existsSync(options[flag]!)) { + throw new Error(`${flag} config not found: ${options[flag]}`); + } + } + + const { host, port } = parseServer(options.server!); + const engine = new BenchmarkEngine({ + host, + port, + driverConfig: ConfigLoader.loadDriverConfig(options.driver!), + workloadConfig: ConfigLoader.loadWorkloadConfig(options.workload!), + metricsPath: options.metrics!, + commitId: options['commit-id'] ?? null, + }); + await engine.run(); + return 0; + } catch (error) { + console.error(`Error: ${(error as Error).message}`); + if (process.env['DEBUG']) console.error((error as Error).stack); + return 1; + } +} + +// `import.meta.url` check keeps the module importable by tests without running. +if (process.argv[1] && import.meta.url === `file://${process.argv[1]}`) { + process.exitCode = await main(); +} diff --git a/node/src/client/benchmarkClient.ts b/node/src/client/benchmarkClient.ts new file mode 100644 index 0000000..214e655 --- /dev/null +++ b/node/src/client/benchmarkClient.ts @@ -0,0 +1,59 @@ +/** + * The interface every driver implements. + * + * One client instance maps to exactly one transport connection (the + * `client == connection` invariant shared by all engines); the engine never + * shares a client across workers. + * + * `measure()` is the single place latency is captured, so every driver reports it + * identically: `process.hrtime.bigint()` around the awaited command, truncated to + * whole microseconds, recorded even when the command throws. + */ + +import type { DriverConfig } from '../config/driverConfig.js'; +import type { TimedResult } from './timedResult.js'; + +export interface BenchmarkClient { + connect(host: string, port: number, config: DriverConfig): Promise; + ping(): Promise>; + get(key: string): Promise>; + set(key: string, value: Buffer): Promise>; + close(): Promise; + driverVersion(): string; + /** Secondary driver version, for composite drivers only. */ + secondaryDriverVersion?(): string | null; + /** + * Mark the client as warming up, mirroring Java's `setWarmupMode` + * (BenchmarkEngine.java:212-237). + * + * Real drivers ignore this. The recording driver uses it to suppress simulated + * errors so that an `error_rate` workload does not abort in warmup โ€” the + * warmup fail-fast is meant to catch an unreachable server, not injected + * errors the phase is deliberately measuring. + */ + setWarmupMode?(warmup: boolean): void; +} + +const NANOS_PER_MICRO = 1000n; + +/** + * Await `operation` and record its latency in microseconds. + * + * Errors are captured, not thrown: the engine records a failed request and keeps + * going, matching the other engines. Latency is measured on the error path too. + */ +export async function measure(operation: () => Promise): Promise> { + const start = process.hrtime.bigint(); + try { + const value = await operation(); + const latencyMicros = Number((process.hrtime.bigint() - start) / NANOS_PER_MICRO); + return { value, latencyMicros }; + } catch (error) { + const latencyMicros = Number((process.hrtime.bigint() - start) / NANOS_PER_MICRO); + return { + value: null, + latencyMicros, + error: error instanceof Error ? error : new Error(String(error)), + }; + } +} diff --git a/node/src/client/driverVersion.ts b/node/src/client/driverVersion.ts new file mode 100644 index 0000000..bf7c933 --- /dev/null +++ b/node/src/client/driverVersion.ts @@ -0,0 +1,45 @@ +/** + * Reads an installed package's version for the NDJSON `metadata` block. + * + * `require('/package.json')` is NOT usable here: `@valkey/valkey-glide` + * declares an `exports` map with no `./package.json` entry, so the subpath import + * fails with ERR_PACKAGE_PATH_NOT_EXPORTED. Resolving the module's entry point + * and walking up to the nearest package.json with `fs` sidesteps `exports` + * entirely (it only governs specifier resolution, not file reads) and works for + * every driver. + */ + +import { readFileSync } from 'node:fs'; +import { createRequire } from 'node:module'; +import { dirname, join } from 'node:path'; + +const require = createRequire(import.meta.url); + +/** Depth to walk up from the resolved entry point looking for package.json. */ +const MAX_WALK_UP = 6; + +export function packageVersion(name: string): string { + let dir: string; + try { + dir = dirname(require.resolve(name)); + } catch { + return 'unknown'; + } + + for (let i = 0; i < MAX_WALK_UP; i++) { + try { + const manifest = JSON.parse(readFileSync(join(dir, 'package.json'), 'utf8')) as { + name?: string; + version?: string; + }; + // Guard against picking up a nested manifest of a different package. + if (manifest.name === name && typeof manifest.version === 'string') return manifest.version; + } catch { + // Not here (or unreadable) -- keep walking up. + } + const parent = dirname(dir); + if (parent === dir) break; + dir = parent; + } + return 'unknown'; +} diff --git a/node/src/client/factory.ts b/node/src/client/factory.ts new file mode 100644 index 0000000..a5dac3d --- /dev/null +++ b/node/src/client/factory.ts @@ -0,0 +1,81 @@ +/** + * Driver registry: maps `driver_id` to a client implementation. + * + * Implementations are loaded with a dynamic `import()` so `--info` and the unit + * tests work even if a driver's native bits are missing or broken on this + * platform -- only the driver actually requested gets loaded. + * + * The ids matter beyond this file. `DRIVER_ENGINE_MAP` in + * scripts/run_benchmark_matrix.py is a single global map shared by every engine, + * and `valkey-glide` there already means *Java*. Hence `valkey-glide-node`: a + * bare `valkey-glide` here would silently reroute Java's glide runs to Node. + */ + +import type { DriverConfig } from '../config/driverConfig.js'; +import type { BenchmarkClient } from './benchmarkClient.js'; + +interface DriverEntry { + description: string; + load: () => Promise; +} + +const DRIVERS = new Map([ + [ + 'valkey-glide-node', + { + description: 'Valkey GLIDE for Node.js (@valkey/valkey-glide)', + load: async () => new (await import('./impl/glideClient.js')).GlideBenchmarkClient(), + }, + ], + [ + 'ioredis', + { + description: 'ioredis โ€” the most widely used Node.js Redis client', + load: async () => new (await import('./impl/ioredisClient.js')).IoredisBenchmarkClient(), + }, + ], + [ + 'iovalkey', + { + description: 'iovalkey โ€” the Valkey-maintained fork of ioredis', + load: async () => new (await import('./impl/iovalkeyClient.js')).IovalkeyBenchmarkClient(), + }, + ], + [ + 'recording', + { + description: 'In-memory synthetic-latency client (no server required)', + load: async () => new (await import('./impl/recordingClient.js')).RecordingClient(), + }, + ], +]); + +export class BenchmarkClientFactory { + static supportedDrivers(): string[] { + return [...DRIVERS.keys()]; + } + + static describe(): Array<{ driverId: string; description: string }> { + return [...DRIVERS.entries()].map(([driverId, { description }]) => ({ driverId, description })); + } + + static async create(driverId: string): Promise { + const entry = DRIVERS.get((driverId ?? '').toLowerCase()); + if (entry === undefined) { + throw new Error( + `Unknown driver: ${driverId}. Supported: ${BenchmarkClientFactory.supportedDrivers().join(', ')}`, + ); + } + return entry.load(); + } + + static async createAndConnect( + host: string, + port: number, + config: DriverConfig, + ): Promise { + const client = await BenchmarkClientFactory.create(config.driverId); + await client.connect(host, port, config); + return client; + } +} diff --git a/node/src/client/impl/glideClient.ts b/node/src/client/impl/glideClient.ts new file mode 100644 index 0000000..74e3d2d --- /dev/null +++ b/node/src/client/impl/glideClient.ts @@ -0,0 +1,88 @@ +/** + * valkey-glide driver (`@valkey/valkey-glide`). + * + * One GlideClient per connection, honouring the `client == connection` invariant + * shared across engines. + * + * Two glide-specific shapes to note: + * - `close()` is **synchronous** (returns void, not a Promise), unlike every + * other driver here. + * - `get()` returns a `string` by default. That is deliberate and matches the + * Python engine and the ioredis/iovalkey clients here, so all drivers are + * charged for the same UTF-8 decode. Do not switch one driver to bytes. + */ + +import type { + GlideClient, + GlideClientConfiguration, + GlideClusterClient, + GlideClusterClientConfiguration, + ServerCredentials, +} from '@valkey/valkey-glide'; + +import type { DriverConfig } from '../../config/driverConfig.js'; +import { measure, type BenchmarkClient } from '../benchmarkClient.js'; +import { packageVersion } from '../driverVersion.js'; +import type { TimedResult } from '../timedResult.js'; + +const PACKAGE = '@valkey/valkey-glide'; + +export class GlideBenchmarkClient implements BenchmarkClient { + private client: GlideClient | GlideClusterClient | null = null; + + async connect(host: string, port: number, config: DriverConfig): Promise { + const glide = await import('@valkey/valkey-glide'); + + const addresses = [{ host, port }]; + const credentials: ServerCredentials | undefined = config.hasAuth() + ? ({ + password: config.auth?.password ?? '', + ...(config.auth?.username ? { username: config.auth.username } : {}), + } as ServerCredentials) + : undefined; + + const shared = { + addresses, + useTLS: config.tlsEnabled(), + ...(credentials ? { credentials } : {}), + ...(config.commandTimeoutMs ? { requestTimeout: config.commandTimeoutMs } : {}), + }; + + this.client = config.isCluster() + ? await glide.GlideClusterClient.createClient(shared as GlideClusterClientConfiguration) + : await glide.GlideClient.createClient(shared as GlideClientConfiguration); + } + + private requireClient(): GlideClient | GlideClusterClient { + if (this.client === null) throw new Error('glide client is not connected'); + return this.client; + } + + async ping(): Promise> { + const client = this.requireClient(); + return measure(async () => String(await client.ping())); + } + + async get(key: string): Promise> { + const client = this.requireClient(); + return measure(async () => { + const value = await client.get(key); + return value === null ? null : String(value); + }) as Promise>; + } + + async set(key: string, value: Buffer): Promise> { + const client = this.requireClient(); + return measure(async () => String(await client.set(key, value))); + } + + async close(): Promise { + // Synchronous in glide -- there is nothing to await. + this.client?.close(); + this.client = null; + } + + driverVersion(): string { + return packageVersion(PACKAGE); + } +} diff --git a/node/src/client/impl/ioredisClient.ts b/node/src/client/impl/ioredisClient.ts new file mode 100644 index 0000000..85b609b --- /dev/null +++ b/node/src/client/impl/ioredisClient.ts @@ -0,0 +1,16 @@ +/** ioredis driver โ€” the most widely used Node.js Redis client. */ + +import { IoredisFamilyClient, type RedisModuleLike } from './ioredisFamilyClient.js'; + +export class IoredisBenchmarkClient extends IoredisFamilyClient { + protected override packageName(): string { + return 'ioredis'; + } + + protected override async loadModule(): Promise { + const module = await import('ioredis'); + // ioredis' constructors carry overloads that RedisModuleLike narrows to the + // one form this engine calls; the shapes are compatible at runtime. + return { Redis: module.Redis, Cluster: module.Cluster } as unknown as RedisModuleLike; + } +} diff --git a/node/src/client/impl/ioredisFamilyClient.ts b/node/src/client/impl/ioredisFamilyClient.ts new file mode 100644 index 0000000..2959874 --- /dev/null +++ b/node/src/client/impl/ioredisFamilyClient.ts @@ -0,0 +1,131 @@ +/** + * Shared implementation for the ioredis-family drivers. + * + * `ioredis` and `iovalkey` (the Valkey-maintained fork) expose the same + * constructor options and the same `Redis`/`Cluster` exports, so both drivers + * differ only in which module they load and which package they report a version + * for. They are described structurally here rather than against either package's + * types, so neither becomes a compile-time dependency of the other's driver. + * + * Fairness note: `enableAutoPipelining` is forced **off**. Left on (it is off by + * default, but that default has changed before) ioredis transparently batches + * commands issued in the same event-loop tick, which would inflate throughput + * against every other engine and silently make the comparison meaningless. + */ + +import { readFileSync } from 'node:fs'; + +import type { DriverConfig } from '../../config/driverConfig.js'; +import { measure, type BenchmarkClient } from '../benchmarkClient.js'; +import { packageVersion } from '../driverVersion.js'; +import type { TimedResult } from '../timedResult.js'; + +/** The slice of the ioredis surface this engine uses. */ +interface RedisLike { + connect(): Promise; + ping(): Promise; + get(key: string): Promise; + set(key: string, value: Buffer): Promise; + quit(): Promise; + disconnect(): void; +} + +export interface RedisModuleLike { + Redis: new (options: Record) => RedisLike; + Cluster: new ( + nodes: Array<{ host: string; port: number }>, + options: Record, + ) => RedisLike; +} + +function buildTlsOptions(config: DriverConfig): Record | undefined { + if (!config.tlsEnabled()) return undefined; + const tls: Record = {}; + if (config.tls?.ca_path) tls['ca'] = readFileSync(config.tls.ca_path); + if (config.tls?.cert_path) tls['cert'] = readFileSync(config.tls.cert_path); + if (config.tls?.key_path) tls['key'] = readFileSync(config.tls.key_path); + if (config.tls?.verify_hostname === false) tls['rejectUnauthorized'] = false; + return tls; +} + +export abstract class IoredisFamilyClient implements BenchmarkClient { + private client: RedisLike | null = null; + + /** npm package name, used for both loading and version reporting. */ + protected abstract packageName(): string; + + protected abstract loadModule(): Promise; + + async connect(host: string, port: number, config: DriverConfig): Promise { + const module = await this.loadModule(); + const tls = buildTlsOptions(config); + + const options: Record = { + // Batching commands issued in one tick would not be comparable to the + // other engines -- keep every request a distinct round trip. + enableAutoPipelining: false, + // Connect explicitly below so a connection failure surfaces here rather + // than as a first-command error, and so cps_limit really gates setup. + lazyConnect: true, + // Fail a stuck request instead of retrying it under a different latency. + maxRetriesPerRequest: 0, + // Never silently reconnect. ioredis' default retryStrategy retries + // forever, so a wrong host would hang the run instead of failing it, and a + // mid-phase reconnect would fold connection setup into request latency. + retryStrategy: () => null, + // Bound the initial connect so an unreachable host fails fast. + connectTimeout: config.commandTimeoutMs ?? 10_000, + ...(config.hasAuth() && config.auth?.username ? { username: config.auth.username } : {}), + ...(config.hasAuth() && config.auth?.password ? { password: config.auth.password } : {}), + ...(config.commandTimeoutMs ? { commandTimeout: config.commandTimeoutMs } : {}), + ...(tls ? { tls } : {}), + }; + + if (config.isCluster()) { + this.client = new module.Cluster([{ host, port }], { + lazyConnect: true, + redisOptions: options, + }); + } else { + this.client = new module.Redis({ ...options, host, port }); + } + await this.client.connect(); + } + + private requireClient(): RedisLike { + if (this.client === null) throw new Error(`${this.packageName()} client is not connected`); + return this.client; + } + + async ping(): Promise> { + const client = this.requireClient(); + return measure(() => client.ping()); + } + + async get(key: string): Promise> { + const client = this.requireClient(); + return measure(() => client.get(key)) as Promise>; + } + + async set(key: string, value: Buffer): Promise> { + const client = this.requireClient(); + return measure(() => client.set(key, value)) as Promise>; + } + + async close(): Promise { + if (this.client === null) return; + const client = this.client; + this.client = null; + try { + await client.quit(); + } catch { + // A server that already dropped the connection makes QUIT reject; the + // socket still has to go, or the process will not exit. + client.disconnect(); + } + } + + driverVersion(): string { + return packageVersion(this.packageName()); + } +} diff --git a/node/src/client/impl/iovalkeyClient.ts b/node/src/client/impl/iovalkeyClient.ts new file mode 100644 index 0000000..0340d77 --- /dev/null +++ b/node/src/client/impl/iovalkeyClient.ts @@ -0,0 +1,19 @@ +/** + * iovalkey driver โ€” the Valkey-maintained fork of ioredis. + * + * API-identical to ioredis, so all behaviour lives in the shared base class. + */ + +import { IoredisFamilyClient, type RedisModuleLike } from './ioredisFamilyClient.js'; + +export class IovalkeyBenchmarkClient extends IoredisFamilyClient { + protected override packageName(): string { + return 'iovalkey'; + } + + protected override async loadModule(): Promise { + const module = await import('iovalkey'); + // Same overload-narrowing as the ioredis driver; iovalkey mirrors its API. + return { Redis: module.Redis, Cluster: module.Cluster } as unknown as RedisModuleLike; + } +} diff --git a/node/src/client/impl/recordingClient.ts b/node/src/client/impl/recordingClient.ts new file mode 100644 index 0000000..2b60618 --- /dev/null +++ b/node/src/client/impl/recordingClient.ts @@ -0,0 +1,131 @@ +/** + * In-memory recording client for server-free testing. + * + * Records operations and supports simulated latency and error injection via + * `specific_driver_config` (`operation_delay_micros`, `delay_variation_micros`, + * `error_rate`, `error_message`). This lets the integration tests exercise the + * whole engine without a live server, mirroring the Ruby/Python recording + * drivers. + */ + +import type { DriverConfig } from '../../config/driverConfig.js'; +import type { BenchmarkClient } from '../benchmarkClient.js'; +import type { TimedResult } from '../timedResult.js'; + +export interface RecordedOperation { + command: string; + key: string | null; + value: Buffer | null; + success: boolean; + errorMessage: string | null; +} + +const NANOS_PER_MICRO = 1000n; + +function readNumber(source: Record, key: string, fallback: number): number { + const value = source[key]; + return typeof value === 'number' && Number.isFinite(value) ? value : fallback; +} + +/** Sleep for whole microseconds, tolerating setTimeout's ~1ms floor. */ +function sleepMicros(micros: number): Promise { + return new Promise((resolve) => setTimeout(resolve, micros / 1000)); +} + +export class RecordingClient implements BenchmarkClient { + readonly operations: RecordedOperation[] = []; + private readonly storedData = new Map(); + private operationDelayMicros = 0; + private delayVariationMicros = 0; + private errorRate = 0; + private errorMessage = 'Simulated error'; + private warmupMode = false; + + async connect(_host: string, _port: number, config: DriverConfig): Promise { + const cfg = config.specificDriverConfig; + this.operationDelayMicros = readNumber(cfg, 'operation_delay_micros', 0); + this.delayVariationMicros = readNumber(cfg, 'delay_variation_micros', 0); + this.errorRate = readNumber(cfg, 'error_rate', 0); + const message = cfg['error_message']; + if (typeof message === 'string') this.errorMessage = message; + this.record('CONNECT', null, null, true, null); + } + + async ping(): Promise> { + const { latencyMicros, success } = await this.simulate(); + this.record('PING', null, null, success, success ? null : this.errorMessage); + return success + ? { value: 'PONG', latencyMicros } + : { value: null, latencyMicros, error: new Error(this.errorMessage) }; + } + + async get(key: string): Promise> { + const { latencyMicros, success } = await this.simulate(); + this.record('GET', key, null, success, success ? null : this.errorMessage); + if (!success) return { value: null, latencyMicros, error: new Error(this.errorMessage) }; + const stored = this.storedData.get(key); + return { value: stored === undefined ? null : stored.toString('latin1'), latencyMicros }; + } + + async set(key: string, value: Buffer): Promise> { + const { latencyMicros, success } = await this.simulate(); + if (success) this.storedData.set(key, value); + this.record('SET', key, value, success, success ? null : this.errorMessage); + return success + ? { value: 'OK', latencyMicros } + : { value: null, latencyMicros, error: new Error(this.errorMessage) }; + } + + async close(): Promise { + this.record('CLOSE', null, null, true, null); + } + + driverVersion(): string { + return '1.0.0'; + } + + /** + * Suppress simulated errors during warmup, matching Java's recording client. + * The engine's warmup fail-fast exists to catch an unreachable server, so + * injected errors must not trip it. + */ + setWarmupMode(warmup: boolean): void { + this.warmupMode = warmup; + } + + private record( + command: string, + key: string | null, + value: Buffer | null, + success: boolean, + errorMessage: string | null, + ): void { + this.operations.push({ command, key, value, success, errorMessage }); + } + + private async simulate(): Promise<{ latencyMicros: number; success: boolean }> { + const start = process.hrtime.bigint(); + const delayMicros = this.calculateDelayMicros(); + if (delayMicros > 0) await sleepMicros(delayMicros); + const latencyMicros = Number((process.hrtime.bigint() - start) / NANOS_PER_MICRO); + return { latencyMicros, success: !this.shouldSimulateError() }; + } + + private calculateDelayMicros(): number { + if (this.operationDelayMicros <= 0) return 0; + let delay = this.operationDelayMicros; + if (this.delayVariationMicros > 0) { + const spread = 2 * this.delayVariationMicros + 1; + const variation = Math.floor(Math.random() * spread) - this.delayVariationMicros; + delay = Math.max(0, delay + variation); + } + return delay; + } + + private shouldSimulateError(): boolean { + if (this.warmupMode) return false; + if (this.errorRate <= 0) return false; + if (this.errorRate >= 1) return true; + return Math.random() < this.errorRate; + } +} diff --git a/node/src/client/timedResult.ts b/node/src/client/timedResult.ts new file mode 100644 index 0000000..3d36c5a --- /dev/null +++ b/node/src/client/timedResult.ts @@ -0,0 +1,12 @@ +/** + * The outcome of a single measured command. + * + * `error === undefined` means success. Latency is populated on both paths so a + * failure still contributes a measured duration, matching the other engines. + */ +export interface TimedResult { + readonly value: T | null; + /** Command latency in whole microseconds. Recorded even on error. */ + readonly latencyMicros: number; + readonly error?: Error; +} diff --git a/node/src/command/command.ts b/node/src/command/command.ts new file mode 100644 index 0000000..8c74699 --- /dev/null +++ b/node/src/command/command.ts @@ -0,0 +1,19 @@ +/** A benchmark command: a weighted operation the engine can issue. */ + +import type { BenchmarkClient } from '../client/benchmarkClient.js'; + +export interface CommandResult { + readonly commandName: string; + readonly latencyMicros: number; + readonly success: boolean; + readonly errorMessage?: string; +} + +export interface Command { + /** Upper-case command name, used as the NDJSON metrics key (GET/SET/PING). */ + readonly name: string; + readonly weight: number; + /** Whether this command consumes a generated key (PING does not). */ + readonly usesKey: boolean; + execute(client: BenchmarkClient, key: string): Promise; +} diff --git a/node/src/command/factory.ts b/node/src/command/factory.ts new file mode 100644 index 0000000..635a85a --- /dev/null +++ b/node/src/command/factory.ts @@ -0,0 +1,39 @@ +/** Maps a workload's `command` strings to Command implementations. */ + +import type { CommandConfig } from '../config/commandConfig.js'; +import type { Command } from './command.js'; +import { GetCommand } from './impl/getCommand.js'; +import { PingCommand } from './impl/pingCommand.js'; +import { SetCommand } from './impl/setCommand.js'; + +type CommandBuilder = (config: CommandConfig) => Command; + +const BUILDERS = new Map([ + ['get', { build: (c) => new GetCommand(c), description: 'GET key' }], + ['set', { build: (c) => new SetCommand(c), description: 'SET key value' }], + ['ping', { build: (c) => new PingCommand(c), description: 'PING' }], +]); + +export class CommandFactory { + static supportedCommands(): string[] { + return [...BUILDERS.keys()]; + } + + static describe(): Array<{ name: string; description: string }> { + return [...BUILDERS.entries()].map(([name, { description }]) => ({ name, description })); + } + + static create(config: CommandConfig): Command { + const entry = BUILDERS.get(config.command.toLowerCase()); + if (entry === undefined) { + throw new Error( + `Unknown command: ${config.command}. Supported: ${CommandFactory.supportedCommands().join(', ')}`, + ); + } + return entry.build(config); + } + + static createAll(configs: CommandConfig[]): Command[] { + return configs.map((config) => CommandFactory.create(config)); + } +} diff --git a/node/src/command/impl/getCommand.ts b/node/src/command/impl/getCommand.ts new file mode 100644 index 0000000..723e512 --- /dev/null +++ b/node/src/command/impl/getCommand.ts @@ -0,0 +1,23 @@ +import type { BenchmarkClient } from '../../client/benchmarkClient.js'; +import type { CommandConfig } from '../../config/commandConfig.js'; +import type { Command, CommandResult } from '../command.js'; + +export class GetCommand implements Command { + readonly name = 'GET'; + readonly weight: number; + readonly usesKey = true; + + constructor(config: CommandConfig) { + this.weight = config.weight; + } + + async execute(client: BenchmarkClient, key: string): Promise { + const result = await client.get(key); + return { + commandName: this.name, + latencyMicros: result.latencyMicros, + success: result.error === undefined, + ...(result.error ? { errorMessage: result.error.message } : {}), + }; + } +} diff --git a/node/src/command/impl/pingCommand.ts b/node/src/command/impl/pingCommand.ts new file mode 100644 index 0000000..86ac70f --- /dev/null +++ b/node/src/command/impl/pingCommand.ts @@ -0,0 +1,23 @@ +import type { BenchmarkClient } from '../../client/benchmarkClient.js'; +import type { CommandConfig } from '../../config/commandConfig.js'; +import type { Command, CommandResult } from '../command.js'; + +export class PingCommand implements Command { + readonly name = 'PING'; + readonly weight: number; + readonly usesKey = false; + + constructor(config: CommandConfig) { + this.weight = config.weight; + } + + async execute(client: BenchmarkClient): Promise { + const result = await client.ping(); + return { + commandName: this.name, + latencyMicros: result.latencyMicros, + success: result.error === undefined, + ...(result.error ? { errorMessage: result.error.message } : {}), + }; + } +} diff --git a/node/src/command/impl/setCommand.ts b/node/src/command/impl/setCommand.ts new file mode 100644 index 0000000..eae42ff --- /dev/null +++ b/node/src/command/impl/setCommand.ts @@ -0,0 +1,41 @@ +import type { BenchmarkClient } from '../../client/benchmarkClient.js'; +import type { CommandConfig } from '../../config/commandConfig.js'; +import type { Command, CommandResult } from '../command.js'; + +/** + * Deterministic filler pattern, matching the Ruby and Python engines. Java uses + * random bytes instead, but only the payload *length* affects RESP framing and + * server work, and a fixed pattern makes runs reproducible. + */ +const PATTERN = '0123456789ABCDEF'; + +export class SetCommand implements Command { + readonly name = 'SET'; + readonly weight: number; + readonly usesKey = true; + /** + * Built once at construction, not per request: allocating a fresh payload in + * the hot loop would charge the driver for the engine's own GC churn. + */ + private readonly value: Buffer; + + constructor(config: CommandConfig) { + this.weight = config.weight; + this.value = SetCommand.generateValue(config.dataSizeBytes); + } + + async execute(client: BenchmarkClient, key: string): Promise { + const result = await client.set(key, this.value); + return { + commandName: this.name, + latencyMicros: result.latencyMicros, + success: result.error === undefined, + ...(result.error ? { errorMessage: result.error.message } : {}), + }; + } + + static generateValue(size: number): Buffer { + const repeats = Math.floor(size / PATTERN.length) + 1; + return Buffer.from(PATTERN.repeat(repeats).slice(0, size), 'latin1'); + } +} diff --git a/node/src/config/commandConfig.ts b/node/src/config/commandConfig.ts new file mode 100644 index 0000000..eb85f02 --- /dev/null +++ b/node/src/config/commandConfig.ts @@ -0,0 +1,21 @@ +/** A single weighted command entry within a phase. */ + +/** + * Default SET payload size. 256 is the cross-engine default: Java + * (SetCommand.java `getDataSizeBytesOrDefault(256)`), C# (SetCommand.cs), + * Ruby and Python all use it. Do not change without changing them too. + */ +export const DEFAULT_DATA_SIZE_BYTES = 256; + +export class CommandConfig { + readonly command: string; + readonly weight: number; + readonly dataSizeBytes: number; + + constructor(init: { command: string; weight?: number | null; dataSizeBytes?: number | null }) { + this.command = init.command.toLowerCase(); + // A missing weight defaults to 1.0 (matching Java), rather than NaN. + this.weight = init.weight ?? 1.0; + this.dataSizeBytes = init.dataSizeBytes ?? DEFAULT_DATA_SIZE_BYTES; + } +} diff --git a/node/src/config/completionConfig.ts b/node/src/config/completionConfig.ts new file mode 100644 index 0000000..1dd8770 --- /dev/null +++ b/node/src/config/completionConfig.ts @@ -0,0 +1,29 @@ +/** Phase completion criteria: run for a duration, or until a request count. */ + +export class CompletionConfig { + readonly type: string; + readonly seconds: number | null; + readonly requests: number | null; + + constructor(init: { type: string; seconds?: number | null; requests?: number | null }) { + this.type = init.type; + this.seconds = init.seconds ?? null; + this.requests = init.requests ?? null; + } + + isDurationBased(): boolean { + return this.type === 'duration'; + } + + isRequestBased(): boolean { + return this.type === 'requests'; + } + + durationSeconds(): number { + return this.seconds ?? 0; + } + + totalRequests(): number { + return this.requests ?? 0; + } +} diff --git a/node/src/config/driverConfig.ts b/node/src/config/driverConfig.ts new file mode 100644 index 0000000..c432107 --- /dev/null +++ b/node/src/config/driverConfig.ts @@ -0,0 +1,80 @@ +/** + * Driver (client library) configuration. + * + * Maps to configs/schemas/driver-config.schema.json. Field names and defaults + * mirror the Java/Ruby/Python engines so the same JSON files work unchanged + * across every engine. + */ + +export interface TlsConfig { + enabled?: boolean; + cert_path?: string; + key_path?: string; + ca_path?: string; + verify_hostname?: boolean; +} + +export interface AuthConfig { + username?: string; + password?: string; +} + +export const DEFAULT_MODE = 'standalone'; + +export class DriverConfig { + readonly schemaVersion: string; + readonly description: string | null; + readonly driverId: string; + readonly mode: string; + readonly commandTimeoutMs: number | null; + readonly tls: TlsConfig | null; + readonly auth: AuthConfig | null; + readonly specificDriverConfig: Record; + + constructor(init: { + schemaVersion?: string; + description?: string | null; + driverId: string; + mode?: string | null; + commandTimeoutMs?: number | null; + tls?: TlsConfig | null; + auth?: AuthConfig | null; + specificDriverConfig?: Record | null; + }) { + this.schemaVersion = init.schemaVersion ?? '1.0'; + this.description = init.description ?? null; + this.driverId = init.driverId; + this.mode = init.mode ?? DEFAULT_MODE; + this.commandTimeoutMs = init.commandTimeoutMs ?? null; + this.tls = init.tls ?? null; + this.auth = init.auth ?? null; + this.specificDriverConfig = init.specificDriverConfig ?? {}; + } + + /** Secondary driver id for composite drivers (e.g. Java's spring-data-*). */ + secondaryDriverId(): string | null { + const value = this.specificDriverConfig['secondary_driver_id']; + return typeof value === 'string' ? value : null; + } + + isStandalone(): boolean { + return this.mode === 'standalone'; + } + + isCluster(): boolean { + return this.mode === 'cluster'; + } + + isSentinel(): boolean { + return this.mode === 'sentinel'; + } + + tlsEnabled(): boolean { + return this.tls?.enabled === true; + } + + /** Username/password only count as auth when at least one is non-empty. */ + hasAuth(): boolean { + return Boolean(this.auth && (this.auth.password || this.auth.username)); + } +} diff --git a/node/src/config/keyspaceConfig.ts b/node/src/config/keyspaceConfig.ts new file mode 100644 index 0000000..b20b4f9 --- /dev/null +++ b/node/src/config/keyspaceConfig.ts @@ -0,0 +1,41 @@ +/** Key-generation configuration for a benchmark phase. */ + +export const DEFAULT_KEY_SIZE_BYTES = 16; +export const DEFAULT_KEY_PREFIX = 'bench:'; +export const DEFAULT_GENERATION_ALG = 'sequential_int'; + +export class KeyspaceConfig { + readonly keysCount: number; + readonly keySizeBytes: number; + readonly keyPrefix: string; + readonly generationAlg: string; + readonly seed: number | null; + + constructor(init: { + keysCount: number; + keySizeBytes?: number | null; + keyPrefix?: string | null; + generationAlg?: string | null; + seed?: number | null; + }) { + this.keysCount = init.keysCount; + // A null/undefined value falls back to the default rather than staying + // null, matching the Ruby/Python engines. + this.keySizeBytes = init.keySizeBytes ?? DEFAULT_KEY_SIZE_BYTES; + this.keyPrefix = init.keyPrefix ?? DEFAULT_KEY_PREFIX; + this.generationAlg = init.generationAlg ?? DEFAULT_GENERATION_ALG; + this.seed = init.seed ?? null; + } + + isSequentialInt(): boolean { + return this.generationAlg === 'sequential_int'; + } + + isUniformRand(): boolean { + return this.generationAlg === 'uniform_rand'; + } + + seedValue(): number { + return this.seed ?? 0; + } +} diff --git a/node/src/config/loader.ts b/node/src/config/loader.ts new file mode 100644 index 0000000..000a914 --- /dev/null +++ b/node/src/config/loader.ts @@ -0,0 +1,198 @@ +/** + * Loads driver and workload configuration from the shared configs/ JSON files. + * + * Field names and defaults mirror the Java/Ruby/Python engines exactly, so the + * same JSON is consumed identically by every engine. Unlike those engines this + * one also validates the required fields up front: a typo'd config should fail + * with a clear message rather than surface later as a null-shaped error deep in + * a worker loop. + */ + +import { readFileSync } from 'node:fs'; + +import { CommandConfig } from './commandConfig.js'; +import { CompletionConfig } from './completionConfig.js'; +import { DriverConfig } from './driverConfig.js'; +import { KeyspaceConfig } from './keyspaceConfig.js'; +import { PhaseConfig } from './phaseConfig.js'; +import { WorkloadConfig } from './workloadConfig.js'; + +type Json = Record; + +export class ConfigError extends Error {} + +function asRecord(value: unknown, what: string): Json { + if (value === null || typeof value !== 'object' || Array.isArray(value)) { + throw new ConfigError(`${what} must be a JSON object`); + } + return value as Json; +} + +function optString(data: Json, key: string): string | null { + const value = data[key]; + if (value === undefined || value === null) return null; + if (typeof value !== 'string') throw new ConfigError(`"${key}" must be a string`); + return value; +} + +function reqString(data: Json, key: string, what: string): string { + const value = optString(data, key); + if (value === null || value === '') throw new ConfigError(`${what} is missing required "${key}"`); + return value; +} + +function optNumber(data: Json, key: string): number | null { + const value = data[key]; + if (value === undefined || value === null) return null; + if (typeof value !== 'number' || !Number.isFinite(value)) { + throw new ConfigError(`"${key}" must be a finite number`); + } + return value; +} + +function reqInt(data: Json, key: string, what: string): number { + const value = optNumber(data, key); + if (value === null) throw new ConfigError(`${what} is missing required "${key}"`); + if (!Number.isInteger(value)) throw new ConfigError(`"${key}" must be an integer`); + return value; +} + +function readJson(path: string, what: string): Json { + let text: string; + try { + text = readFileSync(path, 'utf8'); + } catch (error) { + throw new ConfigError(`cannot read ${what} "${path}": ${(error as Error).message}`); + } + try { + return asRecord(JSON.parse(text), what); + } catch (error) { + if (error instanceof ConfigError) throw error; + throw new ConfigError(`${what} "${path}" is not valid JSON: ${(error as Error).message}`); + } +} + +export class ConfigLoader { + static loadDriverConfig(path: string): DriverConfig { + return ConfigLoader.parseDriverConfig(readJson(path, 'driver config')); + } + + static loadWorkloadConfig(path: string): WorkloadConfig { + return ConfigLoader.parseWorkloadConfig(readJson(path, 'workload config')); + } + + static parseDriverConfig(data: Json): DriverConfig { + const mode = optString(data, 'mode') ?? 'standalone'; + if (!['standalone', 'cluster', 'sentinel'].includes(mode)) { + throw new ConfigError( + `driver config "mode" must be standalone, cluster or sentinel (got "${mode}")`, + ); + } + return new DriverConfig({ + schemaVersion: optString(data, 'schema_version') ?? '1.0', + description: optString(data, 'description'), + driverId: reqString(data, 'driver_id', 'driver config'), + mode, + commandTimeoutMs: optNumber(data, 'command_timeout_ms'), + tls: (data['tls'] ?? null) as DriverConfig['tls'], + auth: (data['auth'] ?? null) as DriverConfig['auth'], + specificDriverConfig: (data['specific_driver_config'] ?? {}) as Record, + }); + } + + static parseWorkloadConfig(data: Json): WorkloadConfig { + const rawPhases = data['phases']; + if (!Array.isArray(rawPhases) || rawPhases.length === 0) { + throw new ConfigError('workload config must have a non-empty "phases" array'); + } + return new WorkloadConfig({ + schemaVersion: optString(data, 'schema_version') ?? '1.0', + benchmarkProfile: (data['benchmark_profile'] ?? {}) as WorkloadConfig['benchmarkProfile'], + phases: rawPhases.map((phase, index) => + ConfigLoader.parsePhase(asRecord(phase, `phases[${index}]`), index), + ), + }); + } + + static parsePhase(data: Json, index = 0): PhaseConfig { + const what = `phases[${index}]`; + const rawCommands = data['commands']; + if (!Array.isArray(rawCommands) || rawCommands.length === 0) { + throw new ConfigError(`${what} must have a non-empty "commands" array`); + } + const connections = reqInt(data, 'connections', what); + if (connections <= 0) throw new ConfigError(`${what} "connections" must be positive`); + + return new PhaseConfig({ + id: reqString(data, 'id', what), + description: optString(data, 'description'), + connections, + cpsLimit: optNumber(data, 'cps_limit'), + rpsLimit: optNumber(data, 'rps_limit'), + pipelineDepth: optNumber(data, 'pipeline_depth'), + warmupRequests: optNumber(data, 'warmup_requests'), + completion: ConfigLoader.parseCompletion( + asRecord(data['completion'] ?? {}, `${what}.completion`), + what, + ), + keyspace: ConfigLoader.parseKeyspace( + asRecord(data['keyspace'] ?? {}, `${what}.keyspace`), + what, + ), + commands: rawCommands.map((command, i) => + ConfigLoader.parseCommand(asRecord(command, `${what}.commands[${i}]`), `${what}.commands[${i}]`), + ), + }); + } + + static parseCompletion(data: Json, what = 'completion'): CompletionConfig { + const type = reqString(data, 'type', `${what}.completion`); + if (!['duration', 'requests'].includes(type)) { + throw new ConfigError( + `${what}.completion "type" must be duration or requests (got "${type}")`, + ); + } + const seconds = optNumber(data, 'seconds'); + const requests = optNumber(data, 'requests'); + if (type === 'duration' && (seconds === null || seconds <= 0)) { + throw new ConfigError(`${what}.completion type=duration requires a positive "seconds"`); + } + if (type === 'requests' && (requests === null || requests <= 0)) { + throw new ConfigError(`${what}.completion type=requests requires a positive "requests"`); + } + return new CompletionConfig({ type, seconds, requests }); + } + + static parseKeyspace(data: Json, what = 'keyspace'): KeyspaceConfig { + const keysCount = reqInt(data, 'keys_count', `${what}.keyspace`); + if (keysCount <= 0) throw new ConfigError(`${what}.keyspace "keys_count" must be positive`); + const generationAlg = optString(data, 'generation_alg') ?? 'sequential_int'; + if (!['sequential_int', 'uniform_rand'].includes(generationAlg)) { + throw new ConfigError( + `${what}.keyspace "generation_alg" must be sequential_int or uniform_rand ` + + `(got "${generationAlg}")`, + ); + } + return new KeyspaceConfig({ + keysCount, + keySizeBytes: optNumber(data, 'key_size_bytes'), + keyPrefix: optString(data, 'key_prefix'), + generationAlg, + seed: optNumber(data, 'seed'), + }); + } + + static parseCommand(data: Json, what = 'command'): CommandConfig { + const command = reqString(data, 'command', what); + const weight = optNumber(data, 'weight'); + if (weight !== null && (weight < 0 || weight > 1)) { + // Matches Java's CommandConfig.validate(): weights are fractions of 1. + throw new ConfigError(`${what} "weight" must be between 0 and 1 (got ${weight})`); + } + return new CommandConfig({ + command, + weight, + dataSizeBytes: optNumber(data, 'data_size_bytes'), + }); + } +} diff --git a/node/src/config/phaseConfig.ts b/node/src/config/phaseConfig.ts new file mode 100644 index 0000000..c9186f5 --- /dev/null +++ b/node/src/config/phaseConfig.ts @@ -0,0 +1,58 @@ +/** Configuration for a single benchmark phase. */ + +import type { CommandConfig } from './commandConfig.js'; +import type { CompletionConfig } from './completionConfig.js'; +import type { KeyspaceConfig } from './keyspaceConfig.js'; + +export const DEFAULT_PIPELINE_DEPTH = 1; +export const DEFAULT_WARMUP_REQUESTS = 1; + +export class PhaseConfig { + readonly id: string; + readonly description: string | null; + readonly connections: number; + readonly completion: CompletionConfig; + readonly keyspace: KeyspaceConfig; + readonly commands: CommandConfig[]; + readonly cpsLimit: number; + readonly rpsLimit: number; + readonly pipelineDepth: number; + readonly warmupRequests: number; + + constructor(init: { + id: string; + description?: string | null; + connections: number; + completion: CompletionConfig; + keyspace: KeyspaceConfig; + commands: CommandConfig[]; + cpsLimit?: number | null; + rpsLimit?: number | null; + pipelineDepth?: number | null; + warmupRequests?: number | null; + }) { + this.id = init.id; + this.description = init.description ?? null; + this.connections = init.connections; + this.completion = init.completion; + this.keyspace = init.keyspace; + this.commands = init.commands; + // -1 is the configs' "unlimited" sentinel; null/undefined means the same. + this.cpsLimit = init.cpsLimit ?? -1; + this.rpsLimit = init.rpsLimit ?? -1; + this.pipelineDepth = init.pipelineDepth ?? DEFAULT_PIPELINE_DEPTH; + this.warmupRequests = init.warmupRequests ?? DEFAULT_WARMUP_REQUESTS; + } + + hasCpsLimit(): boolean { + return this.cpsLimit > 0; + } + + hasRpsLimit(): boolean { + return this.rpsLimit > 0; + } + + effectivePipelineDepth(): number { + return this.pipelineDepth > 0 ? this.pipelineDepth : DEFAULT_PIPELINE_DEPTH; + } +} diff --git a/node/src/config/workloadConfig.ts b/node/src/config/workloadConfig.ts new file mode 100644 index 0000000..74c235e --- /dev/null +++ b/node/src/config/workloadConfig.ts @@ -0,0 +1,29 @@ +/** A whole workload: a benchmark profile plus an ordered list of phases. */ + +import type { PhaseConfig } from './phaseConfig.js'; + +export interface BenchmarkProfile { + name?: string; + description?: string; + version?: string; +} + +export class WorkloadConfig { + readonly schemaVersion: string; + readonly benchmarkProfile: BenchmarkProfile; + readonly phases: PhaseConfig[]; + + constructor(init: { + schemaVersion?: string; + benchmarkProfile?: BenchmarkProfile | null; + phases: PhaseConfig[]; + }) { + this.schemaVersion = init.schemaVersion ?? '1.0'; + this.benchmarkProfile = init.benchmarkProfile ?? {}; + this.phases = init.phases; + } + + name(): string { + return this.benchmarkProfile.name ?? 'unnamed'; + } +} diff --git a/node/src/engine/benchmark.ts b/node/src/engine/benchmark.ts new file mode 100644 index 0000000..15a9d48 --- /dev/null +++ b/node/src/engine/benchmark.ts @@ -0,0 +1,414 @@ +/** + * Benchmark engine. + * + * Concurrency model: a single event loop with **one client per connection** (the + * `client == connection` invariant every engine holds) and **one worker per + * connection**, all started together via `Promise.all`. Node is single-threaded + * with async I/O, so this is the faithful analogue of Java's + * virtual-thread-per-client design -- an awaited command parks the worker, not + * the loop, so the other connections keep making progress. + * + * Two loops, mirroring Java (BenchmarkEngine.java:353-452): + * - `pipeline_depth <= 1`: issue one command, await it, record, repeat. + * - `pipeline_depth > 1`: keep up to `pipelineDepth` requests in flight per + * connection, awaiting whichever settles first and immediately refilling. + * + * The request budget is **shared across all workers**, claimed one request at a + * time, exactly as Java does with its per-phase `AtomicLong` + * (BenchmarkEngine.java:249, 363-367). It is deliberately not pre-divided per + * worker: with a shared budget a slow connection cannot cap the run -- faster + * workers absorb the slack and the phase ends when the total budget is spent. + * Pre-splitting would bound wall-clock by the slowest connection and change the + * per-connection distribution, which is a real cross-engine comparability gap. + */ + +import type { BenchmarkClient } from '../client/benchmarkClient.js'; +import { BenchmarkClientFactory } from '../client/factory.js'; +import type { Command, CommandResult } from '../command/command.js'; +import { CommandFactory } from '../command/factory.js'; +import type { DriverConfig } from '../config/driverConfig.js'; +import type { PhaseConfig } from '../config/phaseConfig.js'; +import type { WorkloadConfig } from '../config/workloadConfig.js'; +import { MetricsCollector } from '../metrics/collector.js'; +import { NdjsonWriter } from '../metrics/ndjsonWriter.js'; +import { CommandSelector } from './commandSelector.js'; +import { Counter, KeyGenerator } from './keyGenerator.js'; +import { RateLimiter } from './rateLimiter.js'; + +const PROGRESS_LOG_INTERVAL_MS = 10_000; +const CONNECTION_LOG_INTERVAL = 50; + +export interface Logger { + info(message: string): void; + warn(message: string): void; + error(message: string): void; +} + +export const consoleLogger: Logger = { + info: (message) => console.log(`${new Date().toISOString()} INFO ${message}`), + warn: (message) => console.warn(`${new Date().toISOString()} WARN ${message}`), + error: (message) => console.error(`${new Date().toISOString()} ERROR ${message}`), +}; + +/** + * A shared, monotonically-drained request budget for one phase. + * + * `claim()` is atomic without a lock: it reads and writes with no `await` in + * between, so concurrent workers on the single event-loop thread can never + * interleave inside it. + */ +class RequestBudget { + private remaining: number; + + constructor(total: number) { + this.remaining = total; + } + + claim(): boolean { + if (this.remaining <= 0) return false; + this.remaining -= 1; + return true; + } +} + +export interface BenchmarkEngineOptions { + host: string; + port: number; + driverConfig: DriverConfig; + workloadConfig: WorkloadConfig; + metricsPath: string; + commitId?: string | null; + logger?: Logger; +} + +export class BenchmarkEngine { + private readonly host: string; + private readonly port: number; + private readonly driverConfig: DriverConfig; + private readonly workloadConfig: WorkloadConfig; + private readonly writer: NdjsonWriter; + private readonly commitId: string | null; + private readonly log: Logger; + + constructor(options: BenchmarkEngineOptions) { + this.host = options.host; + this.port = options.port; + this.driverConfig = options.driverConfig; + this.workloadConfig = options.workloadConfig; + this.writer = new NdjsonWriter(options.metricsPath); + this.commitId = options.commitId ?? null; + this.log = options.logger ?? consoleLogger; + } + + async run(): Promise { + this.log.info(`Starting benchmark: ${this.workloadConfig.name()}`); + this.log.info(`Driver: ${this.driverConfig.driverId}, Server mode: ${this.driverConfig.mode}`); + this.log.info('Concurrency: event-loop task-per-connection (one client per connection)'); + this.log.info(`Server: ${this.host}:${this.port}`); + + await this.setupMetadata(); + + for (const phase of this.workloadConfig.phases) { + await this.executePhase(phase); + } + + this.log.info('Benchmark completed'); + } + + /** Best-effort: a version lookup failure must not fail the benchmark. */ + private async setupMetadata(): Promise { + try { + const sample = await BenchmarkClientFactory.createAndConnect( + this.host, + this.port, + this.driverConfig, + ); + const version = sample.driverVersion(); + this.writer.setMetadata({ + commitId: this.commitId, + driverId: this.driverConfig.driverId, + primaryDriverVersion: version, + secondaryDriverId: this.driverConfig.secondaryDriverId(), + secondaryDriverVersion: sample.secondaryDriverVersion?.() ?? null, + }); + this.log.info( + `Metadata: commit=${this.commitId ?? 'N/A'}, driver=${this.driverConfig.driverId}, version=${version}`, + ); + await sample.close(); + } catch (error) { + this.log.warn(`Failed to get driver version for metadata: ${(error as Error).message}`); + this.writer.setMetadata({ + commitId: this.commitId, + driverId: this.driverConfig.driverId, + primaryDriverVersion: 'unknown', + secondaryDriverId: this.driverConfig.secondaryDriverId(), + secondaryDriverVersion: null, + }); + } + } + + private async executePhase(phase: PhaseConfig): Promise { + this.log.info(`=== Starting phase: ${phase.id} (${phase.description ?? ''}) ===`); + + const collector = new MetricsCollector(); + const clients = await this.createClients(phase); + const commands = CommandFactory.createAll(phase.commands); + + let status: string; + try { + if (phase.warmupRequests > 0) await this.warmup(clients, phase.warmupRequests); + + // Created *after* warmup, deliberately. The limiter starts its clock at + // construction, so building it earlier would bank the whole warmup + // duration as credit and release a burst of + // (warmup_duration / interval) requests the moment the workload starts -- + // defeating the evenly-spaced, no-burst property the limiter exists for. + const rateLimiter = phase.hasRpsLimit() ? RateLimiter.create(phase.rpsLimit) : null; + + collector.start(); + status = await this.runWorkload(phase, clients, commands, rateLimiter, collector); + collector.stop(); + } finally { + await this.closeClients(clients); + } + + this.writer.writePhaseResults({ + phaseId: phase.id, + status, + connections: phase.connections, + collector, + }); + this.logPhaseSummary(phase, collector, status); + } + + private async createClients(phase: PhaseConfig): Promise { + this.log.info(`Creating ${phase.connections} connections...`); + const cpsLimiter = phase.hasCpsLimit() ? RateLimiter.create(phase.cpsLimit) : null; + + const clients: BenchmarkClient[] = []; + for (let i = 0; i < phase.connections; i++) { + if (cpsLimiter !== null) await cpsLimiter.acquire(); + clients.push( + await BenchmarkClientFactory.createAndConnect(this.host, this.port, this.driverConfig), + ); + if ((i + 1) % CONNECTION_LOG_INTERVAL === 0) { + this.log.info(`Created ${i + 1}/${phase.connections} connections`); + } + } + this.log.info(`All ${clients.length} connections established`); + return clients; + } + + /** + * Send warmup PINGs on every client, failing fast if any of them errors. + * + * A dead or misconfigured server would otherwise produce a whole phase of + * nothing but errors, which is far harder to diagnose than an upfront throw. + * + * Uses `allSettled`, not `all`: `all` rejects on the first failure while the + * remaining warmup loops keep running unawaited, so `executePhase`'s `finally` + * would close the clients underneath them. Settling every loop first means a + * warmup failure leaves nothing in flight. + */ + private async warmup(clients: BenchmarkClient[], warmupRequests: number): Promise { + this.log.info(`Warmup: ${warmupRequests} PING(s) per client...`); + // Warmup mode lets the recording driver suppress simulated errors, so an + // error_rate workload is not aborted by the very errors it is measuring. + for (const client of clients) client.setWarmupMode?.(true); + let outcomes: PromiseSettledResult[]; + try { + outcomes = await Promise.allSettled( + clients.map(async (client) => { + for (let i = 0; i < warmupRequests; i++) { + const result = await client.ping(); + if (result.error !== undefined) { + throw new Error(`Warmup PING failed: ${result.error.message}`); + } + } + }), + ); + } finally { + for (const client of clients) client.setWarmupMode?.(false); + } + + const failure = outcomes.find((o): o is PromiseRejectedResult => o.status === 'rejected'); + if (failure !== undefined) { + throw failure.reason instanceof Error ? failure.reason : new Error(String(failure.reason)); + } + this.log.info('Warmup completed'); + } + + private async runWorkload( + phase: PhaseConfig, + clients: BenchmarkClient[], + commands: Command[], + rateLimiter: RateLimiter | null, + collector: MetricsCollector, + ): Promise { + const { completion } = phase; + const pipelineDepth = phase.effectivePipelineDepth(); + const seedBase = phase.keyspace.seedValue(); + // Shared across workers for sequential_int, so they collectively emit + // 0, 1, 2, ... exactly as the Java reference does. + const sharedCounter = new Counter(); + + const budget = completion.isRequestBased() ? new RequestBudget(completion.totalRequests()) : null; + const deadlineMs = completion.isDurationBased() + ? Date.now() + completion.durationSeconds() * 1000 + : null; + + const keepGoing = (): boolean => { + if (deadlineMs !== null && Date.now() >= deadlineMs) return false; + if (budget !== null) return budget.claim(); + return true; + }; + + this.log.info( + `Starting ${clients.length} workers (pipeline_depth=${pipelineDepth})...`, + ); + + const progressTimer = setInterval(() => { + this.logProgress(collector, completion.isRequestBased() ? completion.totalRequests() : null); + }, PROGRESS_LOG_INTERVAL_MS); + // Do not let the interval hold the event loop open past the phase. + progressTimer.unref(); + + try { + await Promise.all( + clients.map((client, index) => { + const keyGen = KeyGenerator.createWithSeed(phase.keyspace, seedBase + index, sharedCounter); + const selector = new CommandSelector(commands); + return pipelineDepth <= 1 + ? this.runSyncLoop(client, selector, keyGen, rateLimiter, collector, keepGoing) + : this.runPipelinedLoop( + client, + selector, + keyGen, + rateLimiter, + collector, + keepGoing, + pipelineDepth, + ); + }), + ); + this.log.info(`All operations completed (${collector.totalRequests} total requests)`); + return 'COMPLETED'; + } catch (error) { + this.log.error(`Error during workload execution: ${(error as Error).message}`); + return 'ERROR'; + } finally { + clearInterval(progressTimer); + } + } + + /** One in-flight request per connection (pipeline_depth <= 1). */ + private async runSyncLoop( + client: BenchmarkClient, + selector: CommandSelector, + keyGen: KeyGenerator, + rateLimiter: RateLimiter | null, + collector: MetricsCollector, + keepGoing: () => boolean, + ): Promise { + while (keepGoing()) { + if (rateLimiter !== null) await rateLimiter.acquire(); + collector.record(await this.issue(client, selector, keyGen)); + } + } + + /** + * Up to `pipelineDepth` in-flight requests per connection. + * + * Each slot runs its own claim/issue/record cycle, so a settled request is + * replaced immediately rather than waiting on a whole batch -- the same + * "refill as they land" behaviour as Java's `anyOf` loop, expressed as N + * independent slot loops sharing the connection. + */ + private async runPipelinedLoop( + client: BenchmarkClient, + selector: CommandSelector, + keyGen: KeyGenerator, + rateLimiter: RateLimiter | null, + collector: MetricsCollector, + keepGoing: () => boolean, + pipelineDepth: number, + ): Promise { + const slot = async (): Promise => { + while (keepGoing()) { + if (rateLimiter !== null) await rateLimiter.acquire(); + collector.record(await this.issue(client, selector, keyGen)); + } + }; + await Promise.all(Array.from({ length: pipelineDepth }, slot)); + } + + /** + * Select and run one command. + * + * Drivers already convert a rejection into a failed TimedResult, so a throw + * here means an engine-level bug rather than a server error; record it as a + * failed request and keep the phase running. + */ + private async issue( + client: BenchmarkClient, + selector: CommandSelector, + keyGen: KeyGenerator, + ): Promise { + const command = selector.select(); + // Only advance the key sequence for commands that consume a key, so PING + // does not silently shift the sequence other engines produce. + const key = command.usesKey ? keyGen.nextKey() : ''; + try { + return await command.execute(client, key); + } catch (error) { + return { + commandName: command.name, + latencyMicros: 0, + success: false, + errorMessage: (error as Error).message, + }; + } + } + + private async closeClients(clients: BenchmarkClient[]): Promise { + this.log.info(`Closing ${clients.length} connections...`); + for (const client of clients) { + try { + await client.close(); + } catch (error) { + this.log.warn(`Error closing client: ${(error as Error).message}`); + } + } + } + + private logProgress(collector: MetricsCollector, target: number | null): void { + const elapsedMs = Date.now() - collector.startTime(); + if (elapsedMs <= 0) return; + const current = collector.totalRequests; + const rate = Math.round((current * 1000) / elapsedMs); + if (target !== null) { + const percent = ((current * 100) / target).toFixed(1); + this.log.info(`Progress: ${current}/${target} requests (${percent}%) - ${rate} req/s`); + } else { + this.log.info(`Progress: ${current} requests - ${rate} req/s`); + } + } + + private logPhaseSummary(phase: PhaseConfig, collector: MetricsCollector, status: string): void { + const durationSeconds = collector.durationMillis() / 1000; + const rps = durationSeconds > 0 ? Math.round(collector.totalRequests / durationSeconds) : 0; + this.log.info(`=== Phase ${phase.id} completed: ${status} ===`); + this.log.info( + ` Duration: ${durationSeconds.toFixed(1)}s | Requests: ${collector.totalRequests} | ` + + `Errors: ${collector.totalErrors} | RPS: ${rps}`, + ); + for (const [name, m] of collector.commandMetrics) { + if (m.count() === 0 && m.errors === 0) continue; + this.log.info( + ` ${name}: ${m.requests} req (${m.errors} err) | ` + + `p50=${m.percentile(50)}us p95=${m.percentile(95)}us p99=${m.percentile(99)}us ` + + `p99.9=${m.percentile(99.9)}us | min=${m.min()}us max=${m.max()}us`, + ); + } + } +} diff --git a/node/src/engine/commandSelector.ts b/node/src/engine/commandSelector.ts new file mode 100644 index 0000000..d8e2731 --- /dev/null +++ b/node/src/engine/commandSelector.ts @@ -0,0 +1,43 @@ +/** + * Weighted command selection. + * + * Uses normalized cumulative weights, matching Java's inline CommandSelector + * (BenchmarkEngine.java:520-545) and the Python engine. Selection intentionally + * uses the platform RNG (`Math.random`), not the Java LCG: only *key* generation + * must be cross-engine deterministic. Each worker owns its own selector, so + * there is no shared state. + */ + +import type { Command } from '../command/command.js'; + +export class CommandSelector { + private readonly commands: Command[]; + private readonly cumulativeWeights: number[]; + + constructor(commands: Command[]) { + if (commands.length === 0) throw new Error('CommandSelector requires at least one command'); + this.commands = commands; + this.cumulativeWeights = CommandSelector.buildCumulativeWeights(commands); + } + + select(): Command { + const r = Math.random(); + for (let i = 0; i < this.cumulativeWeights.length; i++) { + if (r <= this.cumulativeWeights[i]!) return this.commands[i]!; + } + return this.commands[this.commands.length - 1]!; + } + + private static buildCumulativeWeights(commands: Command[]): number[] { + let totalWeight = commands.reduce((sum, command) => sum + command.weight, 0); + if (totalWeight === 0) totalWeight = 1.0; + + const cumulative: number[] = []; + let running = 0; + for (const command of commands) { + running += command.weight / totalWeight; + cumulative.push(running); + } + return cumulative; + } +} diff --git a/node/src/engine/javaRandom.ts b/node/src/engine/javaRandom.ts new file mode 100644 index 0000000..ce5efba --- /dev/null +++ b/node/src/engine/javaRandom.ts @@ -0,0 +1,68 @@ +/** + * Faithful port of `java.util.Random` (48-bit LCG). + * + * This guarantees identical `uniform_rand` key sequences across the Java + * (reference), Ruby, C#, Python and Node engines. Java is the canonical source, + * so this port reproduces `nextInt(bound)` exactly -- including the 32-bit + * signed-overflow rejection in the general case, which is what avoids modulo + * bias. (The Ruby port omits that rejection because Ruby integers are + * arbitrary-precision; this port emulates the int32 wraparound so it matches + * the Java reference rather than the Ruby approximation.) + * + * BigInt is mandatory for the state, not a stylistic choice: `seed * MULTIPLIER` + * reaches ~2^83, far past the 2^53 that a JS `number` can represent exactly, so + * a `number`-based port silently diverges from Java after the first step. + * + * @see https://docs.oracle.com/javase/8/docs/api/java/util/Random.html + */ + +const MULTIPLIER = 0x5deece66dn; +const ADDEND = 0xbn; +const MASK = (1n << 48n) - 1n; + +/** Interpret the low 32 bits of `value` as a signed 32-bit integer. */ +export function toInt32(value: number): number { + return value | 0; +} + +export class JavaRandom { + private seed: bigint; + + constructor(seed: number | bigint) { + this.seed = JavaRandom.initialScramble(seed); + } + + setSeed(seed: number | bigint): void { + this.seed = JavaRandom.initialScramble(seed); + } + + /** Return a random int in [0, bound) matching Java's nextInt(int). */ + nextInt(bound: number): number { + if (!Number.isInteger(bound) || bound <= 0) { + throw new Error(`bound must be a positive integer (got ${bound})`); + } + + // Power-of-two fast path (matches Java exactly). + if ((bound & -bound) === bound) { + return Number((BigInt(bound) * BigInt(this.next(31))) >> 31n); + } + + // General case: rejection sampling to avoid modulo bias. The rejection + // condition relies on 32-bit signed overflow, which `| 0` emulates. + for (;;) { + const bits = this.next(31); + const val = bits % bound; + if (toInt32(bits - val + (bound - 1)) >= 0) return val; + } + } + + private static initialScramble(seed: number | bigint): bigint { + return (BigInt(seed) ^ MULTIPLIER) & MASK; + } + + /** Java's `protected int next(int bits)`. Exposed for parity tests. */ + next(bits: number): number { + this.seed = (this.seed * MULTIPLIER + ADDEND) & MASK; + return Number(this.seed >> BigInt(48 - bits)); + } +} diff --git a/node/src/engine/keyGenerator.ts b/node/src/engine/keyGenerator.ts new file mode 100644 index 0000000..c44a87c --- /dev/null +++ b/node/src/engine/keyGenerator.ts @@ -0,0 +1,93 @@ +/** + * Key generator producing sequences identical to the other engines. + * + * - `sequential_int`: keys 0, 1, 2, ... N-1, wrapping around. + * - `uniform_rand`: Java-LCG random keys (see ./javaRandom.ts). + * + * Key formatting matches Java's `String.format("%0Nd", index)`: the numeric part + * is zero-padded to `max(keySizeBytes - prefix.length, 1)` digits. With the + * reference configs (`key_prefix: "bench:"`, `key_size_bytes: 16`) that yields + * `bench:` + 10 digits, e.g. `bench:0000000042`. + * + * Cross-worker semantics follow the Java reference: + * - `sequential_int` uses a counter SHARED across all workers in a phase, so the + * workers collectively emit 0, 1, 2, ... (this is what populates the whole + * keyspace during a WARMUP/populate phase). Pass a shared Counter. + * - `uniform_rand` uses a per-worker RNG seeded `baseSeed + workerIndex`. + */ + +import type { KeyspaceConfig } from '../config/keyspaceConfig.js'; +import { JavaRandom } from './javaRandom.js'; + +/** + * A monotonic 0-based counter, shared across a phase's workers. + * + * Safe to share across concurrent workers: `nextValue` performs its + * read-increment with no `await` in between, so it is atomic on the single + * JS event-loop thread. + */ +export class Counter { + private value: number; + + constructor(start = 0) { + this.value = start; + } + + nextValue(): number { + return this.value++; + } + + reset(): void { + this.value = 0; + } +} + +export class KeyGenerator { + private readonly config: KeyspaceConfig; + private readonly keyPrefix: string; + private readonly keysCount: number; + private readonly paddingWidth: number; + private readonly seed: number; + private readonly sequentialCounter: Counter; + private readonly random: JavaRandom; + + constructor(config: KeyspaceConfig, seedOverride?: number, sequentialCounter?: Counter) { + this.config = config; + this.keyPrefix = config.keyPrefix; + this.keysCount = config.keysCount; + this.paddingWidth = Math.max(config.keySizeBytes - config.keyPrefix.length, 1); + this.seed = seedOverride ?? config.seedValue(); + this.sequentialCounter = sequentialCounter ?? new Counter(); + this.random = new JavaRandom(this.seed); + } + + static create(config: KeyspaceConfig): KeyGenerator { + return new KeyGenerator(config); + } + + /** Per-worker generator with a unique seed and an optional shared counter. */ + static createWithSeed( + config: KeyspaceConfig, + seed: number, + sequentialCounter?: Counter, + ): KeyGenerator { + return new KeyGenerator(config, seed, sequentialCounter); + } + + nextKey(): string { + const rawIndex = this.config.isSequentialInt() + ? this.sequentialCounter.nextValue() + : this.random.nextInt(this.keysCount); + + return this.formatKey(rawIndex % this.keysCount); + } + + reset(): void { + this.sequentialCounter.reset(); + this.random.setSeed(this.seed); + } + + private formatKey(keyIndex: number): string { + return this.keyPrefix + String(keyIndex).padStart(this.paddingWidth, '0'); + } +} diff --git a/node/src/engine/rateLimiter.ts b/node/src/engine/rateLimiter.ts new file mode 100644 index 0000000..d0deef9 --- /dev/null +++ b/node/src/engine/rateLimiter.ts @@ -0,0 +1,66 @@ +/** + * Leaky-bucket rate limiter. + * + * Enforces a constant rate with no burst (evenly-spaced operations), matching + * the Java reference's interval math exactly: + * `intervalNanos = 1_000_000_000 / ratePerSecond` (RateLimiter.java:28). Despite + * what docs/ARCHITECTURE.md says about a "token bucket", every engine actually + * implements this leaky bucket, so this follows the code rather than the doc. + * + * A single limiter is shared across all of a phase's workers. Because JS is + * single-threaded, the check-and-advance of `nextAllowedNanos` has no `await` + * between read and write, so it is atomic -- no CAS loop is needed (Java needs + * one only because its issuers are real threads). + * + * Node-specific wrinkle: `setTimeout` clamps to ~1ms, so at high rates (a 100k + * rps limit is a 10us interval) a timer-based wait would undershoot the target + * badly. Sub-millisecond waits therefore yield via `setImmediate`, which returns + * to the event loop -- letting other connections progress -- without sleeping a + * whole millisecond. + */ + +const NANOS_PER_SECOND = 1_000_000_000n; +const NANOS_PER_MILLI = 1_000_000n; + +/** Yield to the event loop without a clamped timer delay. */ +function yieldToEventLoop(): Promise { + return new Promise((resolve) => setImmediate(resolve)); +} + +function sleepMillis(millis: number): Promise { + return new Promise((resolve) => setTimeout(resolve, millis)); +} + +export class RateLimiter { + readonly ratePerSecond: number; + private readonly intervalNanos: bigint; + private nextAllowedNanos: bigint; + + private constructor(ratePerSecond: number) { + this.ratePerSecond = ratePerSecond; + this.intervalNanos = NANOS_PER_SECOND / BigInt(ratePerSecond); + // The first operation is allowed immediately. + this.nextAllowedNanos = process.hrtime.bigint(); + } + + /** Return a limiter, or null for unlimited (rate <= 0). */ + static create(ratePerSecond: number): RateLimiter | null { + return ratePerSecond > 0 ? new RateLimiter(ratePerSecond) : null; + } + + async acquire(): Promise { + for (;;) { + const now = process.hrtime.bigint(); + if (now >= this.nextAllowedNanos) { + this.nextAllowedNanos += this.intervalNanos; + return; + } + const waitNanos = this.nextAllowedNanos - now; + if (waitNanos >= NANOS_PER_MILLI) { + await sleepMillis(Number(waitNanos / NANOS_PER_MILLI)); + } else { + await yieldToEventLoop(); + } + } + } +} diff --git a/node/src/metrics/collector.ts b/node/src/metrics/collector.ts new file mode 100644 index 0000000..ecca794 --- /dev/null +++ b/node/src/metrics/collector.ts @@ -0,0 +1,111 @@ +/** + * Latency metrics collection. + * + * Single-event-loop design: no locks are needed because `record` runs to + * completion without awaiting, so concurrent workers never interleave inside it. + * Latencies are clamped to 600s before recording, and errors are counted but not + * recorded into the histogram -- matching the other engines. + */ + +import type { CommandResult } from '../command/command.js'; +import { HIGHEST_TRACKABLE_VALUE, newHistogram, type Histogram } from './hdrHistogram.js'; + +export class CommandMetrics { + readonly commandName: string; + requests = 0; + errors = 0; + /** + * Created eagerly (like the Java reference) so the NDJSON `hdr` block and + * summary are always present, even for a command that only ever errors -- an + * empty histogram reports count 0 and zero percentiles. + */ + readonly histogram: Histogram = newHistogram(); + + constructor(commandName: string) { + this.commandName = commandName; + } + + record(result: CommandResult): void { + this.requests += 1; + if (result.success) { + this.histogram.recordValue(Math.min(result.latencyMicros, HIGHEST_TRACKABLE_VALUE)); + } else { + this.errors += 1; + } + } + + count(): number { + return this.histogram.totalCount; + } + + /** + * Lowest recorded latency, matching Java's `Histogram.getMinValue()`. + * + * Deliberately NOT `minNonZeroValue`: that skips a legitimately recorded 0us + * sample, and on an empty histogram it returns Number.MAX_SAFE_INTEGER, which + * would land in the NDJSON as a nonsense min. `getValueAtPercentile(0)` returns + * 0 when empty and Java's `getMinValue()` otherwise -- verified equal for + * 1us..599s (and it is what the Ruby encoder uses). + */ + min(): number { + return this.histogram.getValueAtPercentile(0); + } + + /** + * Highest recorded latency, matching Java's `Histogram.getMaxValue()`. + * + * Deliberately NOT `maxValue`: hdr-histogram-js returns the raw recorded + * sample there, while Java returns the *bucket's* highest equivalent value. At + * 3 significant figures those diverge above ~1000us -- recording 50000us gives + * 50000 in JS but 50015 in Java -- which would make summary.max quietly + * incomparable across engines. `getValueAtPercentile(100)` is Java's value. + */ + max(): number { + return this.histogram.getValueAtPercentile(100); + } + + percentile(pct: number): number { + return this.histogram.getValueAtPercentile(pct); + } +} + +export class MetricsCollector { + readonly commandMetrics = new Map(); + totalRequests = 0; + totalErrors = 0; + private startTimeMs: number | null = null; + private endTimeMs: number | null = null; + + start(): void { + this.startTimeMs = Date.now(); + } + + stop(): void { + this.endTimeMs = Date.now(); + } + + record(result: CommandResult): void { + this.totalRequests += 1; + if (!result.success) this.totalErrors += 1; + + let metrics = this.commandMetrics.get(result.commandName); + if (metrics === undefined) { + metrics = new CommandMetrics(result.commandName); + this.commandMetrics.set(result.commandName, metrics); + } + metrics.record(result); + } + + startTime(): number { + return this.startTimeMs ?? 0; + } + + endTime(): number { + return this.endTimeMs ?? 0; + } + + durationMillis(): number { + if (this.startTimeMs === null || this.endTimeMs === null) return 0; + return this.endTimeMs - this.startTimeMs; + } +} diff --git a/node/src/metrics/hdrHistogram.ts b/node/src/metrics/hdrHistogram.ts new file mode 100644 index 0000000..f6c748b --- /dev/null +++ b/node/src/metrics/hdrHistogram.ts @@ -0,0 +1,48 @@ +/** + * HdrHistogram helpers. + * + * Uses `hdr-histogram-js`, the TypeScript port of HdrHistogram. Its + * `encodeIntoCompressedBase64()` emits the base64-encoded V2 *compressed* + * payload -- the same format Java's `encodeIntoCompressedByteBuffer` + + * `Base64.getEncoder()` produces (NdjsonMetricsWriter.java:164-180) and the same + * the Ruby/Python engines emit -- so payloads are mutually decodable across + * engines for cross-language analysis. (Byte-identity is not guaranteed since + * zlib compression levels may differ, but decodability -- what merge/analysis + * needs -- is.) + * + * The returned string is ALREADY base64 (it starts `HIST`). Never base64 it + * again: double-encoding produces a payload Java and Ruby cannot decode. + * + * Histograms use range (1, 600_000_000, 3): 1 microsecond to 600 seconds at 3 + * significant figures, matching every other engine (Java + * `SynchronizedHistogram(600_000_000, 3)`, C# `LongConcurrentHistogram(1, + * 600_000_000, 3)`, Ruby `HDRHistogram.new(1, 600_000_000, 3)`). + */ + +import * as hdr from 'hdr-histogram-js'; + +export const LOWEST_TRACKABLE_VALUE = 1; +export const HIGHEST_TRACKABLE_VALUE = 600_000_000; // 600 seconds in microseconds +export const SIGNIFICANT_FIGURES = 3; + +export type Histogram = hdr.Histogram; + +export function newHistogram(): Histogram { + return hdr.build({ + bitBucketSize: 64, + autoResize: false, + lowestDiscernibleValue: LOWEST_TRACKABLE_VALUE, + highestTrackableValue: HIGHEST_TRACKABLE_VALUE, + numberOfSignificantValueDigits: SIGNIFICANT_FIGURES, + }); +} + +/** Return the base64 V2-compressed encoding, ready for `payload_b64`. */ +export function encodeBase64(histogram: Histogram): string { + return hdr.encodeIntoCompressedBase64(histogram); +} + +/** Inverse of `encodeBase64`, used by the parity tests. */ +export function decodeBase64(payload: string): Histogram { + return hdr.decodeFromCompressedBase64(payload); +} diff --git a/node/src/metrics/ndjsonWriter.ts b/node/src/metrics/ndjsonWriter.ts new file mode 100644 index 0000000..9aa7d60 --- /dev/null +++ b/node/src/metrics/ndjsonWriter.ts @@ -0,0 +1,122 @@ +/** + * Writes benchmark metrics as NDJSON (newline-delimited JSON), one line per + * phase, so the orchestrator can detect phase completion by watching for new + * lines. + * + * The record shape is byte-for-byte the contract in docs/ARCHITECTURE.md + * "Metrics Output Format" plus the `metadata` block the Java writer emits + * (NdjsonMetricsWriter.java:92-111). The matrix runner treats "exited 0 but wrote + * no record" as a cell failure (run_benchmark_matrix.py:1063-1069), so appending + * really must happen. + */ + +import { appendFileSync, mkdirSync } from 'node:fs'; +import { dirname } from 'node:path'; + +import type { CommandMetrics, MetricsCollector } from './collector.js'; +import { encodeBase64, SIGNIFICANT_FIGURES } from './hdrHistogram.js'; + +export interface Metadata { + commitId?: string | null; + driverId?: string | null; + primaryDriverVersion?: string | null; + secondaryDriverId?: string | null; + secondaryDriverVersion?: string | null; +} + +export class NdjsonWriter { + private readonly outputPath: string; + private metadata: Metadata = {}; + + constructor(outputPath: string) { + this.outputPath = outputPath; + } + + setMetadata(metadata: Metadata): void { + this.metadata = metadata; + } + + writePhaseResults(options: { + phaseId: string; + status: string; + connections: number; + collector: MetricsCollector; + }): void { + const record = this.buildPhaseRecord(options); + const parent = dirname(this.outputPath); + if (parent && parent !== '.') mkdirSync(parent, { recursive: true }); + appendFileSync(this.outputPath, `${JSON.stringify(record)}\n`, 'utf8'); + } + + private buildPhaseRecord(options: { + phaseId: string; + status: string; + connections: number; + collector: MetricsCollector; + }): Record { + const { phaseId, status, connections, collector } = options; + const record: Record = {}; + + const { commitId, driverId, primaryDriverVersion, secondaryDriverId, secondaryDriverVersion } = + this.metadata; + if (commitId != null || driverId != null) { + const metadata: Record = {}; + if (commitId != null) metadata['commit_id'] = commitId; + metadata['timestamp'] = new Date().toISOString(); + if (driverId != null) metadata['driver_id'] = driverId; + if (primaryDriverVersion != null) metadata['primary_driver_version'] = primaryDriverVersion; + if (secondaryDriverId != null) metadata['secondary_driver_id'] = secondaryDriverId; + if (secondaryDriverVersion != null) { + metadata['secondary_driver_version'] = secondaryDriverVersion; + } + record['metadata'] = metadata; + } + + record['phase'] = { + id: phaseId, + status, + start_timestamp: new Date(collector.startTime()).toISOString(), + finish_timestamp: new Date(collector.endTime()).toISOString(), + duration_ms: collector.durationMillis(), + connections, + }; + + record['totals'] = { + requests: collector.totalRequests, + errors: collector.totalErrors, + }; + + const metrics: Record = {}; + for (const [commandName, commandMetrics] of collector.commandMetrics) { + metrics[commandName] = NdjsonWriter.buildCommandRecord(commandMetrics); + } + record['metrics'] = metrics; + + return record; + } + + private static buildCommandRecord(m: CommandMetrics): Record { + return { + requests: m.requests, + errors: m.errors, + latency: { + unit: 'us', + count: m.count(), + summary: { + min: m.min(), + p50: m.percentile(50), + p95: m.percentile(95), + p99: m.percentile(99), + p999: m.percentile(99.9), + max: m.max(), + }, + hdr: { + format: 'hdr', + sigfig: SIGNIFICANT_FIGURES, + // Already base64 -- do NOT encode again (see metrics/hdrHistogram.ts). + payload_b64: encodeBase64(m.histogram), + }, + }, + }; + } +} diff --git a/node/src/version.ts b/node/src/version.ts new file mode 100644 index 0000000..966a9ef --- /dev/null +++ b/node/src/version.ts @@ -0,0 +1,2 @@ +/** Engine version, reported by `--version` and `--info`. */ +export const VERSION = '1.0.0'; diff --git a/node/test/integration/clients.test.ts b/node/test/integration/clients.test.ts new file mode 100644 index 0000000..24548d4 --- /dev/null +++ b/node/test/integration/clients.test.ts @@ -0,0 +1,197 @@ +/** + * Live-server tests for the real drivers. + * + * Skipped unless VALKEY_HOST/VALKEY_PORT point at a running server, matching the + * Java/Ruby/C# integration suites (`make node-integration-test` starts one). + */ + +import assert from 'node:assert/strict'; +import { mkdtempSync, readFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { describe, it } from 'node:test'; + +import { BenchmarkClientFactory } from '../../src/client/factory.js'; +import { ConfigLoader } from '../../src/config/loader.js'; +import { BenchmarkEngine, type Logger } from '../../src/engine/benchmark.js'; +import { decodeBase64 } from '../../src/metrics/hdrHistogram.js'; + +const HOST = process.env['VALKEY_HOST'] ?? 'localhost'; +const PORT = Number(process.env['VALKEY_PORT'] ?? 6379); +const REAL_DRIVERS = ['valkey-glide-node', 'ioredis', 'iovalkey'] as const; + +const silentLogger: Logger = { info: () => {}, warn: () => {}, error: () => {} }; + +/** True when VALKEY_HOST is set, i.e. a server is expected to be reachable. */ +const enabled = process.env['VALKEY_HOST'] !== undefined; +const skip = enabled ? false : 'set VALKEY_HOST to run the live-server tests'; + +function driverConfig(driverId: string, extra: Record = {}) { + return ConfigLoader.parseDriverConfig({ + schema_version: '1.0', + driver_id: driverId, + mode: 'standalone', + specific_driver_config: {}, + ...extra, + }); +} + +for (const driverId of REAL_DRIVERS) { + describe(`${driverId} against a live server`, { skip }, () => { + it('connects, pings, sets, gets and closes', async () => { + const client = await BenchmarkClientFactory.createAndConnect(HOST, PORT, driverConfig(driverId)); + try { + const ping = await client.ping(); + assert.equal(ping.error, undefined); + assert.equal(ping.value, 'PONG'); + assert.ok(ping.latencyMicros >= 0); + + const key = `node-it:${driverId}:${process.pid}`; + const payload = Buffer.from('x'.repeat(64), 'latin1'); + + const set = await client.set(key, payload); + assert.equal(set.error, undefined); + assert.equal(set.value, 'OK'); + + const get = await client.get(key); + assert.equal(get.error, undefined); + // Every driver must return the same decoded shape, or the engines are + // charged for different work: a string of the payload's length. + assert.equal(typeof get.value, 'string'); + assert.equal(get.value, payload.toString('latin1')); + } finally { + await client.close(); + } + }); + + it('returns null for a missing key rather than erroring', async () => { + const client = await BenchmarkClientFactory.createAndConnect(HOST, PORT, driverConfig(driverId)); + try { + const get = await client.get(`node-it:absent:${process.pid}:${Math.random()}`); + assert.equal(get.error, undefined); + assert.equal(get.value, null); + } finally { + await client.close(); + } + }); + + it('reports a real driver version, not "unknown"', async () => { + const client = await BenchmarkClientFactory.create(driverId); + assert.match(client.driverVersion(), /^\d+\.\d+\.\d+/); + }); + + it('applies command_timeout_ms without breaking normal commands', async () => { + const client = await BenchmarkClientFactory.createAndConnect( + HOST, + PORT, + driverConfig(driverId, { command_timeout_ms: 5000 }), + ); + try { + assert.equal((await client.ping()).error, undefined); + } finally { + await client.close(); + } + }); + + it('runs a short workload and writes decodable metrics', async () => { + const metricsPath = join(mkdtempSync(join(tmpdir(), 'resp-bench-node-')), 'metrics.ndjson'); + await new BenchmarkEngine({ + host: HOST, + port: PORT, + driverConfig: driverConfig(driverId), + workloadConfig: ConfigLoader.parseWorkloadConfig({ + benchmark_profile: { name: `${driverId} smoke` }, + phases: [ + { + id: 'STEADY', + connections: 2, + warmup_requests: 1, + completion: { type: 'requests', requests: 200 }, + keyspace: { keys_count: 100, key_prefix: 'node-it:', key_size_bytes: 16 }, + commands: [ + { command: 'set', weight: 0.5, data_size_bytes: 64 }, + { command: 'get', weight: 0.5 }, + ], + }, + ], + }), + metricsPath, + commitId: 'integration-test', + logger: silentLogger, + }).run(); + + const record = JSON.parse(readFileSync(metricsPath, 'utf8').trimEnd().split('\n')[0]!); + assert.equal(record.phase.status, 'COMPLETED'); + assert.equal(record.totals.requests, 200); + assert.equal(record.totals.errors, 0, 'a healthy server should produce no errors'); + assert.equal(record.metadata.driver_id, driverId); + assert.match(record.metadata.primary_driver_version, /^\d+\.\d+\.\d+/); + for (const name of Object.keys(record.metrics)) { + const latency = record.metrics[name].latency; + assert.ok(latency.hdr.payload_b64.startsWith('HIST')); + assert.equal(decodeBase64(latency.hdr.payload_b64).totalCount, latency.count); + assert.ok(latency.summary.p50 >= 0); + assert.ok(latency.summary.max >= latency.summary.p50); + } + }); + + it('runs a pipelined workload', async () => { + const metricsPath = join(mkdtempSync(join(tmpdir(), 'resp-bench-node-')), 'metrics.ndjson'); + await new BenchmarkEngine({ + host: HOST, + port: PORT, + driverConfig: driverConfig(driverId), + workloadConfig: ConfigLoader.parseWorkloadConfig({ + phases: [ + { + id: 'PIPELINED', + connections: 2, + pipeline_depth: 8, + warmup_requests: 1, + completion: { type: 'requests', requests: 200 }, + keyspace: { keys_count: 100, key_prefix: 'node-it:', key_size_bytes: 16 }, + commands: [{ command: 'get', weight: 1.0 }], + }, + ], + }), + metricsPath, + logger: silentLogger, + }).run(); + + const record = JSON.parse(readFileSync(metricsPath, 'utf8').trimEnd().split('\n')[0]!); + assert.equal(record.phase.status, 'COMPLETED'); + // 2 connections x depth 8 = 16 in-flight slots on one shared budget of 200. + assert.equal(record.totals.requests, 200); + assert.equal(record.totals.errors, 0); + }); + }); +} + +describe('unreachable server', { skip }, () => { + it('fails fast instead of recording a phase of pure errors', async () => { + // Port 1 is reserved and never listening. A dead server must abort the run + // with a clear error, not silently produce 100% error metrics. + await assert.rejects( + () => + new BenchmarkEngine({ + host: '127.0.0.1', + port: 1, + driverConfig: driverConfig('ioredis'), + workloadConfig: ConfigLoader.parseWorkloadConfig({ + phases: [ + { + id: 'STEADY', + connections: 1, + warmup_requests: 1, + completion: { type: 'requests', requests: 10 }, + keyspace: { keys_count: 10 }, + commands: [{ command: 'get', weight: 1.0 }], + }, + ], + }), + metricsPath: join(mkdtempSync(join(tmpdir(), 'resp-bench-node-')), 'metrics.ndjson'), + logger: silentLogger, + }).run(), + ); + }); +}); diff --git a/node/test/integration/recordingWorkload.test.ts b/node/test/integration/recordingWorkload.test.ts new file mode 100644 index 0000000..df67dbd --- /dev/null +++ b/node/test/integration/recordingWorkload.test.ts @@ -0,0 +1,330 @@ +/** + * Full-engine tests against the `recording` driver โ€” no server required. + * + * These are the tests that catch engine-level regressions the unit tests cannot: + * the shared request budget, warmup fail-fast, pipelining, rate limiting, and the + * NDJSON a real run produces. + */ + +import assert from 'node:assert/strict'; +import { mkdtempSync, readFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { describe, it } from 'node:test'; + +import { ConfigLoader } from '../../src/config/loader.js'; +import { BenchmarkEngine, type Logger } from '../../src/engine/benchmark.js'; +import { decodeBase64 } from '../../src/metrics/hdrHistogram.js'; + +const silentLogger: Logger = { info: () => {}, warn: () => {}, error: () => {} }; + +function tempMetricsPath(): string { + return join(mkdtempSync(join(tmpdir(), 'resp-bench-node-')), 'metrics.ndjson'); +} + +function readRecords(path: string): Array> { + return readFileSync(path, 'utf8') + .trimEnd() + .split('\n') + .map((line) => JSON.parse(line)); +} + +function driverConfig(specific: Record = {}) { + return ConfigLoader.parseDriverConfig({ + schema_version: '1.0', + driver_id: 'recording', + mode: 'standalone', + specific_driver_config: specific, + }); +} + +function workload(phases: unknown[]) { + return ConfigLoader.parseWorkloadConfig({ + schema_version: '1.0', + benchmark_profile: { name: 'Node engine test' }, + phases, + }); +} + +const STEADY_PHASE = { + id: 'STEADY', + description: 'short steady phase', + connections: 4, + warmup_requests: 1, + completion: { type: 'requests', requests: 200 }, + keyspace: { keys_count: 100, key_prefix: 'e2e:', key_size_bytes: 16 }, + commands: [ + { command: 'set', weight: 0.5, data_size_bytes: 32 }, + { command: 'get', weight: 0.5 }, + ], +}; + +async function run(options: { + phases: unknown[]; + specific?: Record; + commitId?: string; +}): Promise<{ path: string; records: Array> }> { + const path = tempMetricsPath(); + await new BenchmarkEngine({ + host: 'localhost', + port: 6379, + driverConfig: driverConfig(options.specific), + workloadConfig: workload(options.phases), + metricsPath: path, + commitId: options.commitId ?? 'test-commit', + logger: silentLogger, + }).run(); + return { path, records: readRecords(path) }; +} + +describe('recording-driver workload', () => { + it('runs a phase end to end and writes valid NDJSON', async () => { + const { records } = await run({ phases: [STEADY_PHASE] }); + + assert.equal(records.length, 1); + const record = records[0]!; + assert.equal(record['phase'].id, 'STEADY'); + assert.equal(record['phase'].status, 'COMPLETED'); + assert.equal(record['phase'].connections, 4); + assert.equal(record['totals'].requests, 200); + assert.equal(record['totals'].errors, 0); + assert.equal(record['metadata'].commit_id, 'test-commit'); + assert.equal(record['metadata'].driver_id, 'recording'); + assert.equal(record['metadata'].primary_driver_version, '1.0.0'); + + // Both commands were exercised and their counts add up to the total. + const commands = Object.keys(record['metrics']).sort(); + assert.deepEqual(commands, ['GET', 'SET']); + const sum = commands.reduce((acc, name) => acc + record['metrics'][name].requests, 0); + assert.equal(sum, 200); + + for (const name of commands) { + const payload = record['metrics'][name].latency.hdr.payload_b64; + assert.ok(payload.startsWith('HIST')); + assert.equal(decodeBase64(payload).totalCount, record['metrics'][name].latency.count); + } + }); + + it('honours the shared request budget exactly, not a per-worker split', async () => { + // 7 requests across 4 connections does not divide evenly. A pre-split budget + // would round to 4 or 8; a shared budget lands on exactly 7. + const { records } = await run({ + phases: [{ ...STEADY_PHASE, connections: 4, completion: { type: 'requests', requests: 7 } }], + }); + assert.equal(records[0]!['totals'].requests, 7); + }); + + it('honours the budget when connections exceed the request count', async () => { + const { records } = await run({ + phases: [{ ...STEADY_PHASE, connections: 8, completion: { type: 'requests', requests: 3 } }], + }); + assert.equal(records[0]!['totals'].requests, 3); + }); + + it('runs each phase in order, appending one record per phase', async () => { + const { records } = await run({ + phases: [ + { + ...STEADY_PHASE, + id: 'WARMUP', + connections: 2, + completion: { type: 'requests', requests: 20 }, + commands: [{ command: 'set', weight: 1.0, data_size_bytes: 16 }], + }, + { ...STEADY_PHASE, id: 'STEADY', completion: { type: 'requests', requests: 30 } }, + ], + }); + + assert.deepEqual( + records.map((r) => r['phase'].id), + ['WARMUP', 'STEADY'], + ); + assert.equal(records[0]!['totals'].requests, 20); + assert.equal(records[1]!['totals'].requests, 30); + assert.deepEqual(Object.keys(records[0]!['metrics']), ['SET']); + }); + + it('stops a duration-based phase at the deadline', async () => { + const started = Date.now(); + const { records } = await run({ + phases: [{ ...STEADY_PHASE, connections: 2, completion: { type: 'duration', seconds: 1 } }], + }); + const elapsed = Date.now() - started; + + assert.equal(records[0]!['phase'].status, 'COMPLETED'); + assert.ok(records[0]!['totals'].requests > 0, 'a duration phase should do some work'); + assert.ok(elapsed >= 900, `finished suspiciously early: ${elapsed}ms`); + assert.ok(elapsed < 6000, `overran the 1s deadline: ${elapsed}ms`); + }); + + it('keeps pipelined runs on the same shared budget', async () => { + const { records } = await run({ + phases: [ + { + ...STEADY_PHASE, + connections: 3, + pipeline_depth: 4, + completion: { type: 'requests', requests: 50 }, + }, + ], + }); + // 3 connections x depth 4 = 12 in-flight slots all drawing on one budget of + // 50; an over-issuing pipeline would show more than 50 requests. + assert.equal(records[0]!['totals'].requests, 50); + }); + + it('surfaces injected errors as errors, not as latency samples', async () => { + const { records } = await run({ + phases: [{ ...STEADY_PHASE, connections: 2, completion: { type: 'requests', requests: 100 } }], + specific: { error_rate: 1.0, error_message: 'Simulated failure' }, + }); + + const record = records[0]!; + assert.equal(record['totals'].requests, 100); + assert.equal(record['totals'].errors, 100); + for (const name of Object.keys(record['metrics'])) { + const metrics = record['metrics'][name]; + assert.equal(metrics.errors, metrics.requests); + // Failed requests are counted but never recorded into the histogram. + assert.equal(metrics.latency.count, 0); + assert.ok(metrics.latency.hdr.payload_b64.startsWith('HIST')); + } + }); + + it('records a partial error rate on both sides of the split', async () => { + const { records } = await run({ + phases: [{ ...STEADY_PHASE, connections: 2, completion: { type: 'requests', requests: 400 } }], + specific: { error_rate: 0.5 }, + }); + const { requests, errors } = records[0]!['totals']; + assert.equal(requests, 400); + assert.ok(errors > 100 && errors < 300, `error count ${errors} not near half of 400`); + }); + + it('does not count warmup requests toward the phase totals', async () => { + // Warmup runs 3 PINGs on each of 2 connections. If those leaked into the + // measured metrics we would see 6 extra requests and a PING command key. + const { records } = await run({ + phases: [ + { + ...STEADY_PHASE, + connections: 2, + warmup_requests: 3, + completion: { type: 'requests', requests: 10 }, + }, + ], + }); + assert.equal(records[0]!['totals'].requests, 10); + assert.equal('PING' in records[0]!['metrics'], false); + }); + + it('suppresses injected errors during warmup so the phase still runs', async () => { + // Warmup's fail-fast exists to catch an unreachable server. An error_rate + // workload is deliberately measuring errors, so warmup must not abort on + // them -- Java does the same via setWarmupMode. + const { records } = await run({ + phases: [ + { + ...STEADY_PHASE, + connections: 2, + warmup_requests: 2, + completion: { type: 'requests', requests: 20 }, + }, + ], + specific: { error_rate: 1.0, error_message: 'Simulated failure' }, + }); + assert.equal(records[0]!['phase'].status, 'COMPLETED'); + assert.equal(records[0]!['totals'].requests, 20); + assert.equal(records[0]!['totals'].errors, 20); + }); + + it('applies an rps_limit to the whole phase', async () => { + const started = Date.now(); + const { records } = await run({ + phases: [ + { + ...STEADY_PHASE, + connections: 4, + rps_limit: 100, + completion: { type: 'requests', requests: 50 }, + }, + ], + }); + const elapsed = Date.now() - started; + + assert.equal(records[0]!['totals'].requests, 50); + // 50 requests at 100/s cannot finish faster than ~0.49s however many + // connections are issuing them. + assert.ok(elapsed >= 400, `rps_limit was not enforced: ${elapsed}ms for 50 requests at 100/s`); + }); + + it('does not bank warmup time as rate-limiter credit', async () => { + // Regression: the limiter starts its clock at construction, so building it + // before warmup banked the whole warmup duration as credit and released a + // burst of (warmup_duration / interval) requests once the workload began. + // + // Sized so the burst would swallow the entire workload: warmup is 25 PINGs + // at 20ms = ~500ms of idle limiter time, and at 50 rps (a 20ms interval) + // that banks ~25 free requests -- every request this phase issues. A banked + // limiter therefore finishes in ~500ms (warmup only); a correctly-scoped one + // additionally paces 24 intervals, so ~980ms. + const started = Date.now(); + const { records } = await run({ + phases: [ + { + ...STEADY_PHASE, + connections: 4, + warmup_requests: 25, + rps_limit: 50, + completion: { type: 'requests', requests: 25 }, + }, + ], + specific: { operation_delay_micros: 20_000 }, + }); + const elapsed = Date.now() - started; + + assert.equal(records[0]!['totals'].requests, 25); + assert.ok( + elapsed >= 850, + `rate limiter released a warmup-banked burst: 25 requests at 50 rps after ` + + `a ~500ms warmup took only ${elapsed}ms (expected ~980ms)`, + ); + }); + + it('gates connection setup with a cps_limit', async () => { + const started = Date.now(); + await run({ + phases: [ + { + ...STEADY_PHASE, + connections: 5, + cps_limit: 20, + completion: { type: 'requests', requests: 5 }, + }, + ], + }); + const elapsed = Date.now() - started; + // 5 connections at 20/s means the last one opens ~200ms in. + assert.ok(elapsed >= 150, `cps_limit was not enforced: ${elapsed}ms to open 5 connections`); + }); + + it('generates keys inside the configured keyspace', async () => { + const { records } = await run({ + phases: [ + { + ...STEADY_PHASE, + connections: 2, + completion: { type: 'requests', requests: 40 }, + keyspace: { + keys_count: 10, + key_prefix: 'e2e:', + key_size_bytes: 8, + generation_alg: 'uniform_rand', + seed: 12345, + }, + }, + ], + }); + assert.equal(records[0]!['totals'].requests, 40); + }); +}); diff --git a/node/test/unit/collector.test.ts b/node/test/unit/collector.test.ts new file mode 100644 index 0000000..da926e9 --- /dev/null +++ b/node/test/unit/collector.test.ts @@ -0,0 +1,97 @@ +import assert from 'node:assert/strict'; +import { describe, it } from 'node:test'; + +import { CommandMetrics, MetricsCollector } from '../../src/metrics/collector.js'; +import { HIGHEST_TRACKABLE_VALUE } from '../../src/metrics/hdrHistogram.js'; + +describe('CommandMetrics', () => { + it('counts requests and records only successes into the histogram', () => { + const metrics = new CommandMetrics('GET'); + metrics.record({ commandName: 'GET', latencyMicros: 100, success: true }); + metrics.record({ commandName: 'GET', latencyMicros: 200, success: true }); + metrics.record({ commandName: 'GET', latencyMicros: 300, success: false }); + + assert.equal(metrics.requests, 3); + assert.equal(metrics.errors, 1); + // The failed request's latency must not skew the distribution. + assert.equal(metrics.count(), 2); + assert.equal(metrics.max(), 200); + }); + + it('reports zeros for an empty histogram rather than sentinels', () => { + // minNonZeroValue would return Number.MAX_SAFE_INTEGER here and land a + // nonsense min in the NDJSON. + const metrics = new CommandMetrics('GET'); + assert.equal(metrics.count(), 0); + assert.equal(metrics.min(), 0); + assert.equal(metrics.max(), 0); + assert.equal(metrics.percentile(50), 0); + assert.equal(metrics.percentile(99.9), 0); + }); + + it('reports a legitimately recorded 0us sample as min 0', () => { + const metrics = new CommandMetrics('PING'); + metrics.record({ commandName: 'PING', latencyMicros: 0, success: true }); + metrics.record({ commandName: 'PING', latencyMicros: 5, success: true }); + assert.equal(metrics.min(), 0); + }); + + it('matches Java getMinValue/getMaxValue bucket quantization', () => { + // Verified against org.HdrHistogram.Histogram(1, 600_000_000, 3): at 3 + // significant figures Java reports the bucket's equivalent bounds, not the + // raw sample. hdr-histogram-js' maxValue/minNonZeroValue do NOT do this, so + // these anchors keep summary.min/max comparable across engines. + for (const [recorded, expectedMin, expectedMax] of [ + [1, 1, 1], + [100, 100, 100], + [1234, 1234, 1234], + [50_000, 49_984, 50_015], + [599_000_000, 598_736_896, 599_261_183], + ] as const) { + const metrics = new CommandMetrics('GET'); + metrics.record({ commandName: 'GET', latencyMicros: recorded, success: true }); + assert.equal(metrics.min(), expectedMin, `min for ${recorded}`); + assert.equal(metrics.max(), expectedMax, `max for ${recorded}`); + } + }); + + it('clamps a latency above the trackable range instead of throwing', () => { + const metrics = new CommandMetrics('GET'); + metrics.record({ + commandName: 'GET', + latencyMicros: HIGHEST_TRACKABLE_VALUE * 2, + success: true, + }); + assert.equal(metrics.count(), 1); + }); +}); + +describe('MetricsCollector', () => { + it('aggregates totals across commands', () => { + const collector = new MetricsCollector(); + collector.record({ commandName: 'GET', latencyMicros: 10, success: true }); + collector.record({ commandName: 'SET', latencyMicros: 20, success: true }); + collector.record({ commandName: 'SET', latencyMicros: 30, success: false }); + + assert.equal(collector.totalRequests, 3); + assert.equal(collector.totalErrors, 1); + assert.deepEqual([...collector.commandMetrics.keys()], ['GET', 'SET']); + assert.equal(collector.commandMetrics.get('SET')!.requests, 2); + }); + + it('preserves first-seen command order for stable NDJSON output', () => { + const collector = new MetricsCollector(); + collector.record({ commandName: 'SET', latencyMicros: 1, success: true }); + collector.record({ commandName: 'GET', latencyMicros: 1, success: true }); + assert.deepEqual([...collector.commandMetrics.keys()], ['SET', 'GET']); + }); + + it('reports zero duration until both start and stop have run', () => { + const collector = new MetricsCollector(); + assert.equal(collector.durationMillis(), 0); + collector.start(); + assert.equal(collector.durationMillis(), 0); + collector.stop(); + assert.ok(collector.durationMillis() >= 0); + }); +}); diff --git a/node/test/unit/commandSelector.test.ts b/node/test/unit/commandSelector.test.ts new file mode 100644 index 0000000..f29ae9e --- /dev/null +++ b/node/test/unit/commandSelector.test.ts @@ -0,0 +1,59 @@ +import assert from 'node:assert/strict'; +import { describe, it } from 'node:test'; + +import { CommandFactory } from '../../src/command/factory.js'; +import { CommandConfig } from '../../src/config/commandConfig.js'; +import { CommandSelector } from '../../src/engine/commandSelector.js'; + +function selectorFor(weights: Array<[string, number]>): CommandSelector { + return new CommandSelector( + CommandFactory.createAll( + weights.map(([command, weight]) => new CommandConfig({ command, weight })), + ), + ); +} + +function distribution(selector: CommandSelector, draws: number): Map { + const counts = new Map(); + for (let i = 0; i < draws; i++) { + const name = selector.select().name; + counts.set(name, (counts.get(name) ?? 0) + 1); + } + return counts; +} + +describe('CommandSelector', () => { + it('respects an 80/20 weighting within tolerance', () => { + const draws = 20_000; + const counts = distribution(selectorFor([['get', 0.8], ['set', 0.2]]), draws); + const getShare = (counts.get('GET') ?? 0) / draws; + assert.ok(Math.abs(getShare - 0.8) < 0.02, `GET share ${getShare.toFixed(3)} not near 0.8`); + }); + + it('normalizes weights that do not sum to 1', () => { + const draws = 20_000; + const counts = distribution(selectorFor([['get', 0.25], ['set', 0.25]]), draws); + const getShare = (counts.get('GET') ?? 0) / draws; + assert.ok(Math.abs(getShare - 0.5) < 0.02, `GET share ${getShare.toFixed(3)} not near 0.5`); + }); + + it('always returns the sole command', () => { + const selector = selectorFor([['ping', 1.0]]); + for (let i = 0; i < 100; i++) assert.equal(selector.select().name, 'PING'); + }); + + it('never returns a zero-weight command', () => { + const counts = distribution(selectorFor([['get', 1.0], ['set', 0]]), 5000); + assert.equal(counts.get('SET') ?? 0, 0); + }); + + it('falls back to a command when every weight is zero', () => { + // Guards against dividing by a zero total and returning undefined. + const selector = selectorFor([['get', 0], ['set', 0]]); + for (let i = 0; i < 100; i++) assert.ok(['GET', 'SET'].includes(selector.select().name)); + }); + + it('rejects an empty command list', () => { + assert.throws(() => new CommandSelector([]), /at least one command/); + }); +}); diff --git a/node/test/unit/configLoader.test.ts b/node/test/unit/configLoader.test.ts new file mode 100644 index 0000000..13971eb --- /dev/null +++ b/node/test/unit/configLoader.test.ts @@ -0,0 +1,269 @@ +import assert from 'node:assert/strict'; +import { mkdtempSync, writeFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { describe, it } from 'node:test'; + +import { DEFAULT_DATA_SIZE_BYTES } from '../../src/config/commandConfig.js'; +import { + DEFAULT_KEY_PREFIX, + DEFAULT_KEY_SIZE_BYTES, +} from '../../src/config/keyspaceConfig.js'; +import { ConfigError, ConfigLoader } from '../../src/config/loader.js'; + +function writeTempJson(name: string, data: unknown): string { + const dir = mkdtempSync(join(tmpdir(), 'resp-bench-node-')); + const path = join(dir, name); + writeFileSync(path, JSON.stringify(data), 'utf8'); + return path; +} + +const MINIMAL_PHASE = { + id: 'STEADY', + connections: 2, + completion: { type: 'requests', requests: 100 }, + keyspace: { keys_count: 50 }, + commands: [{ command: 'get', weight: 1.0 }], +}; + +describe('ConfigLoader โ€” driver config', () => { + it('parses every schema field', () => { + const config = ConfigLoader.parseDriverConfig({ + schema_version: '1.0', + description: 'test driver', + driver_id: 'ioredis', + mode: 'cluster', + command_timeout_ms: 10_000, + tls: { enabled: true, ca_path: '/tmp/ca.pem' }, + auth: { username: 'u', password: 'p' }, + specific_driver_config: { secondary_driver_id: 'other' }, + }); + + assert.equal(config.schemaVersion, '1.0'); + assert.equal(config.description, 'test driver'); + assert.equal(config.driverId, 'ioredis'); + assert.equal(config.mode, 'cluster'); + assert.equal(config.commandTimeoutMs, 10_000); + assert.equal(config.tlsEnabled(), true); + assert.equal(config.hasAuth(), true); + assert.equal(config.isCluster(), true); + assert.equal(config.isStandalone(), false); + assert.equal(config.secondaryDriverId(), 'other'); + }); + + it('applies defaults for the optional fields', () => { + const config = ConfigLoader.parseDriverConfig({ driver_id: 'ioredis' }); + assert.equal(config.schemaVersion, '1.0'); + assert.equal(config.mode, 'standalone'); + assert.equal(config.commandTimeoutMs, null); + assert.equal(config.tlsEnabled(), false); + assert.equal(config.hasAuth(), false); + assert.deepEqual(config.specificDriverConfig, {}); + assert.equal(config.secondaryDriverId(), null); + }); + + it('treats an empty username and password as no auth', () => { + const config = ConfigLoader.parseDriverConfig({ + driver_id: 'ioredis', + auth: { username: '', password: '' }, + }); + assert.equal(config.hasAuth(), false); + }); + + it('rejects a missing driver_id', () => { + assert.throws(() => ConfigLoader.parseDriverConfig({ mode: 'standalone' }), ConfigError); + }); + + it('rejects an unknown mode', () => { + assert.throws( + () => ConfigLoader.parseDriverConfig({ driver_id: 'ioredis', mode: 'galaxy' }), + /must be standalone, cluster or sentinel/, + ); + }); + + it('loads the real repo driver configs', () => { + const path = writeTempJson('driver.json', { + schema_version: '1.0', + description: 'valkey-glide-node client - default configuration', + driver_id: 'valkey-glide-node', + mode: 'standalone', + specific_driver_config: {}, + }); + assert.equal(ConfigLoader.loadDriverConfig(path).driverId, 'valkey-glide-node'); + }); + + it('reports a readable error for a missing file', () => { + assert.throws(() => ConfigLoader.loadDriverConfig('/nope/missing.json'), /cannot read driver config/); + }); + + it('reports a readable error for malformed JSON', () => { + const dir = mkdtempSync(join(tmpdir(), 'resp-bench-node-')); + const path = join(dir, 'bad.json'); + writeFileSync(path, '{ not json', 'utf8'); + assert.throws(() => ConfigLoader.loadDriverConfig(path), /is not valid JSON/); + }); +}); + +describe('ConfigLoader โ€” workload config', () => { + it('parses a full workload', () => { + const workload = ConfigLoader.parseWorkloadConfig({ + schema_version: '1.0', + benchmark_profile: { name: 'Reference', description: 'd', version: '1.0.0' }, + phases: [ + { + id: 'WARMUP', + description: 'populate', + connections: 1, + cps_limit: -1, + rps_limit: -1, + pipeline_depth: 4, + warmup_requests: 3, + completion: { type: 'requests', requests: 1_000_000 }, + keyspace: { + keys_count: 1_000_000, + key_size_bytes: 16, + key_prefix: 'bench:', + generation_alg: 'uniform_rand', + seed: 12345, + }, + commands: [ + { command: 'get', weight: 0.8 }, + { command: 'set', weight: 0.2, data_size_bytes: 512 }, + ], + }, + ], + }); + + assert.equal(workload.name(), 'Reference'); + assert.equal(workload.phases.length, 1); + const phase = workload.phases[0]!; + assert.equal(phase.id, 'WARMUP'); + assert.equal(phase.connections, 1); + assert.equal(phase.pipelineDepth, 4); + assert.equal(phase.effectivePipelineDepth(), 4); + assert.equal(phase.warmupRequests, 3); + assert.equal(phase.hasCpsLimit(), false); + assert.equal(phase.hasRpsLimit(), false); + assert.equal(phase.completion.isRequestBased(), true); + assert.equal(phase.completion.totalRequests(), 1_000_000); + assert.equal(phase.keyspace.isUniformRand(), true); + assert.equal(phase.keyspace.seedValue(), 12345); + assert.equal(phase.commands[1]!.dataSizeBytes, 512); + }); + + it('applies the cross-engine defaults', () => { + const workload = ConfigLoader.parseWorkloadConfig({ phases: [MINIMAL_PHASE] }); + const phase = workload.phases[0]!; + assert.equal(phase.cpsLimit, -1); + assert.equal(phase.rpsLimit, -1); + assert.equal(phase.pipelineDepth, 1); + assert.equal(phase.warmupRequests, 1); + assert.equal(phase.description, null); + assert.equal(phase.keyspace.keySizeBytes, DEFAULT_KEY_SIZE_BYTES); + assert.equal(phase.keyspace.keyPrefix, DEFAULT_KEY_PREFIX); + assert.equal(phase.keyspace.isSequentialInt(), true); + assert.equal(phase.keyspace.seedValue(), 0); + assert.equal(phase.commands[0]!.dataSizeBytes, DEFAULT_DATA_SIZE_BYTES); + assert.equal(phase.commands[0]!.weight, 1.0); + assert.equal(workload.name(), 'unnamed'); + }); + + it('treats an explicit null as absent, not as null', () => { + // The Ruby/Python engines coerce nulls to defaults; a null leaking through + // would surface as NaN padding or a null prefix deep in the worker loop. + const workload = ConfigLoader.parseWorkloadConfig({ + phases: [ + { + ...MINIMAL_PHASE, + cps_limit: null, + rps_limit: null, + pipeline_depth: null, + warmup_requests: null, + keyspace: { keys_count: 50, key_size_bytes: null, key_prefix: null, generation_alg: null }, + commands: [{ command: 'set', weight: null, data_size_bytes: null }], + }, + ], + }); + const phase = workload.phases[0]!; + assert.equal(phase.cpsLimit, -1); + assert.equal(phase.pipelineDepth, 1); + assert.equal(phase.warmupRequests, 1); + assert.equal(phase.keyspace.keySizeBytes, DEFAULT_KEY_SIZE_BYTES); + assert.equal(phase.keyspace.keyPrefix, DEFAULT_KEY_PREFIX); + assert.equal(phase.keyspace.generationAlg, 'sequential_int'); + assert.equal(phase.commands[0]!.weight, 1.0); + assert.equal(phase.commands[0]!.dataSizeBytes, DEFAULT_DATA_SIZE_BYTES); + }); + + it('lower-cases command names', () => { + const workload = ConfigLoader.parseWorkloadConfig({ + phases: [{ ...MINIMAL_PHASE, commands: [{ command: 'GET', weight: 1 }] }], + }); + assert.equal(workload.phases[0]!.commands[0]!.command, 'get'); + }); + + it('recognises limits when set', () => { + const workload = ConfigLoader.parseWorkloadConfig({ + phases: [{ ...MINIMAL_PHASE, cps_limit: 10, rps_limit: 500 }], + }); + const phase = workload.phases[0]!; + assert.equal(phase.hasCpsLimit(), true); + assert.equal(phase.hasRpsLimit(), true); + }); + + it('rejects a workload with no phases', () => { + assert.throws(() => ConfigLoader.parseWorkloadConfig({ phases: [] }), /non-empty "phases"/); + assert.throws(() => ConfigLoader.parseWorkloadConfig({}), /non-empty "phases"/); + }); + + it('rejects a phase with no commands', () => { + assert.throws( + () => ConfigLoader.parseWorkloadConfig({ phases: [{ ...MINIMAL_PHASE, commands: [] }] }), + /non-empty "commands"/, + ); + }); + + it('rejects non-positive connections', () => { + assert.throws( + () => ConfigLoader.parseWorkloadConfig({ phases: [{ ...MINIMAL_PHASE, connections: 0 }] }), + /"connections" must be positive/, + ); + }); + + it('rejects a completion type that carries no target', () => { + assert.throws( + () => + ConfigLoader.parseWorkloadConfig({ + phases: [{ ...MINIMAL_PHASE, completion: { type: 'duration' } }], + }), + /requires a positive "seconds"/, + ); + assert.throws( + () => + ConfigLoader.parseWorkloadConfig({ + phases: [{ ...MINIMAL_PHASE, completion: { type: 'requests' } }], + }), + /requires a positive "requests"/, + ); + }); + + it('rejects an unknown generation_alg', () => { + assert.throws( + () => + ConfigLoader.parseWorkloadConfig({ + phases: [{ ...MINIMAL_PHASE, keyspace: { keys_count: 10, generation_alg: 'zipf' } }], + }), + /sequential_int or uniform_rand/, + ); + }); + + it('rejects a weight outside 0..1, as Java does', () => { + assert.throws( + () => + ConfigLoader.parseWorkloadConfig({ + phases: [{ ...MINIMAL_PHASE, commands: [{ command: 'get', weight: 5 }] }], + }), + /"weight" must be between 0 and 1/, + ); + }); +}); diff --git a/node/test/unit/factory.test.ts b/node/test/unit/factory.test.ts new file mode 100644 index 0000000..2ab2835 --- /dev/null +++ b/node/test/unit/factory.test.ts @@ -0,0 +1,115 @@ +import assert from 'node:assert/strict'; +import { describe, it } from 'node:test'; + +import { BenchmarkClientFactory } from '../../src/client/factory.js'; +import { packageVersion } from '../../src/client/driverVersion.js'; +import { CommandFactory } from '../../src/command/factory.js'; +import { CommandConfig } from '../../src/config/commandConfig.js'; +import { SetCommand } from '../../src/command/impl/setCommand.js'; + +describe('BenchmarkClientFactory', () => { + it('registers the node driver ids', () => { + assert.deepEqual(BenchmarkClientFactory.supportedDrivers(), [ + 'valkey-glide-node', + 'ioredis', + 'iovalkey', + 'recording', + ]); + }); + + it('does not register a bare valkey-glide', () => { + // scripts/run_benchmark_matrix.py's DRIVER_ENGINE_MAP is global and already + // maps "valkey-glide" to the Java engine. Claiming it here would silently + // reroute Java's glide runs to Node. + assert.equal(BenchmarkClientFactory.supportedDrivers().includes('valkey-glide'), false); + }); + + it('describes every driver for --info', () => { + for (const { driverId, description } of BenchmarkClientFactory.describe()) { + assert.ok(driverId.length > 0); + assert.ok(description.length > 0, `${driverId} has no description`); + } + }); + + it('rejects an unknown driver with the supported list', async () => { + await assert.rejects( + () => BenchmarkClientFactory.create('memcached'), + /Unknown driver: memcached\. Supported: valkey-glide-node, ioredis, iovalkey, recording/, + ); + }); + + it('is case-insensitive on driver_id', async () => { + const client = await BenchmarkClientFactory.create('IoRedis'); + assert.ok(client); + }); + + it('loads the recording driver without a server', async () => { + const client = await BenchmarkClientFactory.create('recording'); + assert.equal(client.driverVersion(), '1.0.0'); + }); +}); + +describe('CommandFactory', () => { + it('supports the cross-engine command set', () => { + assert.deepEqual(CommandFactory.supportedCommands(), ['get', 'set', 'ping']); + }); + + it('builds each command with its weight and name', () => { + const get = CommandFactory.create(new CommandConfig({ command: 'get', weight: 0.8 })); + assert.equal(get.name, 'GET'); + assert.equal(get.weight, 0.8); + assert.equal(get.usesKey, true); + + const ping = CommandFactory.create(new CommandConfig({ command: 'ping' })); + assert.equal(ping.name, 'PING'); + // PING must not consume a generated key: Java's PingCommand ignores the key + // generator, so advancing it here would shift the shared key sequence. + assert.equal(ping.usesKey, false); + }); + + it('accepts an upper-case command name', () => { + assert.equal(CommandFactory.create(new CommandConfig({ command: 'SET' })).name, 'SET'); + }); + + it('rejects an unknown command', () => { + assert.throws( + () => CommandFactory.create(new CommandConfig({ command: 'incr' })), + /Unknown command: incr\. Supported: get, set, ping/, + ); + }); + + it('createAll preserves order', () => { + const commands = CommandFactory.createAll([ + new CommandConfig({ command: 'get', weight: 0.8 }), + new CommandConfig({ command: 'set', weight: 0.2 }), + ]); + assert.deepEqual( + commands.map((c) => c.name), + ['GET', 'SET'], + ); + }); +}); + +describe('SetCommand payload', () => { + it('generates exactly data_size_bytes using the cross-engine pattern', () => { + // Ruby and Python use the same repeated "0123456789ABCDEF" filler. + for (const size of [1, 16, 32, 256, 512, 1000]) { + assert.equal(SetCommand.generateValue(size).length, size); + } + assert.equal(SetCommand.generateValue(20).toString('latin1'), '0123456789ABCDEF0123'); + }); +}); + +describe('packageVersion', () => { + it('reads a version despite a restricted exports map', () => { + // @valkey/valkey-glide does not export ./package.json, so a plain + // require('/package.json') throws ERR_PACKAGE_PATH_NOT_EXPORTED. + assert.match(packageVersion('@valkey/valkey-glide'), /^\d+\.\d+\.\d+/); + assert.match(packageVersion('ioredis'), /^\d+\.\d+\.\d+/); + assert.match(packageVersion('iovalkey'), /^\d+\.\d+\.\d+/); + }); + + it('returns "unknown" for a package that is not installed', () => { + assert.equal(packageVersion('definitely-not-installed-xyz'), 'unknown'); + }); +}); diff --git a/node/test/unit/hdrHistogram.test.ts b/node/test/unit/hdrHistogram.test.ts new file mode 100644 index 0000000..f39bef9 --- /dev/null +++ b/node/test/unit/hdrHistogram.test.ts @@ -0,0 +1,63 @@ +import assert from 'node:assert/strict'; +import { describe, it } from 'node:test'; + +import { + decodeBase64, + encodeBase64, + HIGHEST_TRACKABLE_VALUE, + LOWEST_TRACKABLE_VALUE, + newHistogram, + SIGNIFICANT_FIGURES, +} from '../../src/metrics/hdrHistogram.js'; + +describe('HDR histogram', () => { + it('uses the cross-engine range and precision', () => { + // Java SynchronizedHistogram(600_000_000, 3), C# LongConcurrentHistogram(1, + // 600_000_000, 3), Ruby HDRHistogram.new(1, 600_000_000, 3). + assert.equal(LOWEST_TRACKABLE_VALUE, 1); + assert.equal(HIGHEST_TRACKABLE_VALUE, 600_000_000); + assert.equal(SIGNIFICANT_FIGURES, 3); + const histogram = newHistogram(); + assert.equal(histogram.highestTrackableValue, 600_000_000); + assert.equal(histogram.numberOfSignificantValueDigits, 3); + }); + + it('emits a HIST-prefixed payload, not a double-encoded one', () => { + // Java's writer base64s the compressed bytes, which always start with the + // V2 cookie 0x1c849314 -> "HIST". A payload that does not start with HIST + // means it was base64-encoded twice and Java/Ruby cannot decode it. + const histogram = newHistogram(); + histogram.recordValue(1234); + const payload = encodeBase64(histogram); + assert.ok(payload.startsWith('HIST'), `payload should start with HIST, got ${payload.slice(0, 12)}`); + assert.doesNotMatch(payload, /^SElTVA/, 'payload is base64 of "HIST" โ€” encoded twice'); + }); + + it('round-trips percentiles and total count', () => { + const histogram = newHistogram(); + for (let value = 1; value <= 1000; value++) histogram.recordValue(value); + + const decoded = decodeBase64(encodeBase64(histogram)); + assert.equal(decoded.totalCount, histogram.totalCount); + for (const percentile of [0, 50, 95, 99, 99.9, 100]) { + assert.equal( + decoded.getValueAtPercentile(percentile), + histogram.getValueAtPercentile(percentile), + `p${percentile} differs after round trip`, + ); + } + }); + + it('encodes an empty histogram without throwing', () => { + // An all-errors command still needs an hdr block in the NDJSON. + const payload = encodeBase64(newHistogram()); + assert.ok(payload.startsWith('HIST')); + assert.equal(decodeBase64(payload).totalCount, 0); + }); + + it('records the top of the range', () => { + const histogram = newHistogram(); + histogram.recordValue(HIGHEST_TRACKABLE_VALUE); + assert.equal(histogram.totalCount, 1); + }); +}); diff --git a/node/test/unit/javaRandom.test.ts b/node/test/unit/javaRandom.test.ts new file mode 100644 index 0000000..e1f7002 --- /dev/null +++ b/node/test/unit/javaRandom.test.ts @@ -0,0 +1,84 @@ +/** + * JavaRandom parity tests. + * + * The LCG is anchored to a well-known java.util.Random value, which proves + * byte-for-byte compatibility with the Java reference without needing a JVM. The + * two sequences below were additionally cross-checked against the Python + * engine's JavaRandom, so a failure here means Node has diverged from both. + */ + +import assert from 'node:assert/strict'; +import { describe, it } from 'node:test'; + +import { JavaRandom, toInt32 } from '../../src/engine/javaRandom.js'; + +describe('JavaRandom', () => { + it('matches the documented java.util.Random(0).nextInt() value', () => { + // java.util.Random(0).nextInt() -- i.e. next(32) as a signed int -- is the + // well-documented value -1155484576. This anchors the LCG to real Java. + assert.equal(toInt32(new JavaRandom(0).next(32)), -1155484576); + }); + + it('reproduces the cross-engine sequence for seed 12345, bound 1000', () => { + const rng = new JavaRandom(12345); + const actual = Array.from({ length: 10 }, () => rng.nextInt(1000)); + assert.deepEqual(actual, [251, 80, 241, 828, 55, 84, 375, 802, 501, 389]); + }); + + it('reproduces the cross-engine sequence for a power-of-two bound', () => { + // Exercises the power-of-two fast path, a separate branch in Java. + const rng = new JavaRandom(12345); + const actual = Array.from({ length: 8 }, () => rng.nextInt(256)); + assert.deepEqual(actual, [92, 131, 238, 234, 213, 9, 83, 31]); + }); + + it('is deterministic for a given seed', () => { + const rngA = new JavaRandom(12345); + const rngB = new JavaRandom(12345); + assert.deepEqual( + Array.from({ length: 10 }, () => rngA.nextInt(1000)), + Array.from({ length: 10 }, () => rngB.nextInt(1000)), + ); + }); + + it('produces different sequences for different seeds', () => { + const a = new JavaRandom(12345); + const b = new JavaRandom(54321); + assert.notDeepEqual( + Array.from({ length: 10 }, () => a.nextInt(1000)), + Array.from({ length: 10 }, () => b.nextInt(1000)), + ); + }); + + it('setSeed resets the stream', () => { + const rng = new JavaRandom(12345); + const first = Array.from({ length: 5 }, () => rng.nextInt(1000)); + rng.setSeed(12345); + const second = Array.from({ length: 5 }, () => rng.nextInt(1000)); + assert.deepEqual(first, second); + }); + + it('rejects a non-positive bound', () => { + const rng = new JavaRandom(12345); + assert.throws(() => rng.nextInt(0), /bound must be a positive integer/); + assert.throws(() => rng.nextInt(-1), /bound must be a positive integer/); + }); + + it('stays within the bound', () => { + const rng = new JavaRandom(12345); + for (let i = 0; i < 1000; i++) { + const value = rng.nextInt(100); + assert.ok(value >= 0 && value < 100, `${value} out of range`); + } + }); + + it('stays within power-of-two bounds', () => { + const rng = new JavaRandom(12345); + for (const bound of [2, 4, 8, 16, 256, 1024]) { + for (let i = 0; i < 200; i++) { + const value = rng.nextInt(bound); + assert.ok(value >= 0 && value < bound, `${value} out of range for ${bound}`); + } + } + }); +}); diff --git a/node/test/unit/keyGenerator.test.ts b/node/test/unit/keyGenerator.test.ts new file mode 100644 index 0000000..2f6195e --- /dev/null +++ b/node/test/unit/keyGenerator.test.ts @@ -0,0 +1,133 @@ +import assert from 'node:assert/strict'; +import { describe, it } from 'node:test'; + +import { KeyspaceConfig } from '../../src/config/keyspaceConfig.js'; +import { Counter, KeyGenerator } from '../../src/engine/keyGenerator.js'; + +describe('KeyGenerator', () => { + describe('sequential_int', () => { + it('emits 0..N-1 then wraps', () => { + const gen = KeyGenerator.create( + new KeyspaceConfig({ keysCount: 3, keyPrefix: 'test:', keySizeBytes: 6 }), + ); + assert.deepEqual( + Array.from({ length: 4 }, () => gen.nextKey()), + ['test:0', 'test:1', 'test:2', 'test:0'], + ); + }); + + it('shares one counter across workers, as Java does', () => { + // The whole point of the shared counter: several connections collectively + // populate the keyspace instead of each replaying 0, 1, 2, ... + const config = new KeyspaceConfig({ keysCount: 100, keyPrefix: 'k:', keySizeBytes: 4 }); + const counter = new Counter(); + const workerA = KeyGenerator.createWithSeed(config, 0, counter); + const workerB = KeyGenerator.createWithSeed(config, 1, counter); + assert.deepEqual( + [workerA.nextKey(), workerB.nextKey(), workerA.nextKey(), workerB.nextKey()], + ['k:00', 'k:01', 'k:02', 'k:03'], + ); + }); + + it('gives each generator its own counter when none is shared', () => { + const config = new KeyspaceConfig({ keysCount: 100, keyPrefix: 'k:', keySizeBytes: 4 }); + assert.equal(KeyGenerator.create(config).nextKey(), 'k:00'); + assert.equal(KeyGenerator.create(config).nextKey(), 'k:00'); + }); + }); + + describe('uniform_rand', () => { + it('is reproducible for the same seed', () => { + const config = new KeyspaceConfig({ + keysCount: 1000, + keyPrefix: 'test:', + generationAlg: 'uniform_rand', + seed: 12345, + }); + const a = KeyGenerator.create(config); + const b = KeyGenerator.create(config); + for (let i = 0; i < 100; i++) assert.equal(a.nextKey(), b.nextKey()); + }); + + it('derives a per-worker seed of base + index, as Java does', () => { + const config = new KeyspaceConfig({ + keysCount: 1000, + keyPrefix: 'test:', + generationAlg: 'uniform_rand', + seed: 12345, + }); + const worker0 = KeyGenerator.createWithSeed(config, 12345); + const worker1 = KeyGenerator.createWithSeed(config, 12346); + assert.notEqual(worker0.nextKey(), worker1.nextKey()); + }); + + it('matches the JavaRandom sequence for seed 12345', () => { + // Anchored to the same values as javaRandom.test.ts, so a key-formatting + // change cannot silently break cross-engine key parity. + const gen = KeyGenerator.create( + new KeyspaceConfig({ + keysCount: 1000, + keyPrefix: 'bench:', + keySizeBytes: 16, + generationAlg: 'uniform_rand', + seed: 12345, + }), + ); + assert.deepEqual( + Array.from({ length: 4 }, () => gen.nextKey()), + ['bench:0000000251', 'bench:0000000080', 'bench:0000000241', 'bench:0000000828'], + ); + }); + + it('reset restores the stream', () => { + const gen = KeyGenerator.create( + new KeyspaceConfig({ + keysCount: 1000, + keyPrefix: 'test:', + generationAlg: 'uniform_rand', + seed: 999, + }), + ); + const first = Array.from({ length: 5 }, () => gen.nextKey()); + gen.reset(); + assert.deepEqual( + Array.from({ length: 5 }, () => gen.nextKey()), + first, + ); + }); + }); + + describe('key formatting', () => { + it('zero-pads to key_size_bytes minus the prefix, as Java does', () => { + // Reference configs use key_prefix "bench:" (6) and key_size_bytes 16, + // so the numeric part is 10 digits: bench:0000000042 + const gen = KeyGenerator.create( + new KeyspaceConfig({ keysCount: 1_000_000, keyPrefix: 'bench:', keySizeBytes: 16 }), + ); + const key = gen.nextKey(); + assert.equal(key, 'bench:0000000000'); + assert.equal(key.length, 16); + }); + + it('keeps at least one digit when the prefix fills key_size_bytes', () => { + const gen = KeyGenerator.create( + new KeyspaceConfig({ keysCount: 10, keyPrefix: 'averylongprefix:', keySizeBytes: 4 }), + ); + assert.equal(gen.nextKey(), 'averylongprefix:0'); + }); + + it('does not truncate an index wider than the padding', () => { + const gen = KeyGenerator.create( + new KeyspaceConfig({ keysCount: 1000, keyPrefix: 'k:', keySizeBytes: 4 }), + ); + const keys = Array.from({ length: 101 }, () => gen.nextKey()); + assert.equal(keys[0], 'k:00'); + assert.equal(keys[100], 'k:100'); + }); + + it('applies the documented defaults', () => { + const gen = KeyGenerator.create(new KeyspaceConfig({ keysCount: 10 })); + assert.equal(gen.nextKey(), 'bench:0000000000'); + }); + }); +}); diff --git a/node/test/unit/ndjsonWriter.test.ts b/node/test/unit/ndjsonWriter.test.ts new file mode 100644 index 0000000..7bb3e22 --- /dev/null +++ b/node/test/unit/ndjsonWriter.test.ts @@ -0,0 +1,193 @@ +import assert from 'node:assert/strict'; +import { mkdtempSync, readFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { describe, it } from 'node:test'; + +import { MetricsCollector } from '../../src/metrics/collector.js'; +import { decodeBase64 } from '../../src/metrics/hdrHistogram.js'; +import { NdjsonWriter } from '../../src/metrics/ndjsonWriter.js'; + +function tempPath(name = 'metrics.ndjson'): string { + return join(mkdtempSync(join(tmpdir(), 'resp-bench-node-')), name); +} + +function collectorWith(samples: Array<{ name: string; latency: number; ok?: boolean }>) { + const collector = new MetricsCollector(); + collector.start(); + for (const { name, latency, ok = true } of samples) { + collector.record({ commandName: name, latencyMicros: latency, success: ok }); + } + collector.stop(); + return collector; +} + +function readRecords(path: string): Array> { + return readFileSync(path, 'utf8') + .trimEnd() + .split('\n') + .map((line) => JSON.parse(line)); +} + +describe('NdjsonWriter', () => { + it('writes the documented record shape', () => { + const path = tempPath(); + const writer = new NdjsonWriter(path); + writer.setMetadata({ + commitId: 'abc123', + driverId: 'ioredis', + primaryDriverVersion: '5.11.1', + }); + writer.writePhaseResults({ + phaseId: 'STEADY', + status: 'COMPLETED', + connections: 4, + collector: collectorWith([ + { name: 'GET', latency: 100 }, + { name: 'GET', latency: 200 }, + { name: 'SET', latency: 300 }, + ]), + }); + + const [record] = readRecords(path); + assert.ok(record); + + assert.deepEqual(Object.keys(record).sort(), ['metadata', 'metrics', 'phase', 'totals']); + assert.equal(record['metadata'].commit_id, 'abc123'); + assert.equal(record['metadata'].driver_id, 'ioredis'); + assert.equal(record['metadata'].primary_driver_version, '5.11.1'); + assert.ok(record['metadata'].timestamp); + // Absent secondary driver fields must be omitted, not null. + assert.equal('secondary_driver_id' in record['metadata'], false); + + assert.deepEqual(Object.keys(record['phase']).sort(), [ + 'connections', + 'duration_ms', + 'finish_timestamp', + 'id', + 'start_timestamp', + 'status', + ]); + assert.equal(record['phase'].id, 'STEADY'); + assert.equal(record['phase'].status, 'COMPLETED'); + assert.equal(record['phase'].connections, 4); + assert.match(record['phase'].start_timestamp, /^\d{4}-\d{2}-\d{2}T.*Z$/); + + assert.deepEqual(record['totals'], { requests: 3, errors: 0 }); + + assert.deepEqual(Object.keys(record['metrics']).sort(), ['GET', 'SET']); + const get = record['metrics'].GET; + assert.equal(get.requests, 2); + assert.equal(get.errors, 0); + assert.equal(get.latency.unit, 'us'); + assert.equal(get.latency.count, 2); + assert.deepEqual(Object.keys(get.latency.summary).sort(), [ + 'max', + 'min', + 'p50', + 'p95', + 'p99', + 'p999', + ]); + assert.equal(get.latency.hdr.format, 'hdr'); + assert.equal(get.latency.hdr.sigfig, 3); + assert.ok(get.latency.hdr.payload_b64.startsWith('HIST')); + }); + + it('writes one compact line per phase and appends across phases', () => { + const path = tempPath(); + const writer = new NdjsonWriter(path); + writer.setMetadata({ driverId: 'ioredis' }); + for (const phaseId of ['WARMUP', 'STEADY']) { + writer.writePhaseResults({ + phaseId, + status: 'COMPLETED', + connections: 1, + collector: collectorWith([{ name: 'GET', latency: 10 }]), + }); + } + + const raw = readFileSync(path, 'utf8'); + assert.equal(raw.endsWith('\n'), true); + const lines = raw.trimEnd().split('\n'); + assert.equal(lines.length, 2); + // NDJSON requires no embedded newlines -- no pretty printing. + for (const line of lines) assert.doesNotMatch(line, /\n/); + assert.deepEqual( + lines.map((line) => JSON.parse(line).phase.id), + ['WARMUP', 'STEADY'], + ); + }); + + it('creates the parent directory when it does not exist', () => { + const path = join(mkdtempSync(join(tmpdir(), 'resp-bench-node-')), 'nested', 'deep', 'm.ndjson'); + const writer = new NdjsonWriter(path); + writer.setMetadata({ driverId: 'ioredis' }); + writer.writePhaseResults({ + phaseId: 'P', + status: 'COMPLETED', + connections: 1, + collector: collectorWith([{ name: 'GET', latency: 10 }]), + }); + assert.equal(readRecords(path).length, 1); + }); + + it('emits an hdr block for a command that only ever errored', () => { + // Matches Java: the histogram is created eagerly, so analysis tooling can + // always read metrics..latency.hdr without a null check. + const path = tempPath(); + const writer = new NdjsonWriter(path); + writer.setMetadata({ driverId: 'ioredis' }); + writer.writePhaseResults({ + phaseId: 'ERRORS', + status: 'COMPLETED', + connections: 1, + collector: collectorWith([ + { name: 'GET', latency: 50, ok: false }, + { name: 'GET', latency: 60, ok: false }, + ]), + }); + + const [record] = readRecords(path); + const get = record!['metrics'].GET; + assert.equal(get.requests, 2); + assert.equal(get.errors, 2); + assert.equal(get.latency.count, 0); + assert.deepEqual(get.latency.summary, { min: 0, p50: 0, p95: 0, p99: 0, p999: 0, max: 0 }); + assert.ok(get.latency.hdr.payload_b64.startsWith('HIST')); + assert.equal(decodeBase64(get.latency.hdr.payload_b64).totalCount, 0); + assert.deepEqual(record!['totals'], { requests: 2, errors: 2 }); + }); + + it('omits the metadata block entirely when nothing identifies the run', () => { + const path = tempPath(); + const writer = new NdjsonWriter(path); + writer.writePhaseResults({ + phaseId: 'P', + status: 'COMPLETED', + connections: 1, + collector: collectorWith([{ name: 'GET', latency: 10 }]), + }); + assert.equal('metadata' in readRecords(path)[0]!, false); + }); + + it('carries the secondary driver fields when present', () => { + const path = tempPath(); + const writer = new NdjsonWriter(path); + writer.setMetadata({ + driverId: 'composite', + primaryDriverVersion: '1.0.0', + secondaryDriverId: 'ioredis', + secondaryDriverVersion: '5.11.1', + }); + writer.writePhaseResults({ + phaseId: 'P', + status: 'COMPLETED', + connections: 1, + collector: collectorWith([{ name: 'GET', latency: 10 }]), + }); + const metadata = readRecords(path)[0]!['metadata']; + assert.equal(metadata.secondary_driver_id, 'ioredis'); + assert.equal(metadata.secondary_driver_version, '5.11.1'); + }); +}); diff --git a/node/test/unit/rateLimiter.test.ts b/node/test/unit/rateLimiter.test.ts new file mode 100644 index 0000000..e39f96a --- /dev/null +++ b/node/test/unit/rateLimiter.test.ts @@ -0,0 +1,82 @@ +import assert from 'node:assert/strict'; +import { describe, it } from 'node:test'; + +import { RateLimiter } from '../../src/engine/rateLimiter.js'; + +describe('RateLimiter', () => { + it('returns null for an unlimited rate', () => { + assert.equal(RateLimiter.create(0), null); + assert.equal(RateLimiter.create(-1), null); + }); + + it('returns a limiter for a positive rate', () => { + const limiter = RateLimiter.create(100); + assert.ok(limiter); + assert.equal(limiter.ratePerSecond, 100); + }); + + it('achieves the target rate within 5%', async () => { + // docs/ADDING_LANGUAGE.md's stated tolerance for the rate limiter. + const rate = 200; + const requests = 100; + const limiter = RateLimiter.create(rate)!; + + const start = process.hrtime.bigint(); + for (let i = 0; i < requests; i++) await limiter.acquire(); + const elapsedSeconds = Number(process.hrtime.bigint() - start) / 1e9; + + // The first acquire is free (the bucket starts open), so the limiter only + // paces the remaining requests. + const expectedSeconds = (requests - 1) / rate; + assert.ok( + elapsedSeconds >= expectedSeconds * 0.95, + `finished too fast: ${elapsedSeconds.toFixed(3)}s < ${(expectedSeconds * 0.95).toFixed(3)}s`, + ); + assert.ok( + elapsedSeconds <= expectedSeconds * 1.3, + `finished too slow: ${elapsedSeconds.toFixed(3)}s > ${(expectedSeconds * 1.3).toFixed(3)}s`, + ); + }); + + it('paces a rate whose interval is below setTimeout resolution', async () => { + // 5000/s is a 200us interval -- well under setTimeout's ~1ms floor, so this + // only passes if sub-millisecond waits yield via setImmediate instead. + const rate = 5000; + const requests = 500; + const limiter = RateLimiter.create(rate)!; + + const start = process.hrtime.bigint(); + for (let i = 0; i < requests; i++) await limiter.acquire(); + const elapsedSeconds = Number(process.hrtime.bigint() - start) / 1e9; + + const expectedSeconds = (requests - 1) / rate; + assert.ok( + elapsedSeconds <= expectedSeconds * 2, + `sub-ms pacing overshot badly: ${elapsedSeconds.toFixed(3)}s vs ${expectedSeconds.toFixed(3)}s ` + + '(a clamped setTimeout would take ~10x this)', + ); + }); + + it('shares one budget across concurrent callers', async () => { + const rate = 200; + const perWorker = 25; + const workers = 4; + const limiter = RateLimiter.create(rate)!; + + const start = process.hrtime.bigint(); + await Promise.all( + Array.from({ length: workers }, async () => { + for (let i = 0; i < perWorker; i++) await limiter.acquire(); + }), + ); + const elapsedSeconds = Number(process.hrtime.bigint() - start) / 1e9; + + // The limit is global: 100 requests at 200/s takes ~0.5s regardless of how + // many workers issue them. + const expectedSeconds = (workers * perWorker - 1) / rate; + assert.ok( + elapsedSeconds >= expectedSeconds * 0.95, + `concurrent callers bypassed the limit: ${elapsedSeconds.toFixed(3)}s`, + ); + }); +}); diff --git a/node/test/unit/recordingClient.test.ts b/node/test/unit/recordingClient.test.ts new file mode 100644 index 0000000..19af4f6 --- /dev/null +++ b/node/test/unit/recordingClient.test.ts @@ -0,0 +1,101 @@ +import assert from 'node:assert/strict'; +import { describe, it } from 'node:test'; + +import { RecordingClient } from '../../src/client/impl/recordingClient.js'; +import { ConfigLoader } from '../../src/config/loader.js'; + +function config(specific: Record = {}) { + return ConfigLoader.parseDriverConfig({ + driver_id: 'recording', + mode: 'standalone', + specific_driver_config: specific, + }); +} + +async function connected(specific: Record = {}): Promise { + const client = new RecordingClient(); + await client.connect('localhost', 6379, config(specific)); + return client; +} + +describe('RecordingClient', () => { + it('records the operations it was asked to perform', async () => { + const client = await connected(); + await client.ping(); + await client.set('bench:0000000001', Buffer.from('abc', 'latin1')); + await client.get('bench:0000000001'); + await client.close(); + + assert.deepEqual( + client.operations.map((op) => op.command), + ['CONNECT', 'PING', 'SET', 'GET', 'CLOSE'], + ); + // The keys the engine passed through are observable, so a key-generation + // regression shows up as a wrong key here rather than as silent drift. + assert.deepEqual( + client.operations.filter((op) => op.key !== null).map((op) => op.key), + ['bench:0000000001', 'bench:0000000001'], + ); + assert.ok(client.operations.every((op) => op.success)); + }); + + it('behaves like a key-value store for GET after SET', async () => { + const client = await connected(); + assert.equal((await client.get('missing')).value, null); + await client.set('k', Buffer.from('hello', 'latin1')); + assert.equal((await client.get('k')).value, 'hello'); + }); + + it('returns the cross-engine success values', async () => { + const client = await connected(); + assert.equal((await client.ping()).value, 'PONG'); + assert.equal((await client.set('k', Buffer.alloc(4))).value, 'OK'); + }); + + it('injects errors at error_rate 1.0 with the configured message', async () => { + const client = await connected({ error_rate: 1.0, error_message: 'boom' }); + for (const result of [ + await client.ping(), + await client.get('k'), + await client.set('k', Buffer.alloc(1)), + ]) { + assert.equal(result.error?.message, 'boom'); + assert.equal(result.value, null); + } + assert.ok(client.operations.slice(1).every((op) => !op.success)); + }); + + it('never injects errors while in warmup mode', async () => { + // Mirrors Java's setWarmupMode: the engine's warmup fail-fast must not be + // tripped by errors the workload deliberately injects. + const client = await connected({ error_rate: 1.0 }); + client.setWarmupMode(true); + assert.equal((await client.ping()).error, undefined); + client.setWarmupMode(false); + assert.notEqual((await client.ping()).error, undefined); + }); + + it('does not fail a SET when its own error is injected mid-store', async () => { + // A failed SET must not store, or a later GET would report data the server + // never accepted. + const client = await connected({ error_rate: 1.0 }); + await client.set('k', Buffer.from('nope', 'latin1')); + client.setWarmupMode(true); + assert.equal((await client.get('k')).value, null); + }); + + it('applies a configured operation delay', async () => { + const client = await connected({ operation_delay_micros: 5000 }); + const result = await client.ping(); + assert.ok(result.latencyMicros >= 3000, `latency ${result.latencyMicros}us too low for a 5ms delay`); + }); + + it('reports zero-ish latency with no configured delay', async () => { + const client = await connected(); + assert.ok((await client.ping()).latencyMicros < 5000); + }); + + it('reports a fixed driver version', async () => { + assert.equal(new RecordingClient().driverVersion(), '1.0.0'); + }); +}); diff --git a/node/tsconfig.json b/node/tsconfig.json new file mode 100644 index 0000000..8f1e804 --- /dev/null +++ b/node/tsconfig.json @@ -0,0 +1,23 @@ +{ + "compilerOptions": { + "target": "ES2022", + "lib": ["ES2022"], + "module": "NodeNext", + "moduleResolution": "NodeNext", + "rootDir": ".", + "outDir": "dist", + "sourceMap": true, + "declaration": false, + "incremental": true, + "tsBuildInfoFile": "dist/.tsbuildinfo", + + "strict": true, + "noUncheckedIndexedAccess": true, + "noImplicitOverride": true, + "noFallthroughCasesInSwitch": true, + "exactOptionalPropertyTypes": false, + "forceConsistentCasingInFileNames": true, + "skipLibCheck": true + }, + "include": ["src/**/*.ts", "test/**/*.ts"] +} diff --git a/php/.gitignore b/php/.gitignore new file mode 100644 index 0000000..13a49cc --- /dev/null +++ b/php/.gitignore @@ -0,0 +1,5 @@ +/vendor/ +/composer.lock +/.phpunit.cache/ +/output/ +*.ndjson diff --git a/php/README.md b/php/README.md new file mode 100644 index 0000000..bc84f48 --- /dev/null +++ b/php/README.md @@ -0,0 +1,213 @@ +# resp-bench PHP Engine + +PHP implementation of the resp-bench benchmark suite for Redis/Valkey compatible databases. + +## Supported Drivers + +| Driver ID | Package | Description | +|-----------|---------|-------------| +| `valkey-glide-php` | [ext-valkey_glide](https://github.com/valkey-io/valkey-glide-php) | Valkey GLIDE PHP client โ€” Rust-core client exposed as a native PHP extension, with a PHPRedis-compatible API | +| `phpredis` | [ext-redis](https://github.com/phpredis/phpredis) | PHPRedis โ€” the de-facto standard PHP Redis/Valkey client (the incumbent comparison baseline) | +| `recording` | (built-in) | In-memory driver for server-free tests and pipeline validation | + +## Prerequisites + +- PHP 8.2 or 8.3 (the GLIDE PHP extension's tested versions; the engine itself runs on 8.2+) +- `ext-pcntl` (for the multi-process concurrency model; standard on Linux/macOS CLI builds) +- `ext-json` (bundled with PHP) +- Composer (recommended) โ€” or use the bundled minimal autoloader +- For live-server runs: the `valkey_glide` extension (for `valkey-glide-php`) and/or + the `redis` extension (for `phpredis`) installed and enabled + +### Installing the Valkey GLIDE PHP extension + +The `valkey-glide-php` driver requires the `valkey_glide` PHP extension. Install it +via `pie`, PECL, or from source โ€” see the +[upstream instructions](https://github.com/valkey-io/valkey-glide-php#installation-and-setup). +After installing, enable it in `php.ini`: + +```ini +extension=valkey_glide +``` + +Verify: + +```bash +php -m | grep valkey_glide +``` + +The `recording` driver needs neither the extension nor a server, so unit and +integration tests run without either. + +### Installing PHPRedis + +The `phpredis` driver requires the `redis` extension (ext-redis): + +```bash +pecl install redis +# then enable in php.ini: extension=redis +php -m | grep redis +``` + + +## Installation + +```bash +cd php +composer install +``` + +If Composer is unavailable, the CLI falls back to a bundled minimal PSR-4 +autoloader (`php/vendor/autoload.php`), so `make php-run` / `make php-info` still work. + +## Usage + +### Command line + +```bash +# Run a benchmark +php php/bin/resp-bench \ + --server localhost:6379 \ + --driver configs/drivers/example-valkey-glide-php-standalone.json \ + --workload configs/workloads/example-workload.json \ + --metrics output/php.ndjson + +# Show supported drivers and commands (also reports extension/pcntl availability) +php php/bin/resp-bench --info + +# Help +php php/bin/resp-bench --help +``` + +### Using Make (from project root) + +```bash +make php-build # composer install (or bundled autoloader) +make php-test # unit tests +make php-integration-test # integration tests (server-free, recording driver) +make php-run \ + DRIVER=configs/drivers/example-valkey-glide-php-standalone.json \ + WORKLOAD=configs/workloads/example-workload.json \ + SERVER=localhost:6379 +``` + +## Architecture + +The PHP engine follows the same architecture as the Java reference implementation: + +``` +src/ +โ”œโ”€โ”€ Client/ # Client interface + implementations +โ”‚ โ”œโ”€โ”€ BenchmarkClient.php # Abstract base +โ”‚ โ”œโ”€โ”€ TimedResult.php +โ”‚ โ”œโ”€โ”€ Factory.php +โ”‚ โ””โ”€โ”€ Impl/ +โ”‚ โ”œโ”€โ”€ ValkeyGlidePhpClient.php # ext-valkey_glide +โ”‚ โ””โ”€โ”€ RecordingClient.php # in-memory, server-free +โ”œโ”€โ”€ Command/ # GET / SET / PING commands + Factory +โ”œโ”€โ”€ Config/ # JSON config parsing (Driver/Workload/Phase/โ€ฆ) +โ”œโ”€โ”€ Engine/ +โ”‚ โ”œโ”€โ”€ Benchmark.php # Multi-process orchestrator +โ”‚ โ”œโ”€โ”€ CommandSelector.php # Weighted selection +โ”‚ โ”œโ”€โ”€ JavaRandom.php # Java-compatible LCG +โ”‚ โ”œโ”€โ”€ KeyGenerator.php +โ”‚ โ””โ”€โ”€ RateLimiter.php # Leaky bucket +โ””โ”€โ”€ Metrics/ + โ”œโ”€โ”€ Collector.php # Per-command metrics + merge + โ”œโ”€โ”€ HdrHistogram.php # Pure-PHP HdrHistogram + โ”œโ”€โ”€ HdrEncoder.php # V2 compressed (Java-compatible) encoding + โ””โ”€โ”€ NdjsonWriter.php # NDJSON output +``` + +## Concurrency model + +The engine uses a **process-per-connection** model (the PHP analogue of Java's +virtual-thread-per-client and Ruby's thread-per-client): + +- `connections = N` โ†’ fork **N worker processes** (capped at 256). +- Each worker connects to the server **after** forking โ€” a connection is never + inherited across a fork, which is required for correctness with the native + extension. +- Each worker runs one in-flight request at a time against its own client + (the `client == connection` invariant), matching the other engines so results + are comparable. +- Workers stream partial metrics back to the parent over a `stream_socket_pair`; + the parent reconstructs and **merges the HdrHistograms losslessly** (sparse + bucket counts) before writing NDJSON. +- Phase-level `rps_limit` is divided across workers so the aggregate matches the + target rate. + +The chosen model is driven by PHP's runtime: the GLIDE extension is synchronous, +and the standard PHP build is non-thread-safe (NTS), which rules out +`ext-parallel`. `pcntl` multi-processing is the faithful, dependency-free option. + +An **`inline`** mode (`--concurrency inline`) runs all connections sequentially in +a single process. It can't produce true concurrency with a blocking client, but +it exercises the full pipeline and is used for server-free tests. It is selected +automatically for the `recording` driver and when `pcntl` is unavailable. + +## Cross-language parity + +- **JavaRandom**: a port of `java.util.Random`'s 48-bit LCG, verified + byte-identical to Java's canonical `new Random(0).nextInt()` sequence. PHP's + lack of 64-bit integer overflow wraparound is handled with a 24-bit split + multiply. +- **Key generation**: `sequential_int` walks the keyspace; `uniform_rand` seeds + per worker as `seed + workerIndex` for reproducible-yet-distinct sequences. + Key formatting uses `%0Nd` honoring `key_size_bytes`. +- **HdrHistogram**: range `(1, 600_000_000, 3)`; the V2 compressed base64 payload + is byte-compatible with Java's `encodeIntoCompressedByteBuffer()`. +- **Rate limiter**: leaky bucket, constant spacing, no burst. + +## Metrics output + +NDJSON compatible with all other language engines: + +```json +{ + "metadata": {"commit_id": "abc123", "timestamp": "2026-09-14T16:27:11Z", "driver_id": "valkey-glide-php", "primary_driver_version": "1.0.0"}, + "phase": {"id": "STEADY", "status": "COMPLETED", "duration_ms": 60000, "connections": 16}, + "totals": {"requests": 1000000, "errors": 0}, + "metrics": { + "GET": { + "requests": 800000, + "errors": 0, + "latency": {"unit": "us", "count": 800000, "summary": {"p50": 150, "p99": 450}, "hdr": {"format": "hdr", "sigfig": 3, "payload_b64": "..."}} + } + } +} +``` + +## Testing + +```bash +# From php/ with composer-installed PHPUnit: +vendor/bin/phpunit # all tests +vendor/bin/phpunit --testsuite unit # unit only +vendor/bin/phpunit --testsuite integration +``` + +### Test coverage (Java parity) + +| Java Test | PHP Test | Coverage | +|-----------|----------|----------| +| `JavaRandomTest` | `tests/Unit/JavaRandomTest.php` | LCG determinism, Java seed-0 anchor | +| `KeyGeneratorTest` | `tests/Unit/KeyGeneratorTest.php` | Sequential/wrap, uniform-rand, formatting, JavaRandom parity | +| `ConfigLoaderTest` | `tests/Unit/ConfigLoaderTest.php` | Driver/workload parsing, defaults, cluster mode | +| `RateLimiterTest` | `tests/Unit/RateLimiterTest.php` | Leaky-bucket rate enforcement | +| โ€” | `tests/Unit/CommandSelectorTest.php` | Weighted command distribution | +| `MetricsOutputTest` | `tests/Unit/HdrEncoderTest.php`, `tests/Integration/MetricsOutputTest.php` | HDR percentiles + V2 compressed encoding; NDJSON schema, exact request counts, latency accuracy | +| `RateLimitingTest` | `tests/Integration/RateLimitingTest.php` | RPS enforcement, shared limit across (concurrent) connections, unlimited throughput | +| `ErrorMetricsIntegrationTest` | `tests/Integration/ErrorMetricsTest.php` | Error-rate simulation, per-command error counts, errors excluded from latency histogram | +| `RecordingBenchmarkClientTest` | `tests/Integration/RecordingWorkloadTest.php` | Full pipeline โ†’ NDJSON schema, inline + process modes | +| `BenchmarkIntegrationTest` | `tests/Integration/LiveClientTest.php` | Live server: connect/ping/set-get, driver version, multi-process fork-then-connect (gated; skips without extension/server) | + +## Adding a new driver + +1. Create a client in `src/Client/Impl/` extending `BenchmarkClient`. +2. Register it in `Client\Factory::DRIVERS`. +3. Add driver configs under `configs/drivers/`. + +## License + +Apache License 2.0 โ€” see [LICENSE](../LICENSE) diff --git a/php/bin/resp-bench b/php/bin/resp-bench new file mode 100755 index 0000000..679b160 --- /dev/null +++ b/php/bin/resp-bench @@ -0,0 +1,23 @@ +#!/usr/bin/env php +run($argv)); diff --git a/php/composer.json b/php/composer.json new file mode 100644 index 0000000..eb0a3a5 --- /dev/null +++ b/php/composer.json @@ -0,0 +1,33 @@ +{ + "name": "valkey-io/resp-bench-php", + "description": "PHP benchmark engine for resp-bench (RESP protocol / Valkey / Redis)", + "type": "project", + "license": "Apache-2.0", + "require": { + "php": ">=8.2", + "ext-json": "*" + }, + "require-dev": { + "phpunit/phpunit": "^11.0" + }, + "suggest": { + "ext-valkey_glide": "Valkey GLIDE PHP client extension (required for the valkey-glide-php driver)", + "ext-pcntl": "Process control (required for the multi-process concurrency model on live-server runs)" + }, + "autoload": { + "psr-4": { + "RespBench\\": "src/" + } + }, + "autoload-dev": { + "psr-4": { + "RespBench\\Tests\\": "tests/" + } + }, + "bin": [ + "bin/resp-bench" + ], + "config": { + "optimize-autoloader": true + } +} diff --git a/php/phpunit.xml b/php/phpunit.xml new file mode 100644 index 0000000..7ec4625 --- /dev/null +++ b/php/phpunit.xml @@ -0,0 +1,20 @@ + + + + + tests/Unit + + + tests/Integration + + + + + src + + + diff --git a/php/src/Cli.php b/php/src/Cli.php new file mode 100644 index 0000000..d392aa3 --- /dev/null +++ b/php/src/Cli.php @@ -0,0 +1,178 @@ + */ + private array $options = [ + 'host' => 'localhost', + 'port' => 6379, + ]; + + /** + * @param list $argv + */ + public function run(array $argv): int + { + try { + $this->parseOptions($argv); + + if (!empty($this->options['info'])) { + $this->printInfo(); + + return 0; + } + + $this->validateOptions(); + $this->executeBenchmark(); + + return 0; + } catch (\Throwable $e) { + fwrite(STDERR, 'Error: ' . $e->getMessage() . "\n"); + if (getenv('DEBUG')) { + fwrite(STDERR, $e->getTraceAsString() . "\n"); + } + + return 1; + } + } + + /** + * @param list $argv + */ + private function parseOptions(array $argv): void + { + $args = array_slice($argv, 1); + $count = count($args); + + for ($i = 0; $i < $count; $i++) { + $arg = $args[$i]; + $next = static fn (): string => $args[++$i] ?? ''; + + switch ($arg) { + case '--server': + $server = $next(); + $parts = explode(':', $server); + $this->options['host'] = $parts[0]; + if (isset($parts[1])) { + $this->options['port'] = (int) $parts[1]; + } + break; + case '--driver': + $this->options['driver'] = $next(); + break; + case '--workload': + $this->options['workload'] = $next(); + break; + case '--metrics': + $this->options['metrics'] = $next(); + break; + case '--commit-id': + $this->options['commit_id'] = $next(); + break; + case '--concurrency': + $this->options['concurrency_mode'] = $next(); + break; + case '--info': + $this->options['info'] = true; + break; + case '-h': + case '--help': + $this->printHelp(); + exit(0); + case '-v': + case '--version': + echo 'resp-bench PHP Engine v' . Version::VERSION . "\n"; + exit(0); + default: + // ignore unknown args for forward compatibility + break; + } + } + } + + private function validateOptions(): void + { + $missing = []; + foreach (['driver', 'workload', 'metrics'] as $required) { + if (empty($this->options[$required])) { + $missing[] = "--{$required}"; + } + } + if ($missing !== []) { + throw new \InvalidArgumentException('Missing required options: ' . implode(', ', $missing)); + } + + if (!is_file((string) $this->options['driver'])) { + throw new \InvalidArgumentException('Driver config not found: ' . $this->options['driver']); + } + if (!is_file((string) $this->options['workload'])) { + throw new \InvalidArgumentException('Workload config not found: ' . $this->options['workload']); + } + } + + private function executeBenchmark(): void + { + $driverConfig = Loader::loadDriverConfig((string) $this->options['driver']); + $workloadConfig = Loader::loadWorkloadConfig((string) $this->options['workload']); + + $engine = new Benchmark( + host: (string) $this->options['host'], + port: (int) $this->options['port'], + driverConfig: $driverConfig, + workloadConfig: $workloadConfig, + metricsPath: (string) $this->options['metrics'], + commitId: isset($this->options['commit_id']) ? (string) $this->options['commit_id'] : null, + concurrencyMode: isset($this->options['concurrency_mode']) + ? (string) $this->options['concurrency_mode'] + : null, + ); + + $engine->run(); + } + + private function printInfo(): void + { + echo 'resp-bench PHP Engine v' . Version::VERSION . "\n\n"; + echo "Supported Drivers:\n"; + foreach (ClientFactory::supportedDrivers() as $driver) { + echo " - {$driver}\n"; + } + echo "\nSupported Commands:\n"; + foreach (CommandFactory::supportedCommands() as $cmd) { + echo " - {$cmd}\n"; + } + echo "\nConcurrency: process-per-connection (max 256), inline fallback\n"; + echo 'valkey_glide extension: ' . (extension_loaded('valkey_glide') ? 'loaded' : 'NOT loaded') . "\n"; + echo 'pcntl extension: ' . (function_exists('pcntl_fork') ? 'available' : 'NOT available') . "\n"; + } + + private function printHelp(): void + { + echo <<> */ + private const DRIVERS = [ + 'valkey-glide-php' => ValkeyGlidePhpClient::class, + 'phpredis' => PhpRedisClient::class, + 'recording' => RecordingClient::class, + ]; + + public static function create(string $driverId): BenchmarkClient + { + $class = self::DRIVERS[$driverId] ?? null; + if ($class === null) { + throw new InvalidArgumentException( + "Unknown driver: {$driverId}. Supported: " . implode(', ', array_keys(self::DRIVERS)) + ); + } + + return new $class(); + } + + public static function createAndConnect(string $host, int $port, DriverConfig $config): BenchmarkClient + { + $client = self::create((string) $config->driverId); + $client->connect($host, $port, $config); + + return $client; + } + + /** + * @return list + */ + public static function supportedDrivers(): array + { + return array_keys(self::DRIVERS); + } +} diff --git a/php/src/Client/Impl/PhpRedisClient.php b/php/src/Client/Impl/PhpRedisClient.php new file mode 100644 index 0000000..3ffa96e --- /dev/null +++ b/php/src/Client/Impl/PhpRedisClient.php @@ -0,0 +1,176 @@ +connect($host, $port); + * $r->set('k','v'); $r->get('k'); $r->ping(); $r->del('k'); $r->close(); + * + * As with the GLIDE client, each forked worker constructs and connects its own + * instance AFTER forking โ€” connections are never inherited across a fork. + */ +final class PhpRedisClient extends BenchmarkClient +{ + private ?object $client = null; + + public function connect(string $host, int $port, DriverConfig $config): void + { + if (!extension_loaded('redis')) { + throw new RuntimeException( + 'The PHPRedis extension (ext-redis) is not loaded. ' + . 'Install it (pecl install redis) and add extension=redis to php.ini.' + ); + } + + if ($config->isCluster()) { + $this->client = $this->makeCluster($host, $port, $config); + + return; + } + + /** @var object $redis */ + $redis = new \Redis(); + + // Optional TLS: PHPRedis uses a tls:// host scheme. + $connectHost = $host; + $sslContext = null; + if ($config->tls !== null) { + $connectHost = 'tls://' . $host; + $sslContext = $this->buildSslContext($config->tls); + } + + // connect(host, port, timeout, persistent_id, retry_interval, read_timeout, context) + $timeoutSeconds = 0.5; + $ok = $sslContext !== null + ? $redis->connect($connectHost, $port, $timeoutSeconds, null, 0, 0, ['stream' => $sslContext]) + : $redis->connect($connectHost, $port, $timeoutSeconds); + + if ($ok === false) { + throw new RuntimeException("PHPRedis failed to connect to {$host}:{$port}"); + } + + if ($config->auth !== null && isset($config->auth['password'])) { + $auth = isset($config->auth['username']) + ? [(string) $config->auth['username'], (string) $config->auth['password']] + : (string) $config->auth['password']; + $redis->auth($auth); + } + + $this->client = $redis; + } + + private function makeCluster(string $host, int $port, DriverConfig $config): object + { + if (!class_exists('RedisCluster')) { + throw new RuntimeException('RedisCluster not available from ext-redis.'); + } + + // RedisCluster(name, seeds, timeout, read_timeout, persistent, auth) + $seeds = ["{$host}:{$port}"]; + $auth = null; + if ($config->auth !== null && isset($config->auth['password'])) { + $auth = (string) $config->auth['password']; + } + + /** @psalm-suppress MixedMethodCall */ + return new \RedisCluster(null, $seeds, 0.5, 0.5, false, $auth); + } + + /** + * @param array $tls + * @return array + */ + private function buildSslContext(array $tls): array + { + $ssl = []; + if (isset($tls['ca_cert_path'])) { + $ssl['cafile'] = (string) $tls['ca_cert_path']; + } + if (isset($tls['cert_path'])) { + $ssl['local_cert'] = (string) $tls['cert_path']; + } + if (isset($tls['key_path'])) { + $ssl['local_pk'] = (string) $tls['key_path']; + } + if (isset($tls['verify_peer'])) { + $ssl['verify_peer'] = (bool) $tls['verify_peer']; + } + + return $ssl; + } + + public function isConnected(): bool + { + if ($this->client === null) { + return false; + } + + try { + return $this->client->ping() !== false; + } catch (\Throwable) { + return false; + } + } + + public function ping(): TimedResult + { + return $this->measure(fn (): mixed => $this->requireClient()->ping()); + } + + public function get(string $key): TimedResult + { + return $this->measure(fn (): mixed => $this->requireClient()->get($key)); + } + + public function set(string $key, string $value): TimedResult + { + return $this->measure(fn (): mixed => $this->requireClient()->set($key, $value)); + } + + public function del(string $key): TimedResult + { + return $this->measure(fn (): mixed => $this->requireClient()->del($key)); + } + + public function close(): void + { + if ($this->client !== null) { + try { + $this->client->close(); + } catch (\Throwable) { + // ignore + } + $this->client = null; + } + } + + public function driverVersion(): string + { + $version = phpversion('redis'); + + return $version !== false ? $version : 'unknown'; + } + + private function requireClient(): object + { + if ($this->client === null) { + throw new RuntimeException('PHPRedis client is not connected.'); + } + + return $this->client; + } +} diff --git a/php/src/Client/Impl/RecordingClient.php b/php/src/Client/Impl/RecordingClient.php new file mode 100644 index 0000000..264fe81 --- /dev/null +++ b/php/src/Client/Impl/RecordingClient.php @@ -0,0 +1,131 @@ + */ + private array $store = []; + + /** @var list */ + private array $recorded = []; + + private bool $connected = false; + + private float $errorRate = 0.0; + private string $errorMessage = 'Simulated error'; + + public function connect(string $host, int $port, DriverConfig $config): void + { + $this->connected = true; + + // Optional error simulation (parity with the Ruby recording client): + // "specific_driver_config": { "error_rate": 0.1, "error_message": "..." } + $cfg = $config->specificDriverConfig; + if (isset($cfg['error_rate'])) { + $this->errorRate = max(0.0, min(1.0, (float) $cfg['error_rate'])); + } + if (isset($cfg['error_message'])) { + $this->errorMessage = (string) $cfg['error_message']; + } + } + + private function shouldFail(): bool + { + if ($this->errorRate <= 0.0) { + return false; + } + if ($this->errorRate >= 1.0) { + return true; + } + + return (mt_rand() / mt_getrandmax()) < $this->errorRate; + } + + public function isConnected(): bool + { + return $this->connected; + } + + public function ping(): TimedResult + { + return $this->measure(function (): string { + $this->recorded[] = ['op' => 'PING', 'key' => '', 'size' => 0]; + if ($this->shouldFail()) { + throw new \RuntimeException($this->errorMessage); + } + + return 'PONG'; + }); + } + + public function get(string $key): TimedResult + { + return $this->measure(function () use ($key): ?string { + $this->recorded[] = ['op' => 'GET', 'key' => $key, 'size' => 0]; + if ($this->shouldFail()) { + throw new \RuntimeException($this->errorMessage); + } + + return $this->store[$key] ?? null; + }); + } + + public function set(string $key, string $value): TimedResult + { + return $this->measure(function () use ($key, $value): string { + $this->recorded[] = ['op' => 'SET', 'key' => $key, 'size' => strlen($value)]; + if ($this->shouldFail()) { + throw new \RuntimeException($this->errorMessage); + } + $this->store[$key] = $value; + + return 'OK'; + }); + } + + public function del(string $key): TimedResult + { + return $this->measure(function () use ($key): int { + $this->recorded[] = ['op' => 'DEL', 'key' => $key, 'size' => 0]; + if ($this->shouldFail()) { + throw new \RuntimeException($this->errorMessage); + } + $existed = isset($this->store[$key]); + unset($this->store[$key]); + + return $existed ? 1 : 0; + }); + } + + public function close(): void + { + $this->connected = false; + } + + public function driverVersion(): string + { + return 'recording-1.0'; + } + + /** + * @return list + */ + public function recordedOperations(): array + { + return $this->recorded; + } +} diff --git a/php/src/Client/Impl/ValkeyGlidePhpClient.php b/php/src/Client/Impl/ValkeyGlidePhpClient.php new file mode 100644 index 0000000..cb2a007 --- /dev/null +++ b/php/src/Client/Impl/ValkeyGlidePhpClient.php @@ -0,0 +1,175 @@ +connect(addresses: [['host' => 'localhost', 'port' => 6379]]); + * $client->set('foo', 'bar'); $client->get('foo'); $client->ping(); + * $client->close(); + * + * IMPORTANT: In the multi-process engine, each worker constructs and connects its + * own client AFTER forking โ€” a connection is never inherited across a fork. + */ +final class ValkeyGlidePhpClient extends BenchmarkClient +{ + private ?object $client = null; + + public function connect(string $host, int $port, DriverConfig $config): void + { + if (!extension_loaded('valkey_glide')) { + throw new RuntimeException( + 'The valkey_glide PHP extension is not loaded. ' + . 'Install it (see php/README.md) and add extension=valkey_glide to php.ini.' + ); + } + + $addresses = [['host' => $host, 'port' => $port]]; + $useTls = false; + $advancedConfig = null; + $credentials = null; + + if ($config->tls !== null) { + $useTls = true; + $tlsConfig = []; + if (isset($config->tls['ca_cert_path'])) { + $tlsConfig['root_certs'] = (string) file_get_contents((string) $config->tls['ca_cert_path']); + } + if (isset($config->tls['cert_path'])) { + $tlsConfig['client_cert'] = (string) file_get_contents((string) $config->tls['cert_path']); + } + if (isset($config->tls['key_path'])) { + $tlsConfig['client_key'] = (string) file_get_contents((string) $config->tls['key_path']); + } + if ($tlsConfig !== []) { + $advancedConfig = ['tls_config' => $tlsConfig]; + } + } + + if ($config->auth !== null) { + $credentials = []; + if (isset($config->auth['username'])) { + $credentials['username'] = (string) $config->auth['username']; + } + if (isset($config->auth['password'])) { + $credentials['password'] = (string) $config->auth['password']; + } + } + + $this->client = $config->isCluster() + ? $this->makeClient('ValkeyGlideCluster', $addresses, $useTls, $advancedConfig, $credentials) + : $this->makeClient('ValkeyGlide', $addresses, $useTls, $advancedConfig, $credentials); + } + + /** + * @param list $addresses + * @param array|null $advancedConfig + * @param array|null $credentials + */ + private function makeClient( + string $class, + array $addresses, + bool $useTls, + ?array $advancedConfig, + ?array $credentials, + ): object { + if (!class_exists($class)) { + throw new RuntimeException("{$class} class not available from the valkey_glide extension."); + } + + // ValkeyGlideCluster connects via constructor; ValkeyGlide via connect(). + if ($class === 'ValkeyGlideCluster') { + /** @psalm-suppress MixedMethodCall */ + return new $class( + addresses: $addresses, + use_tls: $useTls, + credentials: $credentials, + advanced_config: $advancedConfig, + ); + } + + /** @psalm-suppress MixedMethodCall */ + $client = new $class(); + $client->connect( + addresses: $addresses, + use_tls: $useTls, + credentials: $credentials, + advanced_config: $advancedConfig, + ); + + return $client; + } + + public function isConnected(): bool + { + if ($this->client === null) { + return false; + } + + try { + return $this->client->ping() !== false; + } catch (\Throwable) { + return false; + } + } + + public function ping(): TimedResult + { + return $this->measure(fn (): mixed => $this->requireClient()->ping()); + } + + public function get(string $key): TimedResult + { + return $this->measure(fn (): mixed => $this->requireClient()->get($key)); + } + + public function set(string $key, string $value): TimedResult + { + return $this->measure(fn (): mixed => $this->requireClient()->set($key, $value)); + } + + public function del(string $key): TimedResult + { + return $this->measure(fn (): mixed => $this->requireClient()->del($key)); + } + + public function close(): void + { + if ($this->client !== null) { + try { + $this->client->close(); + } catch (\Throwable) { + // ignore close errors + } + $this->client = null; + } + } + + public function driverVersion(): string + { + $version = phpversion('valkey_glide'); + + return $version !== false ? $version : 'unknown'; + } + + private function requireClient(): object + { + if ($this->client === null) { + throw new RuntimeException('ValkeyGlide client is not connected.'); + } + + return $this->client; + } +} diff --git a/php/src/Client/TimedResult.php b/php/src/Client/TimedResult.php new file mode 100644 index 0000000..7306e31 --- /dev/null +++ b/php/src/Client/TimedResult.php @@ -0,0 +1,30 @@ +error === null; + } + + public function isError(): bool + { + return $this->error !== null; + } +} diff --git a/php/src/Command/Command.php b/php/src/Command/Command.php new file mode 100644 index 0000000..46fa0d8 --- /dev/null +++ b/php/src/Command/Command.php @@ -0,0 +1,31 @@ +weight = $config->weight; + $this->name = strtoupper($config->command); + $this->dataSizeBytes = $config->dataSizeBytes; + } + + /** + * Execute the command and return a result for metrics. + */ + abstract public function execute(BenchmarkClient $client, KeyGenerator $keyGenerator): CommandResult; +} diff --git a/php/src/Command/CommandResult.php b/php/src/Command/CommandResult.php new file mode 100644 index 0000000..dda55c0 --- /dev/null +++ b/php/src/Command/CommandResult.php @@ -0,0 +1,18 @@ +> */ + private const COMMAND_CLASSES = [ + 'ping' => PingCommand::class, + 'get' => GetCommand::class, + 'set' => SetCommand::class, + ]; + + public static function create(CommandConfig $config): Command + { + $class = self::COMMAND_CLASSES[$config->command] ?? null; + if ($class === null) { + throw new InvalidArgumentException( + "Unknown command: {$config->command}. Supported: " + . implode(', ', array_keys(self::COMMAND_CLASSES)) + ); + } + + return new $class($config); + } + + /** + * @param list $configs + * @return list + */ + public static function createAll(array $configs): array + { + return array_map(static fn (CommandConfig $c): Command => self::create($c), $configs); + } + + /** + * @return list + */ + public static function supportedCommands(): array + { + return array_keys(self::COMMAND_CLASSES); + } +} diff --git a/php/src/Command/Impl/GetCommand.php b/php/src/Command/Impl/GetCommand.php new file mode 100644 index 0000000..ac404ec --- /dev/null +++ b/php/src/Command/Impl/GetCommand.php @@ -0,0 +1,21 @@ +nextKey(); + $result = $client->get($key); + + return new CommandResult($this->name, $result->latencyMicros, $result->isSuccess()); + } +} diff --git a/php/src/Command/Impl/PingCommand.php b/php/src/Command/Impl/PingCommand.php new file mode 100644 index 0000000..bbb4d10 --- /dev/null +++ b/php/src/Command/Impl/PingCommand.php @@ -0,0 +1,20 @@ +ping(); + + return new CommandResult($this->name, $result->latencyMicros, $result->isSuccess()); + } +} diff --git a/php/src/Command/Impl/SetCommand.php b/php/src/Command/Impl/SetCommand.php new file mode 100644 index 0000000..3f4cb17 --- /dev/null +++ b/php/src/Command/Impl/SetCommand.php @@ -0,0 +1,45 @@ +value = self::generateValue($this->dataSizeBytes); + } + + public function execute(BenchmarkClient $client, KeyGenerator $keyGenerator): CommandResult + { + $key = $keyGenerator->nextKey(); + $result = $client->set($key, $this->value); + + return new CommandResult($this->name, $result->latencyMicros, $result->isSuccess()); + } + + /** + * Deterministic value of the requested size (matches the Ruby engine pattern). + */ + private static function generateValue(int $size): string + { + if ($size <= 0) { + return ''; + } + + $pattern = '0123456789ABCDEF'; + $repeat = intdiv($size, strlen($pattern)) + 1; + + return substr(str_repeat($pattern, $repeat), 0, $size); + } +} diff --git a/php/src/Config/CommandConfig.php b/php/src/Config/CommandConfig.php new file mode 100644 index 0000000..46ddcc9 --- /dev/null +++ b/php/src/Config/CommandConfig.php @@ -0,0 +1,27 @@ +command = strtolower($command); + $this->weight = $weight; + $this->dataSizeBytes = $dataSizeBytes ?? self::DEFAULT_DATA_SIZE_BYTES; + } +} diff --git a/php/src/Config/CompletionConfig.php b/php/src/Config/CompletionConfig.php new file mode 100644 index 0000000..45ef270 --- /dev/null +++ b/php/src/Config/CompletionConfig.php @@ -0,0 +1,38 @@ +type === 'duration'; + } + + public function isRequestBased(): bool + { + return $this->type === 'requests'; + } + + public function durationSeconds(): int + { + return $this->seconds ?? 0; + } + + public function totalRequests(): int + { + return $this->requests ?? 0; + } +} diff --git a/php/src/Config/DriverConfig.php b/php/src/Config/DriverConfig.php new file mode 100644 index 0000000..951eefa --- /dev/null +++ b/php/src/Config/DriverConfig.php @@ -0,0 +1,50 @@ + $specificDriverConfig + * @param array|null $tls + * @param array|null $auth + */ + public function __construct( + public readonly string $schemaVersion = '1.0', + public readonly ?string $description = null, + public readonly ?string $driverId = null, + public readonly string $mode = 'standalone', + public readonly ?array $tls = null, + public readonly ?array $auth = null, + public readonly array $specificDriverConfig = [], + ) { + } + + public function secondaryDriverId(): ?string + { + $value = $this->specificDriverConfig['secondary_driver_id'] ?? null; + + return is_string($value) ? $value : null; + } + + public function isStandalone(): bool + { + return $this->mode === 'standalone'; + } + + public function isCluster(): bool + { + return $this->mode === 'cluster'; + } + + public function isSentinel(): bool + { + return $this->mode === 'sentinel'; + } +} diff --git a/php/src/Config/KeyspaceConfig.php b/php/src/Config/KeyspaceConfig.php new file mode 100644 index 0000000..283fcfa --- /dev/null +++ b/php/src/Config/KeyspaceConfig.php @@ -0,0 +1,49 @@ +generationAlg === 'sequential_int'; + } + + public function isUniformRand(): bool + { + return $this->generationAlg === 'uniform_rand'; + } + + public function effectiveKeyPrefix(): string + { + return $this->keyPrefix !== '' ? $this->keyPrefix : self::DEFAULT_KEY_PREFIX; + } + + /** + * Returns the seed value (defaults to 0 if not set), matching Ruby's seed_value. + */ + public function seedValue(): int + { + return $this->seed ?? 0; + } +} diff --git a/php/src/Config/Loader.php b/php/src/Config/Loader.php new file mode 100644 index 0000000..32ae2a2 --- /dev/null +++ b/php/src/Config/Loader.php @@ -0,0 +1,171 @@ + $json + */ + public static function parseDriverConfig(array $json): DriverConfig + { + return new DriverConfig( + schemaVersion: (string) ($json['schema_version'] ?? '1.0'), + description: isset($json['description']) ? (string) $json['description'] : null, + driverId: isset($json['driver_id']) ? (string) $json['driver_id'] : null, + mode: (string) ($json['mode'] ?? 'standalone'), + tls: isset($json['tls']) && is_array($json['tls']) ? $json['tls'] : null, + auth: isset($json['auth']) && is_array($json['auth']) ? $json['auth'] : null, + specificDriverConfig: isset($json['specific_driver_config']) && is_array($json['specific_driver_config']) + ? $json['specific_driver_config'] + : [], + ); + } + + /** + * @param array $json + */ + public static function parseWorkloadConfig(array $json): WorkloadConfig + { + $phases = []; + foreach (($json['phases'] ?? []) as $phase) { + $phases[] = self::parsePhaseConfig($phase); + } + + return new WorkloadConfig( + schemaVersion: (string) ($json['schema_version'] ?? '1.0'), + benchmarkProfile: isset($json['benchmark_profile']) && is_array($json['benchmark_profile']) + ? $json['benchmark_profile'] + : [], + phases: $phases, + ); + } + + /** + * @param array $json + */ + private static function parsePhaseConfig(array $json): PhaseConfig + { + $commands = []; + foreach (($json['commands'] ?? []) as $command) { + $commands[] = self::parseCommandConfig($command); + } + + return new PhaseConfig( + id: (string) $json['id'], + connections: (int) $json['connections'], + completion: self::parseCompletionConfig($json['completion'] ?? []), + keyspace: self::parseKeyspaceConfig($json['keyspace'] ?? []), + commands: $commands, + description: isset($json['description']) ? (string) $json['description'] : null, + cpsLimit: isset($json['cps_limit']) ? (int) $json['cps_limit'] : -1, + rpsLimit: isset($json['rps_limit']) ? (int) $json['rps_limit'] : -1, + pipelineDepth: isset($json['pipeline_depth']) + ? (int) $json['pipeline_depth'] + : PhaseConfig::DEFAULT_PIPELINE_DEPTH, + warmupRequests: isset($json['warmup_requests']) + ? (int) $json['warmup_requests'] + : PhaseConfig::DEFAULT_WARMUP_REQUESTS, + ); + } + + /** + * @param array $json + */ + private static function parseCompletionConfig(array $json): CompletionConfig + { + return new CompletionConfig( + type: (string) ($json['type'] ?? 'requests'), + seconds: isset($json['seconds']) ? (int) $json['seconds'] : null, + requests: isset($json['requests']) ? (int) $json['requests'] : null, + ); + } + + /** + * @param array $json + */ + private static function parseKeyspaceConfig(array $json): KeyspaceConfig + { + return new KeyspaceConfig( + keysCount: (int) ($json['keys_count'] ?? 1), + keySizeBytes: isset($json['key_size_bytes']) + ? (int) $json['key_size_bytes'] + : KeyspaceConfig::DEFAULT_KEY_SIZE_BYTES, + keyPrefix: isset($json['key_prefix']) + ? (string) $json['key_prefix'] + : KeyspaceConfig::DEFAULT_KEY_PREFIX, + generationAlg: (string) ($json['generation_alg'] ?? 'sequential_int'), + seed: isset($json['seed']) ? (int) $json['seed'] : null, + ); + } + + /** + * @param array $json + */ + private static function parseCommandConfig(array $json): CommandConfig + { + return new CommandConfig( + command: (string) $json['command'], + weight: (float) $json['weight'], + dataSizeBytes: isset($json['data_size_bytes']) + ? (int) $json['data_size_bytes'] + : CommandConfig::DEFAULT_DATA_SIZE_BYTES, + ); + } + + /** + * @return array + */ + private static function parseJsonFile(string $path): array + { + $content = @file_get_contents($path); + if ($content === false) { + throw new RuntimeException("Cannot read config file: {$path}"); + } + + return self::decode($content); + } + + /** + * @return array + */ + private static function decode(string $json): array + { + try { + /** @var array $decoded */ + $decoded = json_decode($json, true, 512, JSON_THROW_ON_ERROR); + } catch (JsonException $e) { + throw new RuntimeException('Invalid JSON config: ' . $e->getMessage(), 0, $e); + } + + return $decoded; + } +} diff --git a/php/src/Config/PhaseConfig.php b/php/src/Config/PhaseConfig.php new file mode 100644 index 0000000..7d3409a --- /dev/null +++ b/php/src/Config/PhaseConfig.php @@ -0,0 +1,46 @@ + $commands + */ + public function __construct( + public readonly string $id, + public readonly int $connections, + public readonly CompletionConfig $completion, + public readonly KeyspaceConfig $keyspace, + public readonly array $commands, + public readonly ?string $description = null, + public readonly int $cpsLimit = -1, + public readonly int $rpsLimit = -1, + public readonly int $pipelineDepth = self::DEFAULT_PIPELINE_DEPTH, + public readonly int $warmupRequests = self::DEFAULT_WARMUP_REQUESTS, + ) { + } + + public function hasCpsLimit(): bool + { + return $this->cpsLimit > 0; + } + + public function hasRpsLimit(): bool + { + return $this->rpsLimit > 0; + } + + public function effectivePipelineDepth(): int + { + return $this->pipelineDepth > 0 ? $this->pipelineDepth : self::DEFAULT_PIPELINE_DEPTH; + } +} diff --git a/php/src/Config/WorkloadConfig.php b/php/src/Config/WorkloadConfig.php new file mode 100644 index 0000000..a5f3d6c --- /dev/null +++ b/php/src/Config/WorkloadConfig.php @@ -0,0 +1,37 @@ + $benchmarkProfile + * @param list $phases + */ + public function __construct( + public readonly string $schemaVersion, + public readonly array $benchmarkProfile, + public readonly array $phases, + ) { + } + + public function name(): ?string + { + $value = $this->benchmarkProfile['name'] ?? null; + + return is_string($value) ? $value : null; + } + + public function description(): ?string + { + $value = $this->benchmarkProfile['description'] ?? null; + + return is_string($value) ? $value : null; + } +} diff --git a/php/src/Engine/Benchmark.php b/php/src/Engine/Benchmark.php new file mode 100644 index 0000000..817540d --- /dev/null +++ b/php/src/Engine/Benchmark.php @@ -0,0 +1,331 @@ +metricsPath); + $writer->setMetadata( + commitId: $this->commitId, + driverId: $this->driverConfig->driverId, + primaryDriverVersion: $this->probeDriverVersion(), + secondaryDriverId: $this->driverConfig->secondaryDriverId(), + ); + + foreach ($this->workloadConfig->phases as $phase) { + $collector = $this->runPhase($phase); + $writer->writePhaseResults($phase->id, 'COMPLETED', $phase->connections, $collector); + } + } + + private function mode(): string + { + if ($this->concurrencyMode !== null) { + return $this->concurrencyMode; + } + + // Recording driver is server-free and cheap โ€” run inline. + if ($this->driverConfig->driverId === 'recording') { + return 'inline'; + } + + return function_exists('pcntl_fork') ? 'process' : 'inline'; + } + + private function runPhase(PhaseConfig $phase): Collector + { + $collector = new Collector(); + $collector->start(); + + $workerCount = max(1, min($phase->connections, self::MAX_WORKERS)); + + if ($this->mode() === 'process' && function_exists('pcntl_fork')) { + $this->runPhaseMultiProcess($phase, $workerCount, $collector); + } else { + $this->runPhaseInline($phase, $workerCount, $collector); + } + + $collector->stop(); + + return $collector; + } + + // --- Multi-process execution ------------------------------------------- + + private function runPhaseMultiProcess(PhaseConfig $phase, int $workerCount, Collector $collector): void + { + /** @var array $parentEnds */ + $parentEnds = []; + /** @var array $children pid => worker index */ + $children = []; + + for ($i = 0; $i < $workerCount; $i++) { + $pair = stream_socket_pair(STREAM_PF_UNIX, STREAM_SOCK_STREAM, STREAM_IPPROTO_IP); + if ($pair === false) { + throw new \RuntimeException('stream_socket_pair failed'); + } + [$parentEnd, $childEnd] = $pair; + + $pid = pcntl_fork(); + if ($pid === -1) { + throw new \RuntimeException('pcntl_fork failed'); + } + + if ($pid === 0) { + // CHILD: connect after fork, run slice, write partial, exit. + fclose($parentEnd); + $this->runWorkerAndReport($phase, $workerCount, $i, $childEnd); + fclose($childEnd); + exit(0); + } + + fclose($childEnd); + $parentEnds[$i] = $parentEnd; + $children[$pid] = $i; + } + + // Collect partial metrics from each worker. + foreach ($parentEnds as $fh) { + $payload = stream_get_contents($fh); + fclose($fh); + if ($payload === false || $payload === '') { + continue; + } + $this->mergePartial($collector, $payload); + } + + // Reap children. + foreach (array_keys($children) as $pid) { + $status = 0; + pcntl_waitpid($pid, $status); + } + } + + /** + * @param resource $childEnd + */ + private function runWorkerAndReport(PhaseConfig $phase, int $workerCount, int $workerIndex, $childEnd): void + { + $workerCollector = $this->runWorker($phase, $workerCount, $workerIndex); + fwrite($childEnd, $this->serializeCollector($workerCollector)); + } + + // --- Inline execution (fallback / recording) --------------------------- + + private function runPhaseInline(PhaseConfig $phase, int $workerCount, Collector $collector): void + { + for ($i = 0; $i < $workerCount; $i++) { + $workerCollector = $this->runWorker($phase, $workerCount, $i); + $collector->mergeFrom($workerCollector); + } + } + + // --- Shared worker loop ------------------------------------------------ + + private function runWorker(PhaseConfig $phase, int $workerCount, int $workerIndex): Collector + { + $collector = new Collector(); + + $client = ClientFactory::createAndConnect($this->host, $this->port, $this->driverConfig); + try { + $commands = CommandFactory::createAll($phase->commands); + $selector = new CommandSelector($commands); + $keyGen = $this->keyGeneratorForWorker($phase->keyspace, $workerIndex); + + // Divide the phase-level rate limit across workers. + $rps = $phase->hasRpsLimit() ? max(1, intdiv($phase->rpsLimit, $workerCount)) : -1; + $limiter = RateLimiter::create($rps); + + $this->runWarmup($client, $commands, $keyGen, $phase->warmupRequests, $workerCount, $workerIndex); + + $target = $this->workerRequestTarget($phase, $workerCount, $workerIndex); + $deadline = $phase->completion->isDurationBased() + ? microtime(true) + $phase->completion->durationSeconds() + : null; + + $done = 0; + while (true) { + if ($target !== null && $done >= $target) { + break; + } + if ($deadline !== null && microtime(true) >= $deadline) { + break; + } + + $limiter?->acquire(); + $command = $selector->select(); + $collector->record($command->execute($client, $keyGen)); + $done++; + } + } finally { + $client->close(); + } + + return $collector; + } + + private function runWarmup( + BenchmarkClient $client, + array $commands, + KeyGenerator $keyGen, + int $warmupRequests, + int $workerCount, + int $workerIndex, + ): void { + if ($warmupRequests <= 0 || $commands === []) { + return; + } + $share = intdiv($warmupRequests, $workerCount); + if ($workerIndex < ($warmupRequests % $workerCount)) { + $share++; + } + $selector = new CommandSelector($commands); + for ($i = 0; $i < $share; $i++) { + $selector->select()->execute($client, $keyGen); + } + } + + /** + * Per-worker request target for request-based completion, split evenly with + * the remainder distributed to the first workers (matches Java's forkForThread). + */ + private function workerRequestTarget(PhaseConfig $phase, int $workerCount, int $workerIndex): ?int + { + if (!$phase->completion->isRequestBased()) { + return null; + } + $total = $phase->completion->totalRequests(); + $share = intdiv($total, $workerCount); + if ($workerIndex < ($total % $workerCount)) { + $share++; + } + + return $share; + } + + /** + * uniform_rand: seed per worker (seed + index) for reproducible-yet-distinct + * sequences. sequential_int: shared config (each worker walks the keyspace). + */ + private function keyGeneratorForWorker(KeyspaceConfig $keyspace, int $workerIndex): KeyGenerator + { + if ($keyspace->isUniformRand()) { + return KeyGenerator::createWithSeed($keyspace, $keyspace->seedValue() + $workerIndex); + } + + return KeyGenerator::create($keyspace); + } + + // --- Cross-process metrics serialization ------------------------------- + + /** + * Serialize a collector's totals + per-command counts + histogram counts to + * JSON. Histograms are sent as sparse index=>count maps so the parent can + * reconstruct an identical HdrHistogram and merge losslessly. + */ + private function serializeCollector(Collector $collector): string + { + $commands = []; + foreach ($collector->allMetrics() as $name => $cmd) { + $histogram = $cmd->histogram(); + $counts = []; + if ($histogram !== null) { + $len = $histogram->relevantLength(); + for ($i = 0; $i < $len; $i++) { + $c = $histogram->rawCountAt($i); + if ($c > 0) { + $counts[$i] = $c; + } + } + } + $commands[$name] = [ + 'requests' => $cmd->requests(), + 'errors' => $cmd->errors(), + 'counts' => $counts, + ]; + } + + return json_encode([ + 'total_requests' => $collector->totalRequests(), + 'total_errors' => $collector->totalErrors(), + 'commands' => $commands, + ], JSON_THROW_ON_ERROR) . "\n"; + } + + private function mergePartial(Collector $collector, string $payload): void + { + // A worker writes a single JSON line; guard against partial reads. + $line = trim($payload); + if ($line === '') { + return; + } + + /** @var array{total_requests:int,total_errors:int,commands:array}>} $data */ + $data = json_decode($line, true, 512, JSON_THROW_ON_ERROR); + + $partial = new Collector(); + // Rebuild a collector via a temporary histogram-backed merge. + foreach ($data['commands'] as $name => $cmd) { + $histogram = new HdrHistogram(1, 600_000_000, 3); + foreach ($cmd['counts'] as $index => $count) { + $histogram->recordValueWithCount($histogram->valueFromIndex((int) $index), (int) $count); + } + $partial->ingestCommand($name, $cmd['requests'], $cmd['errors'], $histogram); + } + $partial->ingestTotals($data['total_requests'], $data['total_errors']); + + $collector->mergeFrom($partial); + } + + private function probeDriverVersion(): string + { + try { + $client = ClientFactory::create((string) $this->driverConfig->driverId); + + return $client->driverVersion(); + } catch (\Throwable) { + return 'unknown'; + } + } +} diff --git a/php/src/Engine/CommandSelector.php b/php/src/Engine/CommandSelector.php new file mode 100644 index 0000000..608d139 --- /dev/null +++ b/php/src/Engine/CommandSelector.php @@ -0,0 +1,65 @@ + */ + private readonly array $commands; + + /** @var list */ + private readonly array $cumulativeWeights; + + /** + * @param list $commands + */ + public function __construct(array $commands) + { + $this->commands = $commands; + $this->cumulativeWeights = self::buildCumulativeWeights($commands); + } + + public function select(): Command + { + // mt_rand()/mt_getrandmax() gives a float in [0, 1]. + $r = mt_rand() / mt_getrandmax(); + foreach ($this->cumulativeWeights as $index => $threshold) { + if ($r <= $threshold) { + return $this->commands[$index]; + } + } + + return $this->commands[array_key_last($this->commands)]; + } + + /** + * @param list $commands + * @return list + */ + private static function buildCumulativeWeights(array $commands): array + { + $totalWeight = 0.0; + foreach ($commands as $cmd) { + $totalWeight += $cmd->weight; + } + if ($totalWeight === 0.0) { + $totalWeight = 1.0; + } + + $cumulative = []; + $sum = 0.0; + foreach ($commands as $cmd) { + $sum += $cmd->weight / $totalWeight; + $cumulative[] = $sum; + } + + return $cumulative; + } +} diff --git a/php/src/Engine/JavaRandom.php b/php/src/Engine/JavaRandom.php new file mode 100644 index 0000000..0018566 --- /dev/null +++ b/php/src/Engine/JavaRandom.php @@ -0,0 +1,103 @@ +seed = $this->initialScramble($seed); + } + + /** + * Generate the next random integer in range [0, bound). + * Matches Java's Random.nextInt(int bound). + */ + public function nextInt(int $bound): int + { + if ($bound <= 0) { + throw new InvalidArgumentException('bound must be positive'); + } + + // Special case for powers of two. + if (($bound & -$bound) === $bound) { + return (int) (($bound * $this->nextBits(31)) >> 31); + } + + // General case - rejection sampling to avoid modulo bias. + while (true) { + $bits = $this->nextBits(31); + $val = $bits % $bound; + if ($bits - $val + ($bound - 1) >= 0) { + return $val; + } + } + } + + /** + * Reset the generator with a new seed. + */ + public function setSeed(int $seed): void + { + $this->seed = $this->initialScramble($seed); + } + + private function initialScramble(int $seed): int + { + return ($seed ^ self::MULTIPLIER) & self::MASK; + } + + /** + * Generate the next `bits` random bits (1-32). + * + * Java's LCG relies on 64-bit integer overflow: it computes the full + * product `seed * MULTIPLIER` and keeps the low 48 bits. In PHP a direct + * multiply overflows the 64-bit signed int and silently becomes a float, + * corrupting the low bits. We therefore compute `(seed * MULTIPLIER) mod 2^48` + * using a 24-bit split so every partial product stays well under 2^63. + * + * seed = hi * 2^24 + lo (hi, lo < 2^24) + * seed * M โ‰ก (lo*M) + ((hi*M mod 2^24) << 24) (mod 2^48) + * + * With M = 0x5DEECE66D (< 2^35), both lo*M and hi*M are < 2^59 โ€” no overflow. + */ + private function nextBits(int $bits): int + { + $seed = $this->seed; + $lo = $seed & 0xFFFFFF; // low 24 bits + $hi = ($seed >> 24) & 0xFFFFFF; // next 24 bits + + $product = (($lo * self::MULTIPLIER) + + ((($hi * self::MULTIPLIER) & 0xFFFFFF) << 24)) & self::MASK; + + $this->seed = ($product + self::ADDEND) & self::MASK; + + return $this->seed >> (48 - $bits); + } +} diff --git a/php/src/Engine/KeyGenerator.php b/php/src/Engine/KeyGenerator.php new file mode 100644 index 0000000..feb7574 --- /dev/null +++ b/php/src/Engine/KeyGenerator.php @@ -0,0 +1,95 @@ +keyPrefix = $config->effectiveKeyPrefix(); + $this->keySizeBytes = $config->keySizeBytes; + $this->keysCount = $config->keysCount; + $this->seed = $seedOverride ?? $config->seedValue(); + $this->sequential = $config->isSequentialInt(); + $this->random = new JavaRandom($this->seed); + } + + public static function create(KeyspaceConfig $config): self + { + return new self($config); + } + + public static function createWithSeed(KeyspaceConfig $config, int $seed): self + { + return new self($config, $seed); + } + + /** + * Generate the next key. + */ + public function nextKey(): string + { + if ($this->sequential) { + $keyIndex = $this->sequentialCounter; + $this->sequentialCounter++; + } else { + $keyIndex = $this->random->nextInt($this->keysCount); + } + + $keyIndex %= $this->keysCount; + + return $this->formatKey($keyIndex); + } + + /** + * Reset the generator to its initial state. + */ + public function reset(): void + { + $this->sequentialCounter = 0; + $this->random->setSeed($this->seed); + } + + /** + * Format a key index into a full key string. + * Matches Java's String.format("%0Nd", keyIndex): zero-padded so that + * prefix + number is approximately key_size_bytes wide (minimum 1 digit). + */ + private function formatKey(int $keyIndex): string + { + $paddingWidth = max($this->keySizeBytes - strlen($this->keyPrefix), 1); + + return $this->keyPrefix . sprintf('%0' . $paddingWidth . 'd', $keyIndex); + } +} diff --git a/php/src/Engine/RateLimiter.php b/php/src/Engine/RateLimiter.php new file mode 100644 index 0000000..e7df884 --- /dev/null +++ b/php/src/Engine/RateLimiter.php @@ -0,0 +1,80 @@ +intervalNanos = intdiv(1_000_000_000, $ratePerSecond); + $this->nextAllowedNanos = self::monotonicNanos(); + } + + /** + * @return self|null null if the rate is unlimited (<= 0) + */ + public static function create(int $ratePerSecond): ?self + { + if ($ratePerSecond <= 0) { + return null; + } + + return new self($ratePerSecond); + } + + /** + * Block until one operation is allowed. + */ + public function acquire(): void + { + while (true) { + $now = self::monotonicNanos(); + if ($now >= $this->nextAllowedNanos) { + $this->nextAllowedNanos += $this->intervalNanos; + + return; + } + + $waitNanos = $this->nextAllowedNanos - $now; + // usleep takes microseconds. + $micros = intdiv($waitNanos, 1000); + if ($micros > 0) { + usleep($micros); + } + } + } + + /** + * Try to acquire without blocking. + */ + public function tryAcquire(): bool + { + $now = self::monotonicNanos(); + if ($now < $this->nextAllowedNanos) { + return false; + } + $this->nextAllowedNanos += $this->intervalNanos; + + return true; + } + + private static function monotonicNanos(): int + { + return (int) hrtime(true); + } +} diff --git a/php/src/Metrics/Collector.php b/php/src/Metrics/Collector.php new file mode 100644 index 0000000..654be4d --- /dev/null +++ b/php/src/Metrics/Collector.php @@ -0,0 +1,204 @@ +requests++; + if ($result->success) { + $latency = min($result->latencyMicros, 600_000_000); + $this->histogram ??= new HdrHistogram(1, 600_000_000, 3); + $this->histogram->record($latency); + } else { + $this->errors++; + } + } + + public function mergeFrom(self $other): void + { + $this->requests += $other->requests; + $this->errors += $other->errors; + if ($other->histogram !== null && $other->histogram->totalCount() > 0) { + $this->histogram ??= new HdrHistogram(1, 600_000_000, 3); + $this->histogram->merge($other->histogram); + } + } + + /** + * Ingest raw counts plus a fully-built histogram (cross-process reconstruction). + */ + public function ingest(int $requests, int $errors, HdrHistogram $histogram): void + { + $this->requests += $requests; + $this->errors += $errors; + if ($histogram->totalCount() > 0) { + $this->histogram ??= new HdrHistogram(1, 600_000_000, 3); + $this->histogram->merge($histogram); + } + } + + public function requests(): int + { + return $this->requests; + } + + public function errors(): int + { + return $this->errors; + } + + public function histogram(): ?HdrHistogram + { + return $this->histogram; + } + + public function count(): int + { + return $this->histogram?->totalCount() ?? 0; + } + + public function min(): int + { + return $this->histogram?->min() ?? 0; + } + + public function max(): int + { + return $this->histogram?->max() ?? 0; + } + + public function percentile(float $p): int + { + return $this->histogram?->valueAtPercentile($p) ?? 0; + } +} + +/** + * Collects benchmark metrics. Used both inside a worker (record()) and in the + * parent to merge partial collectors from all forked workers. + */ +final class Collector +{ + /** @var array */ + private array $commandMetrics = []; + private int $totalRequests = 0; + private int $totalErrors = 0; + private ?float $startTime = null; + private ?float $endTime = null; + + public function start(): void + { + $this->startTime = microtime(true); + } + + public function stop(): void + { + $this->endTime = microtime(true); + } + + public function record(CommandResult $result): void + { + $this->totalRequests++; + if (!$result->success) { + $this->totalErrors++; + } + + $metrics = $this->commandMetrics[$result->commandName] + ??= new CommandMetrics($result->commandName); + $metrics->record($result); + } + + /** + * Merge another collector's per-command metrics and totals into this one. + */ + public function mergeFrom(self $other): void + { + $this->totalRequests += $other->totalRequests; + $this->totalErrors += $other->totalErrors; + + foreach ($other->commandMetrics as $name => $metrics) { + $merged = $this->commandMetrics[$name] ??= new CommandMetrics($name); + $merged->mergeFrom($metrics); + } + } + + /** + * Ingest a reconstructed command's counts + histogram (used when rebuilding a + * partial collector from a forked worker's serialized payload). + */ + public function ingestCommand(string $name, int $requests, int $errors, HdrHistogram $histogram): void + { + $metrics = $this->commandMetrics[$name] ??= new CommandMetrics($name); + $metrics->ingest($requests, $errors, $histogram); + } + + public function ingestTotals(int $requests, int $errors): void + { + $this->totalRequests += $requests; + $this->totalErrors += $errors; + } + + public function totalRequests(): int + { + return $this->totalRequests; + } + + public function totalErrors(): int + { + return $this->totalErrors; + } + + public function startTime(): ?float + { + return $this->startTime; + } + + public function endTime(): ?float + { + return $this->endTime; + } + + public function setStartTime(float $t): void + { + $this->startTime = $t; + } + + public function setEndTime(float $t): void + { + $this->endTime = $t; + } + + public function durationMillis(): int + { + if ($this->startTime === null || $this->endTime === null) { + return 0; + } + + return (int) (($this->endTime - $this->startTime) * 1000); + } + + /** + * @return array + */ + public function allMetrics(): array + { + return $this->commandMetrics; + } +} diff --git a/php/src/Metrics/HdrEncoder.php b/php/src/Metrics/HdrEncoder.php new file mode 100644 index 0000000..d497fc7 --- /dev/null +++ b/php/src/Metrics/HdrEncoder.php @@ -0,0 +1,155 @@ +significantFigures; + $lowest = $histogram->lowestTrackableValue; + $highest = $histogram->highestTrackableValue; + $conversionRatioBits = self::doubleToLongBits(1.0); + + $countsBytes = self::encodeCounts($histogram); + + // payload_len = everything after the payload_len field itself: + // normalizing(4) + sigfigs(4) + lowest(8) + highest(8) + ratio(8) + counts + $payloadLen = 4 + 4 + 8 + 8 + 8 + strlen($countsBytes); + + $header = pack('N4', self::V2_ENCODING_COOKIE, $payloadLen, $normalizingOffset, $sigFigs); + $int64Fields = self::packInt64BE($lowest) + . self::packInt64BE($highest) + . self::packInt64BE($conversionRatioBits); + + return $header . $int64Fields . $countsBytes; + } + + private static function encodeCounts(HdrHistogram $histogram): string + { + $relevantLength = $histogram->relevantLength(); + $result = ''; + $index = 0; + + while ($index < $relevantLength) { + $count = $histogram->rawCountAt($index); + if ($count === 0) { + $zeros = 1; + while (($index + $zeros) < $relevantLength && $histogram->rawCountAt($index + $zeros) === 0) { + $zeros++; + } + $result .= self::encodeZigZag(-$zeros); + $index += $zeros; + } else { + $result .= self::encodeZigZag($count); + $index++; + } + } + + return $result; + } + + /** + * ZigZag + LEB128 encode a signed 64-bit integer. + */ + private static function encodeZigZag(int $value): string + { + // ZigZag: (value << 1) ^ (value >> 63), using arithmetic shift semantics. + $zz = ($value << 1) ^ ($value >> 63); + + $result = ''; + // Process as unsigned 64-bit via logical right shift. + while (true) { + if (($zz & ~0x7F) === 0) { + $result .= chr($zz & 0x7F); + break; + } + $result .= chr(($zz & 0x7F) | 0x80); + $zz = self::logicalShiftRight($zz, 7); + } + + return $result; + } + + /** + * Logical (unsigned) right shift for 64-bit ints in PHP. + */ + private static function logicalShiftRight(int $value, int $bits): int + { + if ($bits === 0) { + return $value; + } + + // Shift then clear the top `bits` sign-extended bits. + return ($value >> $bits) & (PHP_INT_MAX >> ($bits - 1)); + } + + private static function doubleToLongBits(float $value): int + { + // Pack as big-endian double, unpack as signed big-endian int64. + $packed = pack('E', $value); // 'E' = big-endian double + /** @var array{1:int} $unpacked */ + $unpacked = unpack('J', $packed); // 'J' = big-endian unsigned int64 + + return $unpacked[1]; + } + + private static function packInt64BE(int $value): string + { + // 'J' handles the full 64-bit range; PHP ints are 64-bit signed. + return pack('J', $value); + } +} diff --git a/php/src/Metrics/HdrHistogram.php b/php/src/Metrics/HdrHistogram.php new file mode 100644 index 0000000..dc753b4 --- /dev/null +++ b/php/src/Metrics/HdrHistogram.php @@ -0,0 +1,255 @@ + sparse index => count */ + private array $counts = []; + + private int $totalCount = 0; + private int $minNonZeroValue = PHP_INT_MAX; + private int $maxValue = 0; + + public function __construct( + int $lowestTrackableValue = 1, + int $highestTrackableValue = 600_000_000, + int $significantFigures = 3, + ) { + $this->lowestTrackableValue = $lowestTrackableValue; + $this->highestTrackableValue = $highestTrackableValue; + $this->significantFigures = $significantFigures; + + $largestValueWithSingleUnitResolution = 2 * (int) (10 ** $significantFigures); + $subBucketCountMagnitude = (int) ceil(log($largestValueWithSingleUnitResolution, 2)); + $this->subBucketHalfCountMagnitude = max($subBucketCountMagnitude, 1) - 1; + + $this->unitMagnitude = (int) floor(log($lowestTrackableValue, 2)); + + $this->subBucketCount = 2 ** ($this->subBucketHalfCountMagnitude + 1); + $this->subBucketHalfCount = $this->subBucketCount >> 1; + $this->subBucketMask = ($this->subBucketCount - 1) << $this->unitMagnitude; + + // Number of buckets needed to cover the highest trackable value. + $smallestUntrackableValue = $this->subBucketCount << $this->unitMagnitude; + $bucketsNeeded = 1; + while ($smallestUntrackableValue < $highestTrackableValue) { + if ($smallestUntrackableValue > (PHP_INT_MAX >> 1)) { + $bucketsNeeded++; + break; + } + $smallestUntrackableValue <<= 1; + $bucketsNeeded++; + } + $this->bucketCount = $bucketsNeeded; + $this->countsLen = ($this->bucketCount + 1) * ($this->subBucketCount >> 1); + } + + public function record(int $value): void + { + if ($value < 0) { + return; + } + $index = $this->countsIndexFor($value); + $this->counts[$index] = ($this->counts[$index] ?? 0) + 1; + $this->totalCount++; + + if ($value > $this->maxValue) { + $this->maxValue = $value; + } + if ($value !== 0 && $value < $this->minNonZeroValue) { + $this->minNonZeroValue = $value; + } + } + + public function recordValueWithCount(int $value, int $count): void + { + if ($value < 0 || $count <= 0) { + return; + } + $index = $this->countsIndexFor($value); + $this->counts[$index] = ($this->counts[$index] ?? 0) + $count; + $this->totalCount += $count; + + if ($value > $this->maxValue) { + $this->maxValue = $value; + } + if ($value !== 0 && $value < $this->minNonZeroValue) { + $this->minNonZeroValue = $value; + } + } + + public function merge(self $other): void + { + for ($i = 0; $i < $other->countsLen; $i++) { + $count = $other->rawCountAt($i); + if ($count > 0) { + $value = $other->valueFromIndex($i); + $this->recordValueWithCount($value, $count); + } + } + } + + public function totalCount(): int + { + return $this->totalCount; + } + + public function min(): int + { + if ($this->totalCount === 0) { + return 0; + } + + // If only zero-valued samples were recorded, min is 0. + return $this->minNonZeroValue === PHP_INT_MAX ? 0 : $this->minNonZeroValue; + } + + public function max(): int + { + return $this->maxValue; + } + + public function valueAtPercentile(float $percentile): int + { + if ($this->totalCount === 0) { + return 0; + } + + $requestedPercentile = min(max($percentile, 0.0), 100.0); + $countAtPercentile = (int) ceil(($requestedPercentile / 100.0) * $this->totalCount); + $countAtPercentile = max($countAtPercentile, 1); + + $total = 0; + for ($i = 0; $i < $this->countsLen; $i++) { + $total += $this->rawCountAt($i); + if ($total >= $countAtPercentile) { + $valueAtIndex = $this->valueFromIndex($i); + + return $this->highestEquivalentValue($valueAtIndex); + } + } + + return $this->maxValue; + } + + public function rawCountAt(int $index): int + { + return $this->counts[$index] ?? 0; + } + + /** + * Highest index with a non-zero count, plus one (the "relevant length"). + */ + public function relevantLength(): int + { + if ($this->counts === []) { + return 0; + } + + return max(array_keys($this->counts)) + 1; + } + + // --- HdrHistogram index math (mirrors the Java implementation) --- + + private function countsIndexFor(int $value): int + { + $bucketIndex = $this->bucketIndexFor($value); + $subBucketIndex = $this->subBucketIndexFor($value, $bucketIndex); + + return $this->countsIndex($bucketIndex, $subBucketIndex); + } + + private function bucketIndexFor(int $value): int + { + $pow2ceiling = self::bitLength($value | $this->subBucketMask); + + return $pow2ceiling - $this->unitMagnitude - ($this->subBucketHalfCountMagnitude + 1); + } + + private function subBucketIndexFor(int $value, int $bucketIndex): int + { + return $value >> ($bucketIndex + $this->unitMagnitude); + } + + private function countsIndex(int $bucketIndex, int $subBucketIndex): int + { + $bucketBaseIndex = ($bucketIndex + 1) << $this->subBucketHalfCountMagnitude; + $offsetInBucket = $subBucketIndex - $this->subBucketHalfCount; + + return $bucketBaseIndex + $offsetInBucket; + } + + public function valueFromIndex(int $index): int + { + $bucketIndex = ($index >> $this->subBucketHalfCountMagnitude) - 1; + $subBucketIndex = ($index & ($this->subBucketHalfCount - 1)) + $this->subBucketHalfCount; + + if ($bucketIndex < 0) { + $subBucketIndex -= $this->subBucketHalfCount; + $bucketIndex = 0; + } + + return $subBucketIndex << ($bucketIndex + $this->unitMagnitude); + } + + private function highestEquivalentValue(int $value): int + { + return $this->nextNonEquivalentValue($value) - 1; + } + + private function nextNonEquivalentValue(int $value): int + { + return $this->lowestEquivalentValue($value) + $this->sizeOfEquivalentValueRange($value); + } + + private function lowestEquivalentValue(int $value): int + { + $bucketIndex = $this->bucketIndexFor($value); + $subBucketIndex = $this->subBucketIndexFor($value, $bucketIndex); + + return $subBucketIndex << ($bucketIndex + $this->unitMagnitude); + } + + private function sizeOfEquivalentValueRange(int $value): int + { + $bucketIndex = $this->bucketIndexFor($value); + + return 1 << ($this->unitMagnitude + $bucketIndex); + } + + private static function bitLength(int $value): int + { + $length = 0; + while ($value > 0) { + $value >>= 1; + $length++; + } + + return $length; + } +} diff --git a/php/src/Metrics/NdjsonWriter.php b/php/src/Metrics/NdjsonWriter.php new file mode 100644 index 0000000..151f8c3 --- /dev/null +++ b/php/src/Metrics/NdjsonWriter.php @@ -0,0 +1,156 @@ +commitId = $commitId; + $this->driverId = $driverId; + $this->primaryDriverVersion = $primaryDriverVersion; + $this->secondaryDriverId = $secondaryDriverId; + $this->secondaryDriverVersion = $secondaryDriverVersion; + } + + public function writePhaseResults(string $phaseId, string $status, int $connections, Collector $collector): void + { + $dir = dirname($this->outputPath); + if (!is_dir($dir) && !mkdir($dir, 0o777, true) && !is_dir($dir)) { + throw new RuntimeException("Cannot create output directory: {$dir}"); + } + + $json = $this->buildPhaseJson($phaseId, $status, $connections, $collector); + $line = json_encode($json, JSON_THROW_ON_ERROR | JSON_UNESCAPED_SLASHES); + + file_put_contents($this->outputPath, $line . "\n", FILE_APPEND | LOCK_EX); + } + + /** + * @return array + */ + private function buildPhaseJson(string $phaseId, string $status, int $connections, Collector $collector): array + { + $result = []; + + if ($this->commitId !== null || $this->driverId !== null) { + $metadata = []; + if ($this->commitId !== null) { + $metadata['commit_id'] = $this->commitId; + } + $metadata['timestamp'] = gmdate('Y-m-d\TH:i:s\Z'); + if ($this->driverId !== null) { + $metadata['driver_id'] = $this->driverId; + } + if ($this->primaryDriverVersion !== null) { + $metadata['primary_driver_version'] = $this->primaryDriverVersion; + } + if ($this->secondaryDriverId !== null) { + $metadata['secondary_driver_id'] = $this->secondaryDriverId; + } + if ($this->secondaryDriverVersion !== null) { + $metadata['secondary_driver_version'] = $this->secondaryDriverVersion; + } + $result['metadata'] = $metadata; + } + + $result['phase'] = [ + 'id' => $phaseId, + 'status' => $status, + 'start_timestamp' => self::iso8601($collector->startTime()), + 'finish_timestamp' => self::iso8601($collector->endTime()), + 'duration_ms' => $collector->durationMillis(), + 'connections' => $connections, + ]; + + $result['totals'] = [ + 'requests' => $collector->totalRequests(), + 'errors' => $collector->totalErrors(), + ]; + + $result['metrics'] = $this->buildCommandMetrics($collector); + + return $result; + } + + /** + * @return array + */ + private function buildCommandMetrics(Collector $collector): array + { + $metrics = []; + + foreach ($collector->allMetrics() as $name => $cmd) { + $data = [ + 'requests' => $cmd->requests(), + 'errors' => $cmd->errors(), + 'latency' => [ + 'unit' => 'us', + 'count' => $cmd->count(), + 'summary' => [ + 'min' => $cmd->min(), + 'p50' => $cmd->percentile(50), + 'p95' => $cmd->percentile(95), + 'p99' => $cmd->percentile(99), + 'p999' => $cmd->percentile(99.9), + 'max' => $cmd->max(), + ], + ], + ]; + + $histogram = $cmd->histogram(); + if ($histogram !== null) { + $data['latency']['hdr'] = [ + 'format' => 'hdr', + 'sigfig' => 3, + 'payload_b64' => self::encodeHistogram($histogram), + ]; + } + + $metrics[$name] = $data; + } + + return $metrics; + } + + private static function encodeHistogram(HdrHistogram $histogram): string + { + try { + return HdrEncoder::encodeCompressedBase64($histogram); + } catch (\Throwable) { + return ''; + } + } + + private static function iso8601(?float $time): ?string + { + if ($time === null) { + return null; + } + + return gmdate('Y-m-d\TH:i:s\Z', (int) $time); + } +} diff --git a/php/src/Version.php b/php/src/Version.php new file mode 100644 index 0000000..71610fc --- /dev/null +++ b/php/src/Version.php @@ -0,0 +1,10 @@ +> + */ + private function runEngine(WorkloadConfig $workload, ?string &$metricsPath = null): array + { + $metricsPath = sys_get_temp_dir() . '/resp_bench_php_it_' . uniqid('', true) . '.ndjson'; + + $engine = new Benchmark( + host: 'localhost', + port: 6379, + driverConfig: $this->recordingDriver(), + workloadConfig: $workload, + metricsPath: $metricsPath, + commitId: 'it-test', + concurrencyMode: 'inline', + ); + $engine->run(); + + $lines = array_values(array_filter(explode("\n", (string) file_get_contents($metricsPath)))); + unlink($metricsPath); + + return array_map( + static fn (string $l): array => json_decode($l, true, 512, JSON_THROW_ON_ERROR), + $lines, + ); + } +} diff --git a/php/tests/Integration/ErrorMetricsTest.php b/php/tests/Integration/ErrorMetricsTest.php new file mode 100644 index 0000000..4404d52 --- /dev/null +++ b/php/tests/Integration/ErrorMetricsTest.php @@ -0,0 +1,104 @@ +runWithErrorRate(1.0, 200); + + self::assertSame(200, $phase['totals']['requests']); + self::assertSame(200, $phase['totals']['errors']); + // Per-command error count matches. + self::assertSame(200, $phase['metrics']['SET']['requests']); + self::assertSame(200, $phase['metrics']['SET']['errors']); + } + + public function testZeroErrorRateProducesNoErrors(): void + { + $phase = $this->runWithErrorRate(0.0, 200); + + self::assertSame(200, $phase['totals']['requests']); + self::assertSame(0, $phase['totals']['errors']); + self::assertSame(0, $phase['metrics']['SET']['errors']); + } + + public function testPartialErrorRateIsApproximatelyHonored(): void + { + $target = 0.25; + $requests = 4000; + $phase = $this->runWithErrorRate($target, $requests); + + self::assertSame($requests, $phase['totals']['requests']); + + $errorFraction = $phase['totals']['errors'] / $requests; + // Statistical: expect ~25%, allow +/- 5 points. + self::assertEqualsWithDelta($target, $errorFraction, 0.05); + } + + public function testErrorsExcludedFromLatencyHistogramCount(): void + { + // With a 100% error rate, no successful latencies are recorded, so the + // per-command latency count is 0 while requests/errors are full. + $phase = $this->runWithErrorRate(1.0, 100); + + self::assertSame(100, $phase['metrics']['SET']['requests']); + self::assertSame(100, $phase['metrics']['SET']['errors']); + self::assertSame(0, $phase['metrics']['SET']['latency']['count']); + } + + /** + * @return array the single phase object + */ + private function runWithErrorRate(float $errorRate, int $requests): array + { + $driver = new DriverConfig( + driverId: 'recording', + mode: 'standalone', + specificDriverConfig: ['error_rate' => $errorRate, 'error_message' => 'Simulated failure'], + ); + + $workload = \RespBench\Config\Loader::parseWorkloadConfigString(<<run(); + + $line = trim((string) file_get_contents($metricsPath)); + unlink($metricsPath); + + return json_decode($line, true, 512, JSON_THROW_ON_ERROR); + } +} diff --git a/php/tests/Integration/LiveClientTest.php b/php/tests/Integration/LiveClientTest.php new file mode 100644 index 0000000..f11e367 --- /dev/null +++ b/php/tests/Integration/LiveClientTest.php @@ -0,0 +1,177 @@ +host = getenv('VALKEY_HOST') ?: 'localhost'; + $this->port = (int) (getenv('VALKEY_PORT') ?: '6379'); + + if (!extension_loaded('valkey_glide')) { + self::markTestSkipped('valkey_glide extension not loaded'); + } + if (!$this->serverReachable()) { + self::markTestSkipped("Server not reachable at {$this->host}:{$this->port}"); + } + } + + private function serverReachable(): bool + { + $errno = 0; + $errstr = ''; + $conn = @fsockopen($this->host, $this->port, $errno, $errstr, 1.0); + if ($conn === false) { + return false; + } + fclose($conn); + + return true; + } + + private function connect(): BenchmarkClient + { + $config = new DriverConfig(driverId: 'valkey-glide-php', mode: 'standalone'); + + return Factory::createAndConnect($this->host, $this->port, $config); + } + + public function testConnects(): void + { + $client = $this->connect(); + try { + self::assertTrue($client->isConnected()); + } finally { + $client->close(); + } + } + + public function testPing(): void + { + $client = $this->connect(); + try { + $result = $client->ping(); + self::assertTrue($result->isSuccess(), 'PING failed'); + self::assertGreaterThan(0, $result->latencyMicros); + } finally { + $client->close(); + } + } + + public function testSetGetRoundTrip(): void + { + $client = $this->connect(); + try { + $key = 'php-live-test-key'; + $value = 'value-' . time(); + + $set = $client->set($key, $value); + self::assertTrue($set->isSuccess(), 'SET failed'); + + $get = $client->get($key); + self::assertTrue($get->isSuccess(), 'GET failed'); + self::assertSame($value, $get->value); + + $client->del($key); + } finally { + $client->close(); + } + } + + public function testDriverVersionIsNonEmpty(): void + { + $client = $this->connect(); + try { + $version = $client->driverVersion(); + self::assertNotSame('', $version); + self::assertNotSame('unknown', $version); + } finally { + $client->close(); + } + } + + /** + * End-to-end multi-process (fork) run against a live server. This is the key + * fork-then-connect validation: N workers each open their own connection + * AFTER forking. Asserts totals add up and latencies are realistic (> 0). + */ + public function testMultiProcessLiveRun(): void + { + if (!function_exists('pcntl_fork')) { + self::markTestSkipped('pcntl not available'); + } + + $metricsPath = sys_get_temp_dir() . '/resp_bench_php_live_' . uniqid('', true) . '.ndjson'; + + $driver = new DriverConfig(driverId: 'valkey-glide-php', mode: 'standalone'); + $workload = Loader::loadWorkloadConfig(__DIR__ . '/../fixtures/smoke-workload.json'); + + $engine = new Benchmark( + host: $this->host, + port: $this->port, + driverConfig: $driver, + workloadConfig: $workload, + metricsPath: $metricsPath, + commitId: 'live-test', + concurrencyMode: 'process', + ); + + try { + $engine->run(); + + $lines = array_values(array_filter(explode("\n", (string) file_get_contents($metricsPath)))); + self::assertCount(2, $lines, 'Expected two phases'); + + $phases = array_map( + static fn (string $l): array => json_decode($l, true, 512, JSON_THROW_ON_ERROR), + $lines, + ); + + // WARMUP: 400 requests, no errors. + self::assertSame('WARMUP', $phases[0]['phase']['id']); + self::assertSame(400, $phases[0]['totals']['requests']); + self::assertSame(0, $phases[0]['totals']['errors']); + + // STEADY: 1000 requests total, no errors, GET+SET sum to total. + self::assertSame('STEADY', $phases[1]['phase']['id']); + self::assertSame(1000, $phases[1]['totals']['requests']); + self::assertSame(0, $phases[1]['totals']['errors']); + $sum = $phases[1]['metrics']['GET']['requests'] + $phases[1]['metrics']['SET']['requests']; + self::assertSame(1000, $sum); + + // Real server latencies should be > 0 microseconds at some percentile. + $getMax = $phases[1]['metrics']['GET']['latency']['summary']['max']; + self::assertGreaterThan(0, $getMax, 'Expected non-zero GET latency against a live server'); + } finally { + if (is_file($metricsPath)) { + unlink($metricsPath); + } + } + } +} diff --git a/php/tests/Integration/MetricsOutputTest.php b/php/tests/Integration/MetricsOutputTest.php new file mode 100644 index 0000000..109cf0d --- /dev/null +++ b/php/tests/Integration/MetricsOutputTest.php @@ -0,0 +1,155 @@ +parseWorkload($this->twoPhaseWorkload()); + $phases = $this->runEngine($workload); + + self::assertCount(2, $phases); + + foreach ($phases as $p) { + // Metadata block. + self::assertArrayHasKey('metadata', $p); + self::assertSame('it-test', $p['metadata']['commit_id']); + self::assertSame('recording', $p['metadata']['driver_id']); + self::assertArrayHasKey('timestamp', $p['metadata']); + self::assertMatchesRegularExpression( + '/^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}Z$/', + $p['metadata']['timestamp'], + ); + + // Phase block. + self::assertArrayHasKey('phase', $p); + self::assertContains($p['phase']['id'], ['WARMUP', 'STEADY']); + self::assertSame('COMPLETED', $p['phase']['status']); + self::assertArrayHasKey('duration_ms', $p['phase']); + self::assertArrayHasKey('connections', $p['phase']); + + // Totals block. + self::assertArrayHasKey('totals', $p); + self::assertArrayHasKey('requests', $p['totals']); + self::assertArrayHasKey('errors', $p['totals']); + } + } + + public function testRequestCountsAreExact(): void + { + $workload = $this->parseWorkload($this->twoPhaseWorkload()); + $phases = $this->runEngine($workload); + + // WARMUP: 400 SET requests. + self::assertSame(400, $phases[0]['totals']['requests']); + self::assertSame(400, $phases[0]['metrics']['SET']['requests']); + + // STEADY: 1000 total; GET + SET requests sum to total. + self::assertSame(1000, $phases[1]['totals']['requests']); + $sum = $phases[1]['metrics']['GET']['requests'] + $phases[1]['metrics']['SET']['requests']; + self::assertSame(1000, $sum); + + // Per-command latency count equals its (successful) request count. + self::assertSame( + $phases[1]['metrics']['GET']['requests'], + $phases[1]['metrics']['GET']['latency']['count'], + ); + } + + public function testHdrBlockShapeAndUnit(): void + { + $workload = $this->parseWorkload($this->twoPhaseWorkload()); + $phases = $this->runEngine($workload); + + $latency = $phases[1]['metrics']['GET']['latency']; + self::assertSame('us', $latency['unit']); + + foreach (['min', 'p50', 'p95', 'p99', 'p999', 'max'] as $k) { + self::assertArrayHasKey($k, $latency['summary']); + self::assertIsInt($latency['summary'][$k]); + } + + $hdr = $latency['hdr']; + self::assertSame('hdr', $hdr['format']); + self::assertSame(3, $hdr['sigfig']); + self::assertNotSame('', $hdr['payload_b64']); + + // Payload is a valid base64 V2 compressed HDR blob. + $raw = base64_decode($hdr['payload_b64'], true); + self::assertIsString($raw); + /** @var array{cookie:int} $wrapper */ + $wrapper = unpack('Ncookie', substr($raw, 0, 4)); + self::assertSame(0x1c849314, $wrapper['cookie']); + } + + /** + * Latency-distribution accuracy: with a known latency mix, the HDR summary + * percentiles must land in the right buckets. This exercises the same + * histogram + encoder used by the NDJSON writer. + */ + public function testLatencyPercentileAccuracy(): void + { + $h = new HdrHistogram(1, 600_000_000, 3); + // 95 samples at ~100us, 4 at ~1000us, 1 at ~10000us. + for ($i = 0; $i < 95; $i++) { + $h->record(100); + } + for ($i = 0; $i < 4; $i++) { + $h->record(1000); + } + $h->record(10000); + + self::assertEqualsWithDelta(100, $h->valueAtPercentile(50), 1); + self::assertEqualsWithDelta(100, $h->valueAtPercentile(90), 1); + self::assertEqualsWithDelta(1000, $h->valueAtPercentile(99), 10); + self::assertEqualsWithDelta(10000, $h->valueAtPercentile(100), 50); + + // Round-trips through the encoder without error. + $b64 = HdrEncoder::encodeCompressedBase64($h); + self::assertNotSame('', $b64); + } + + private function twoPhaseWorkload(): string + { + return <<host = getenv('VALKEY_HOST') ?: 'localhost'; + $this->port = (int) (getenv('VALKEY_PORT') ?: '6379'); + + if (!extension_loaded('redis')) { + self::markTestSkipped('redis (phpredis) extension not loaded'); + } + if (!$this->serverReachable()) { + self::markTestSkipped("Server not reachable at {$this->host}:{$this->port}"); + } + } + + private function serverReachable(): bool + { + $conn = @fsockopen($this->host, $this->port, $errno, $errstr, 1.0); + if ($conn === false) { + return false; + } + fclose($conn); + + return true; + } + + private function connect(): BenchmarkClient + { + return Factory::createAndConnect( + $this->host, + $this->port, + new DriverConfig(driverId: 'phpredis', mode: 'standalone'), + ); + } + + public function testConnectsAndPings(): void + { + $client = $this->connect(); + try { + self::assertTrue($client->isConnected()); + $ping = $client->ping(); + self::assertTrue($ping->isSuccess()); + } finally { + $client->close(); + } + } + + public function testSetGetRoundTrip(): void + { + $client = $this->connect(); + try { + $key = 'phpredis-live-test-key'; + $value = 'value-' . time(); + + self::assertTrue($client->set($key, $value)->isSuccess()); + + $get = $client->get($key); + self::assertTrue($get->isSuccess()); + self::assertSame($value, $get->value); + + $client->del($key); + } finally { + $client->close(); + } + } + + public function testDriverVersionIsNonEmpty(): void + { + $client = $this->connect(); + try { + $version = $client->driverVersion(); + self::assertNotSame('', $version); + self::assertNotSame('unknown', $version); + } finally { + $client->close(); + } + } +} diff --git a/php/tests/Integration/RateLimitingTest.php b/php/tests/Integration/RateLimitingTest.php new file mode 100644 index 0000000..8b6174a --- /dev/null +++ b/php/tests/Integration/RateLimitingTest.php @@ -0,0 +1,151 @@ +parseWorkload($this->rpsWorkload($targetRps, $targetRequests)); + $phases = $this->runEngine($workload); + $phase = $phases[0]; + + // Exact request count. + self::assertSame($targetRequests, $phase['totals']['requests']); + + $durationMs = $phase['phase']['duration_ms']; + self::assertEqualsWithDelta( + $expectedDurationMs, + $durationMs, + $expectedDurationMs * self::RATE_TOLERANCE, + "Duration {$durationMs}ms should be ~{$expectedDurationMs}ms", + ); + + $actualRate = $phase['totals']['requests'] / ($durationMs / 1000.0); + self::assertEqualsWithDelta($targetRps, $actualRate, $targetRps * self::RATE_TOLERANCE); + } + + public function testSharedRpsLimitAcrossConnections(): void + { + // 50 rps shared across 4 connections. This only behaves as a *shared* + // limit under real concurrency, so run in process (fork) mode where the + // 4 workers execute in parallel. Each worker gets 50/4 rps and 100/4 + // requests -> ~2s wall clock, not 4x that. + if (!function_exists('pcntl_fork')) { + self::markTestSkipped('pcntl not available (shared-rate semantics need concurrent workers)'); + } + + $targetRps = 50; + $connections = 4; + $targetRequests = 100; + $expectedDurationMs = ($targetRequests * 1000) / $targetRps; // ~2000ms + + $workload = $this->parseWorkload($this->rpsWorkload($targetRps, $targetRequests, $connections)); + + $metricsPath = sys_get_temp_dir() . '/resp_bench_php_sharedrps_' . uniqid('', true) . '.ndjson'; + $engine = new \RespBench\Engine\Benchmark( + host: 'localhost', + port: 6379, + driverConfig: $this->recordingDriver(), + workloadConfig: $workload, + metricsPath: $metricsPath, + commitId: 'it-test', + concurrencyMode: 'process', + ); + + $start = hrtime(true); + $engine->run(); + $wallMs = (hrtime(true) - $start) / 1_000_000; + + $line = trim((string) file_get_contents($metricsPath)); + unlink($metricsPath); + $phase = json_decode($line, true, 512, JSON_THROW_ON_ERROR); + + self::assertSame($connections, $phase['phase']['connections']); + self::assertSame($targetRequests, $phase['totals']['requests']); + + // Concurrent workers -> aggregate honors the shared limit: wall clock is + // ~2s, not ~8s. Allow generous tolerance for fork + CI jitter. + self::assertGreaterThan( + $expectedDurationMs * 0.7, + $wallMs, + "Wall clock {$wallMs}ms too fast โ€” shared limit not enforced", + ); + self::assertLessThan( + $expectedDurationMs * 2.5, + $wallMs, + "Wall clock {$wallMs}ms too slow โ€” workers not running concurrently (shared limit)", + ); + } + + public function testNoRateLimitAllowsMaximumThroughput(): void + { + $targetRequests = 5000; + $workload = $this->parseWorkload($this->rpsWorkload(-1, $targetRequests)); + $phases = $this->runEngine($workload); + $phase = $phases[0]; + + self::assertSame($targetRequests, $phase['totals']['requests']); + $durationMs = $phase['phase']['duration_ms']; + self::assertLessThan(1000, $durationMs, "Unlimited run should be fast (<1s), was {$durationMs}ms"); + } + + public function testUnlimitedMuchFasterThanRateLimited(): void + { + $targetRequests = 100; + $rps = 50; + + $limited = $this->runEngine($this->parseWorkload($this->rpsWorkload($rps, $targetRequests))); + $unlimited = $this->runEngine($this->parseWorkload($this->rpsWorkload(-1, $targetRequests))); + + $limitedMs = $limited[0]['phase']['duration_ms']; + $unlimitedMs = $unlimited[0]['phase']['duration_ms']; + + $expectedLimitedMs = ($targetRequests * 1000) / $rps; // ~2000ms + self::assertEqualsWithDelta($expectedLimitedMs, $limitedMs, $expectedLimitedMs * self::RATE_TOLERANCE); + + // Unlimited should be dramatically faster. + self::assertLessThan( + $limitedMs / 5, + $unlimitedMs, + "Unlimited ({$unlimitedMs}ms) should be far faster than rate-limited ({$limitedMs}ms)", + ); + } + + private function rpsWorkload(int $rpsLimit, int $requests, int $connections = 1): string + { + return <<metricsPath = sys_get_temp_dir() . '/resp_bench_php_test_' . uniqid('', true) . '.ndjson'; + } + + protected function tearDown(): void + { + if ($this->metricsPath !== '' && is_file($this->metricsPath)) { + unlink($this->metricsPath); + } + } + + private function runWorkload(string $mode): array + { + $fixtures = __DIR__ . '/../fixtures'; + $driver = Loader::loadDriverConfig($fixtures . '/recording-driver.json'); + $workload = Loader::loadWorkloadConfig($fixtures . '/smoke-workload.json'); + + $engine = new Benchmark( + host: 'localhost', + port: 6379, + driverConfig: $driver, + workloadConfig: $workload, + metricsPath: $this->metricsPath, + commitId: 'testsha', + concurrencyMode: $mode, + ); + $engine->run(); + + $lines = array_values(array_filter(explode("\n", (string) file_get_contents($this->metricsPath)))); + + return array_map( + static fn (string $line): array => json_decode($line, true, 512, JSON_THROW_ON_ERROR), + $lines, + ); + } + + public function testInlineModeProducesValidNdjson(): void + { + $phases = $this->runWorkload('inline'); + $this->assertSchema($phases); + } + + public function testProcessModeProducesValidNdjson(): void + { + if (!function_exists('pcntl_fork')) { + self::markTestSkipped('pcntl not available'); + } + + $phases = $this->runWorkload('process'); + $this->assertSchema($phases); + } + + public function testInlineAndProcessAgreeOnTotals(): void + { + if (!function_exists('pcntl_fork')) { + self::markTestSkipped('pcntl not available'); + } + + $inline = $this->runWorkload('inline'); + // Reset output between runs. + unlink($this->metricsPath); + $process = $this->runWorkload('process'); + + // Totals are deterministic (request-based completion), independent of mode. + foreach ([0, 1] as $i) { + self::assertSame( + $inline[$i]['totals']['requests'], + $process[$i]['totals']['requests'], + "Phase {$i} request totals differ between modes", + ); + } + } + + private function assertSchema(array $phases): void + { + self::assertCount(2, $phases, 'Expected two phases (WARMUP, STEADY)'); + + // Phase 0: WARMUP, 400 requests, all SET. + $warmup = $phases[0]; + self::assertSame('WARMUP', $warmup['phase']['id']); + self::assertSame('COMPLETED', $warmup['phase']['status']); + self::assertSame(400, $warmup['totals']['requests']); + self::assertSame(0, $warmup['totals']['errors']); + self::assertArrayHasKey('SET', $warmup['metrics']); + + // Metadata + self::assertSame('testsha', $warmup['metadata']['commit_id']); + self::assertSame('recording', $warmup['metadata']['driver_id']); + + // Phase 1: STEADY, 1000 requests total, GET + SET. + $steady = $phases[1]; + self::assertSame('STEADY', $steady['phase']['id']); + self::assertSame(1000, $steady['totals']['requests']); + self::assertArrayHasKey('GET', $steady['metrics']); + self::assertArrayHasKey('SET', $steady['metrics']); + + // GET + SET request counts sum to the total. + $sum = $steady['metrics']['GET']['requests'] + $steady['metrics']['SET']['requests']; + self::assertSame(1000, $sum); + + // Latency block schema. + $latency = $steady['metrics']['GET']['latency']; + self::assertSame('us', $latency['unit']); + self::assertArrayHasKey('summary', $latency); + foreach (['min', 'p50', 'p95', 'p99', 'p999', 'max'] as $k) { + self::assertArrayHasKey($k, $latency['summary']); + self::assertIsInt($latency['summary'][$k]); + } + self::assertSame('hdr', $latency['hdr']['format']); + self::assertSame(3, $latency['hdr']['sigfig']); + self::assertNotSame('', $latency['hdr']['payload_b64']); + } +} diff --git a/php/tests/Unit/CommandSelectorTest.php b/php/tests/Unit/CommandSelectorTest.php new file mode 100644 index 0000000..9ca396e --- /dev/null +++ b/php/tests/Unit/CommandSelectorTest.php @@ -0,0 +1,43 @@ +select()->name); + } + } + + public function testWeightedDistributionApproximatelyHolds(): void + { + $commands = CommandFactory::createAll([ + new CommandConfig('get', 0.8), + new CommandConfig('set', 0.2, 64), + ]); + $selector = new CommandSelector($commands); + + $counts = ['GET' => 0, 'SET' => 0]; + $n = 20000; + for ($i = 0; $i < $n; $i++) { + $counts[$selector->select()->name]++; + } + + $getRatio = $counts['GET'] / $n; + // Expect ~0.8; allow +/- 0.05 for randomness. + self::assertGreaterThan(0.72, $getRatio); + self::assertLessThan(0.88, $getRatio); + } +} diff --git a/php/tests/Unit/ConfigLoaderTest.php b/php/tests/Unit/ConfigLoaderTest.php new file mode 100644 index 0000000..5edb041 --- /dev/null +++ b/php/tests/Unit/ConfigLoaderTest.php @@ -0,0 +1,107 @@ +driverId); + self::assertSame('standalone', $config->mode); + self::assertTrue($config->isStandalone()); + self::assertSame(8, $config->specificDriverConfig['pool_size']); + } + + public function testDriverConfigDefaults(): void + { + $config = Loader::parseDriverConfigString('{}'); + self::assertSame('1.0', $config->schemaVersion); + self::assertSame('standalone', $config->mode); + self::assertNull($config->driverId); + self::assertSame([], $config->specificDriverConfig); + } + + public function testClusterMode(): void + { + $config = Loader::parseDriverConfigString('{"driver_id":"x","mode":"cluster"}'); + self::assertTrue($config->isCluster()); + self::assertFalse($config->isStandalone()); + } + + public function testSecondaryDriverId(): void + { + $json = '{"driver_id":"spring","specific_driver_config":{"secondary_driver_id":"valkey-glide"}}'; + $config = Loader::parseDriverConfigString($json); + self::assertSame('valkey-glide', $config->secondaryDriverId()); + } + + public function testParsesWorkloadConfig(): void + { + $json = <<name()); + self::assertCount(1, $config->phases); + + $phase = $config->phases[0]; + self::assertSame('STEADY', $phase->id); + self::assertSame(16, $phase->connections); + self::assertTrue($phase->hasRpsLimit()); + self::assertSame(5000, $phase->rpsLimit); + self::assertTrue($phase->completion->isRequestBased()); + self::assertSame(100000, $phase->completion->totalRequests()); + + self::assertSame('uniform_rand', $phase->keyspace->generationAlg); + self::assertSame(7, $phase->keyspace->seed); + self::assertSame('k:', $phase->keyspace->keyPrefix); + + self::assertCount(2, $phase->commands); + // Command names are lowercased at config level. + self::assertSame('get', $phase->commands[0]->command); + self::assertEqualsWithDelta(0.7, $phase->commands[0]->weight, 1e-9); + self::assertSame(128, $phase->commands[1]->dataSizeBytes); + } + + public function testWorkloadDefaults(): void + { + $json = '{"phases":[{"id":"P","connections":1,"completion":{"type":"requests","requests":1},"keyspace":{"keys_count":1},"commands":[{"command":"ping","weight":1.0}]}]}'; + $config = Loader::parseWorkloadConfigString($json); + $phase = $config->phases[0]; + + self::assertSame(-1, $phase->cpsLimit); + self::assertSame(-1, $phase->rpsLimit); + self::assertFalse($phase->hasRpsLimit()); + self::assertSame(1, $phase->effectivePipelineDepth()); + // keyspace defaults + self::assertSame('bench:', $phase->keyspace->keyPrefix); + self::assertSame(16, $phase->keyspace->keySizeBytes); + self::assertTrue($phase->keyspace->isSequentialInt()); + } +} diff --git a/php/tests/Unit/FactoryTest.php b/php/tests/Unit/FactoryTest.php new file mode 100644 index 0000000..a1fcfba --- /dev/null +++ b/php/tests/Unit/FactoryTest.php @@ -0,0 +1,45 @@ +expectException(InvalidArgumentException::class); + Factory::create('nonexistent-driver'); + } +} diff --git a/php/tests/Unit/HdrEncoderTest.php b/php/tests/Unit/HdrEncoderTest.php new file mode 100644 index 0000000..f2f7af0 --- /dev/null +++ b/php/tests/Unit/HdrEncoderTest.php @@ -0,0 +1,84 @@ +totalCount()); + self::assertSame(0, $h->min()); + self::assertSame(0, $h->max()); + self::assertSame(0, $h->valueAtPercentile(50)); + } + + public function testRecordsAndComputesPercentiles(): void + { + $h = new HdrHistogram(1, 600_000_000, 3); + for ($i = 1; $i <= 1000; $i++) { + $h->record($i); + } + + self::assertSame(1000, $h->totalCount()); + self::assertSame(1, $h->min()); + // Values are quantized; assert within HdrHistogram equivalence tolerance. + self::assertEqualsWithDelta(500, $h->valueAtPercentile(50), 5); + self::assertEqualsWithDelta(990, $h->valueAtPercentile(99), 10); + self::assertGreaterThanOrEqual(1000, $h->max()); + } + + public function testCompressedEncodingStructure(): void + { + $h = new HdrHistogram(1, 600_000_000, 3); + foreach ([100, 150, 150, 200, 5000, 12345, 250, 250, 250] as $v) { + $h->record($v); + } + + $raw = HdrEncoder::encodeCompressed($h); + + /** @var array{cookie:int,len:int} $wrapper */ + $wrapper = unpack('Ncookie/Nlen', substr($raw, 0, 8)); + self::assertSame(self::COMPRESSED_COOKIE, $wrapper['cookie']); + self::assertSame(strlen($raw) - 8, $wrapper['len']); + + $payload = gzuncompress(substr($raw, 8)); + self::assertIsString($payload); + + /** @var array{cookie:int,plen:int,norm:int,sig:int} $v2 */ + $v2 = unpack('Ncookie/Nplen/Nnorm/Nsig', substr($payload, 0, 16)); + self::assertSame(self::V2_COOKIE, $v2['cookie']); + self::assertSame(0, $v2['norm']); + self::assertSame(3, $v2['sig']); + + /** @var array{lowest:int,highest:int,ratio:int} $fields */ + $fields = unpack('Jlowest/Jhighest/Jratio', substr($payload, 16, 24)); + self::assertSame(1, $fields['lowest']); + self::assertSame(600_000_000, $fields['highest']); + // IEEE754 bits for 1.0 + self::assertSame(4607182418800017408, $fields['ratio']); + } + + public function testBase64Encoding(): void + { + $h = new HdrHistogram(1, 600_000_000, 3); + $h->record(42); + + $b64 = HdrEncoder::encodeCompressedBase64($h); + $decoded = base64_decode($b64, true); + self::assertIsString($decoded); + + /** @var array{cookie:int} $wrapper */ + $wrapper = unpack('Ncookie', substr($decoded, 0, 4)); + self::assertSame(self::COMPRESSED_COOKIE, $wrapper['cookie']); + } +} diff --git a/php/tests/Unit/JavaRandomTest.php b/php/tests/Unit/JavaRandomTest.php new file mode 100644 index 0000000..a5006a3 --- /dev/null +++ b/php/tests/Unit/JavaRandomTest.php @@ -0,0 +1,112 @@ +nextInt(1000); + $values2[] = $rng2->nextInt(1000); + } + + self::assertSame($values1, $values2); + } + + public function testDifferentSeedsProduceDifferentSequences(): void + { + $rng1 = new JavaRandom(12345); + $rng2 = new JavaRandom(54321); + + $values1 = []; + $values2 = []; + for ($i = 0; $i < 10; $i++) { + $values1[] = $rng1->nextInt(1000); + $values2[] = $rng2->nextInt(1000); + } + + self::assertNotSame($values1, $values2); + } + + public function testSetSeedResetsSequence(): void + { + $rng = new JavaRandom(12345); + + $first = []; + for ($i = 0; $i < 5; $i++) { + $first[] = $rng->nextInt(1000); + } + + $rng->setSeed(12345); + + $second = []; + for ($i = 0; $i < 5; $i++) { + $second[] = $rng->nextInt(1000); + } + + self::assertSame($first, $second); + } + + public function testBoundMustBePositive(): void + { + $rng = new JavaRandom(12345); + $this->expectException(InvalidArgumentException::class); + $rng->nextInt(0); + } + + public function testValuesAreWithinBound(): void + { + $rng = new JavaRandom(12345); + $bound = 100; + for ($i = 0; $i < 100; $i++) { + $value = $rng->nextInt($bound); + self::assertGreaterThanOrEqual(0, $value); + self::assertLessThan($bound, $value); + } + } + + public function testPowerOfTwoBounds(): void + { + $rng = new JavaRandom(12345); + foreach ([2, 4, 8, 16, 32, 64, 128, 256, 512, 1024] as $bound) { + for ($i = 0; $i < 100; $i++) { + $value = $rng->nextInt($bound); + self::assertGreaterThanOrEqual(0, $value); + self::assertLessThan($bound, $value); + } + } + } + + /** + * Cross-language anchor: the first 10 outputs of Java's + * `new Random(0).nextInt(1000)`. These are engine-independent facts about + * java.util.Random and are shared by the Ruby/Python/Node ports. + * + * Verified: the underlying LCG reproduces Java's canonical + * `new Random(0).nextInt()` signed-int sequence + * [-1155484576, -723955400, 1033096058, -1690734402, -1557280266, ...]. + */ + public function testMatchesJavaSeedZeroAnchor(): void + { + $rng = new JavaRandom(0); + $actual = []; + for ($i = 0; $i < 10; $i++) { + $actual[] = $rng->nextInt(1000); + } + + $expected = [360, 948, 29, 447, 515, 53, 491, 761, 719, 854]; + self::assertSame($expected, $actual); + } +} diff --git a/php/tests/Unit/KeyGeneratorTest.php b/php/tests/Unit/KeyGeneratorTest.php new file mode 100644 index 0000000..c11d24c --- /dev/null +++ b/php/tests/Unit/KeyGeneratorTest.php @@ -0,0 +1,129 @@ +nextKey(); + $key2 = $gen->nextKey(); + + self::assertStringStartsWith('test:', $key1); + self::assertStringStartsWith('test:', $key2); + self::assertNotSame($key1, $key2); + } + + public function testSequentialKeysWrapAround(): void + { + $config = new KeyspaceConfig(keysCount: 3, keyPrefix: 'test:'); + $gen = new KeyGenerator($config); + + $keys = []; + for ($i = 0; $i < 6; $i++) { + $keys[] = $gen->nextKey(); + } + + // Keys should wrap: 0, 1, 2, 0, 1, 2 + self::assertSame($keys[0], $keys[3]); + self::assertSame($keys[1], $keys[4]); + self::assertSame($keys[2], $keys[5]); + } + + public function testGeneratesUniformRandomKeys(): void + { + $config = new KeyspaceConfig( + keysCount: 1000, + keyPrefix: 'rand:', + generationAlg: 'uniform_rand', + seed: 12345, + ); + $gen = new KeyGenerator($config); + + $keys = []; + for ($i = 0; $i < 100; $i++) { + $keys[] = $gen->nextKey(); + } + + $unique = array_unique($keys); + self::assertGreaterThan(50, count($unique)); + } + + public function testResetRestartsSequentialCounter(): void + { + $config = new KeyspaceConfig(keysCount: 100, keyPrefix: 'test:'); + $gen = new KeyGenerator($config); + + $first1 = $gen->nextKey(); + $gen->nextKey(); + $gen->reset(); + $first2 = $gen->nextKey(); + + self::assertSame($first1, $first2); + } + + public function testResetRestartsRandomSequence(): void + { + $config = new KeyspaceConfig( + keysCount: 1000, + keyPrefix: 'rand:', + generationAlg: 'uniform_rand', + seed: 12345, + ); + $gen = new KeyGenerator($config); + + $first = []; + for ($i = 0; $i < 10; $i++) { + $first[] = $gen->nextKey(); + } + $gen->reset(); + $second = []; + for ($i = 0; $i < 10; $i++) { + $second[] = $gen->nextKey(); + } + + self::assertSame($first, $second); + } + + public function testKeyFormatZeroPadsToKeySize(): void + { + $config = new KeyspaceConfig( + keysCount: 100, + keySizeBytes: 16, + keyPrefix: 'bench:', + ); + $gen = new KeyGenerator($config); + + $key = $gen->nextKey(); + + // prefix "bench:" (6) + padding width max(16-6,1)=10 -> "bench:0000000000" + self::assertSame('bench:0000000000', $key); + self::assertSame(16, strlen($key)); + } + + public function testUniformRandMatchesJavaRandomSequence(): void + { + // With uniform_rand + seed, keys are prefix + JavaRandom.nextInt(keysCount). + $config = new KeyspaceConfig( + keysCount: 1000, + keySizeBytes: 16, + keyPrefix: 'bench:', + generationAlg: 'uniform_rand', + seed: 0, + ); + $gen = new KeyGenerator($config); + + $first = $gen->nextKey(); + // JavaRandom(0).nextInt(1000) == 360 (verified against Java canonical LCG). + self::assertSame('bench:0000000360', $first); + } +} diff --git a/php/tests/Unit/RateLimiterTest.php b/php/tests/Unit/RateLimiterTest.php new file mode 100644 index 0000000..4668608 --- /dev/null +++ b/php/tests/Unit/RateLimiterTest.php @@ -0,0 +1,52 @@ +ratePerSecond); + } + + public function testEnforcesApproximateRate(): void + { + // 200 ops/sec -> 30 ops should take ~145ms (29 intervals of 5ms). + $limiter = RateLimiter::create(200); + self::assertNotNull($limiter); + + $start = hrtime(true); + for ($i = 0; $i < 30; $i++) { + $limiter->acquire(); + } + $elapsedMs = (hrtime(true) - $start) / 1_000_000; + + // Expected ~145ms; allow generous tolerance for CI timing jitter. + self::assertGreaterThan(100, $elapsedMs, "Rate limiter too fast: {$elapsedMs}ms"); + self::assertLessThan(400, $elapsedMs, "Rate limiter too slow: {$elapsedMs}ms"); + } + + public function testTryAcquireRespectsInterval(): void + { + $limiter = RateLimiter::create(1); // 1 op/sec + self::assertNotNull($limiter); + + // First is immediately allowed. + self::assertTrue($limiter->tryAcquire()); + // Second should be denied (1s not elapsed). + self::assertFalse($limiter->tryAcquire()); + } +} diff --git a/php/tests/fixtures/recording-driver.json b/php/tests/fixtures/recording-driver.json new file mode 100644 index 0000000..95daa3a --- /dev/null +++ b/php/tests/fixtures/recording-driver.json @@ -0,0 +1,7 @@ +{ + "schema_version": "1.0", + "description": "Recording client - server-free (PHP engine tests)", + "driver_id": "recording", + "mode": "standalone", + "specific_driver_config": {} +} diff --git a/php/tests/fixtures/smoke-workload.json b/php/tests/fixtures/smoke-workload.json new file mode 100644 index 0000000..21e85c6 --- /dev/null +++ b/php/tests/fixtures/smoke-workload.json @@ -0,0 +1,45 @@ +{ + "schema_version": "1.0", + "benchmark_profile": { + "name": "PHP Smoke Test", + "description": "Short two-phase workload for server-free e2e testing" + }, + "phases": [ + { + "id": "WARMUP", + "connections": 4, + "completion": { + "type": "requests", + "requests": 400 + }, + "keyspace": { + "keys_count": 1000, + "key_size_bytes": 16, + "key_prefix": "bench:", + "generation_alg": "sequential_int" + }, + "commands": [ + {"command": "set", "weight": 1.0, "data_size_bytes": 64} + ] + }, + { + "id": "STEADY", + "connections": 4, + "completion": { + "type": "requests", + "requests": 1000 + }, + "keyspace": { + "keys_count": 1000, + "key_size_bytes": 16, + "key_prefix": "bench:", + "generation_alg": "uniform_rand", + "seed": 12345 + }, + "commands": [ + {"command": "get", "weight": 0.8}, + {"command": "set", "weight": 0.2, "data_size_bytes": 64} + ] + } + ] +} diff --git a/php/tools/hdr-crosscheck/HdrCrossCheck.java b/php/tools/hdr-crosscheck/HdrCrossCheck.java new file mode 100644 index 0000000..f407a8e --- /dev/null +++ b/php/tools/hdr-crosscheck/HdrCrossCheck.java @@ -0,0 +1,56 @@ +// HDR cross-check decoder (Java side). +// +// Reads the two-line output of emit.php (JSON summary on line 1, base64 V2 +// compressed payload on line 2), decodes the payload with the canonical Java +// HdrHistogram library, and prints Java-computed percentiles so they can be +// compared against the PHP summary. Matching values prove the PHP encoder is +// byte-compatible with Java's Histogram.encodeIntoCompressedByteBuffer(). +// +// Requires org.hdrhistogram:HdrHistogram on the classpath. Example: +// PHP: php php/tools/hdr-crosscheck/emit.php > /tmp/hdr.txt +// javac -cp HdrHistogram.jar HdrCrossCheck.java +// java -cp .:HdrHistogram.jar HdrCrossCheck /tmp/hdr.txt +// +// Compare the "java:" percentile line against the JSON on line 1 of /tmp/hdr.txt. + +import java.nio.ByteBuffer; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.Base64; +import java.util.List; +import org.HdrHistogram.Histogram; + +public class HdrCrossCheck { + public static void main(String[] args) throws Exception { + if (args.length < 1) { + System.err.println("Usage: java HdrCrossCheck "); + System.exit(2); + } + + List lines = Files.readAllLines(Path.of(args[0])); + if (lines.size() < 2) { + System.err.println("Expected two lines (summary JSON, base64 payload)"); + System.exit(2); + } + + String phpSummary = lines.get(0); + String base64Payload = lines.get(1).trim(); + + byte[] compressed = Base64.getDecoder().decode(base64Payload); + Histogram h = Histogram.decodeFromCompressedByteBuffer(ByteBuffer.wrap(compressed), 0); + + System.out.println("php: " + phpSummary); + System.out.printf( + "java: {\"count\":%d,\"min\":%d,\"max\":%d,\"p50\":%d,\"p90\":%d,\"p95\":%d,\"p99\":%d,\"p999\":%d}%n", + h.getTotalCount(), + h.getMinValue(), + h.getMaxValue(), + h.getValueAtPercentile(50.0), + h.getValueAtPercentile(90.0), + h.getValueAtPercentile(95.0), + h.getValueAtPercentile(99.0), + h.getValueAtPercentile(99.9) + ); + System.out.println("If php/java percentiles match (within HdrHistogram equivalence), the encoding is compatible."); + } +} diff --git a/php/tools/hdr-crosscheck/README.md b/php/tools/hdr-crosscheck/README.md new file mode 100644 index 0000000..1c1d802 --- /dev/null +++ b/php/tools/hdr-crosscheck/README.md @@ -0,0 +1,41 @@ +# HDR Cross-Check + +Verifies that the PHP engine's HdrHistogram V2 compressed payload is byte-compatible +with the canonical Java HdrHistogram library โ€” the definitive cross-language latency +parity gate. + +## Why + +The PHP `HdrEncoder` produces the same binary V2 compressed format as Java's +`Histogram.encodeIntoCompressedByteBuffer()`. The unit tests assert the payload's +*structure* (cookies, header fields, IEEE754 conversion ratio), but the strongest +check is decoding a real PHP payload with Java and confirming the percentiles match. + +## Run + +```bash +# 1) Emit a payload + PHP-computed percentiles from a fixed sample set. +php php/tools/hdr-crosscheck/emit.php > /tmp/hdr.txt + +# 2) Decode with Java's HdrHistogram and print its percentiles. +# Get the jar, e.g. from Maven Central: org.hdrhistogram:HdrHistogram +javac -cp HdrHistogram.jar php/tools/hdr-crosscheck/HdrCrossCheck.java -d /tmp +java -cp /tmp:HdrHistogram.jar HdrCrossCheck /tmp/hdr.txt +``` + +## Interpret + +The tool prints two lines: + +``` +php: {"count":1005,"min":1,"max":123519,"p50":...,"p90":...,"p95":...,"p99":...,"p999":...} +java: {"count":1005,"min":1,"max":123519,"p50":...,...} +``` + +- `count`, `min`, `max` must match exactly. +- Percentiles must match within HdrHistogram's value-equivalence range (values in the + same bucket are considered equal). Small differences at the bucket boundary are + expected and acceptable; large differences indicate an encoding mismatch. + +A clean match confirms PHP latency data is directly comparable to the Java, Ruby, +and C# engines. diff --git a/php/tools/hdr-crosscheck/emit.php b/php/tools/hdr-crosscheck/emit.php new file mode 100644 index 0000000..dcacb7a --- /dev/null +++ b/php/tools/hdr-crosscheck/emit.php @@ -0,0 +1,55 @@ + /tmp/hdr_payload.txt + */ + +$autoload = __DIR__ . '/../../vendor/autoload.php'; +if (!is_file($autoload)) { + fwrite(STDERR, "Autoloader not found. Run composer install in php/.\n"); + exit(2); +} +require $autoload; + +use RespBench\Metrics\HdrEncoder; +use RespBench\Metrics\HdrHistogram; + +// Fixed, reproducible sample set (microseconds). +$values = []; +for ($i = 1; $i <= 1000; $i++) { + $values[] = $i; // 1..1000 linear +} +foreach ([2500, 5000, 10000, 50000, 123456] as $tail) { + $values[] = $tail; // long tail +} + +$h = new HdrHistogram(1, 600_000_000, 3); +foreach ($values as $v) { + $h->record($v); +} + +$percentiles = [50.0, 90.0, 95.0, 99.0, 99.9]; +$phpSummary = [ + 'count' => $h->totalCount(), + 'min' => $h->min(), + 'max' => $h->max(), +]; +foreach ($percentiles as $p) { + $phpSummary['p' . str_replace('.', '', (string) $p)] = $h->valueAtPercentile($p); +} + +$payload = HdrEncoder::encodeCompressedBase64($h); + +// Machine-readable output: first line = JSON summary, second line = base64 payload. +echo json_encode($phpSummary, JSON_THROW_ON_ERROR) . "\n"; +echo $payload . "\n"; diff --git a/ruby/Gemfile b/ruby/Gemfile index 1bc8b8d..61151c6 100644 --- a/ruby/Gemfile +++ b/ruby/Gemfile @@ -10,7 +10,9 @@ gem "HDRHistogram", "~> 0.1" # HdrHistogram for latency metrics gem "oj", "~> 3.16" # Fast JSON serialization gem "concurrent-ruby", "~> 1.2" # Thread-safe data structures -gem "valkey", github: "valkey-io/valkey-glide-ruby", branch: "main" # valkey-glide-ruby client (drop-in replacement for redis-rb) +# valkey-glide-ruby client; supplies the `Valkey` class via `require "valkey"`. +# Pinned to an exact released version for reproducible benchmark runs. +gem "valkey-glide-rb", "1.0.0" group :development, :test do gem "minitest", "~> 5.0" diff --git a/ruby/README.md b/ruby/README.md index 58815d0..0e34a5f 100644 --- a/ruby/README.md +++ b/ruby/README.md @@ -7,7 +7,7 @@ Ruby implementation of the resp-bench benchmark suite for Redis/Valkey compatibl | Driver ID | Gem | Description | |-----------|-----|-------------| | `redis-rb` | `redis` | Standard Redis client for Ruby | -| `valkey-glide-ruby` | `valkey` | Valkey GLIDE client for Ruby ([GitHub](https://github.com/valkey-io/valkey-glide-ruby)) | +| `valkey-glide-ruby` | `valkey-glide-rb` | Valkey GLIDE client for Ruby, loaded via `require "valkey"` ([GitHub](https://github.com/valkey-io/valkey-glide-ruby)) | ## Installation @@ -223,7 +223,7 @@ The Ruby engine produces NDJSON output compatible with all other language engine ## Dependencies - `redis` (~> 5.0) - redis-rb client -- `valkey` (~> 0.1) - valkey-glide-ruby client +- `valkey-glide-rb` (1.0.0) - valkey-glide-ruby client - `concurrent-ruby` - Thread-safe data structures - `hdrhistogram` - Latency histograms - `oj` - Fast JSON serialization diff --git a/ruby/lib/resp_bench/client/impl/valkey_glide_client.rb b/ruby/lib/resp_bench/client/impl/valkey_glide_client.rb index 52c7ceb..19d7e7e 100644 --- a/ruby/lib/resp_bench/client/impl/valkey_glide_client.rb +++ b/ruby/lib/resp_bench/client/impl/valkey_glide_client.rb @@ -76,15 +76,7 @@ def close end def driver_version - # For GitHub-sourced gems, report the git commit SHA instead of - # the hardcoded VERSION constant (which may lag behind main). - spec = Bundler.load.specs.find { |s| s.name == "valkey" } - if spec&.source.is_a?(Bundler::Source::Git) - spec.source.revision - else - Valkey::VERSION - end - rescue StandardError + # The Gemfile pins an exact released version of valkey-glide-rb. Valkey::VERSION end diff --git a/ruby/resp_bench.gemspec b/ruby/resp_bench.gemspec index 16ec38d..3af640d 100644 --- a/ruby/resp_bench.gemspec +++ b/ruby/resp_bench.gemspec @@ -25,7 +25,7 @@ Gem::Specification.new do |spec| spec.require_paths = ["lib"] spec.add_dependency "redis", "~> 5.0" - spec.add_dependency "valkey", "~> 0.1" + spec.add_dependency "valkey-glide-rb", "~> 1.0" spec.add_dependency "async", "~> 2.6" spec.add_dependency "async-redis", "~> 0.8" spec.add_dependency "HDRHistogram", "~> 0.1" diff --git a/scripts/generate_graphs.py b/scripts/generate_graphs.py index 177a7e3..36f58d7 100644 --- a/scripts/generate_graphs.py +++ b/scripts/generate_graphs.py @@ -71,6 +71,13 @@ # C# drivers "stackexchange-redis": "csharp", "valkey-glide-csharp": "csharp", + # PHP drivers + "valkey-glide-php": "php", + "phpredis": "php", + # Node.js drivers + "valkey-glide-node": "node", + "ioredis": "node", + "iovalkey": "node", # Python drivers (future) "redis-py": "python", "aioredis": "python", @@ -118,7 +125,10 @@ def parse_args(): ) parser.add_argument( "--language", - choices=["java", "ruby", "csharp", "python"], + # Keep in sync with the values in DRIVER_LANGUAGE_MAP โ€” argparse rejects + # anything not listed here, so a language added to the map alone fails at + # the CLI rather than silently producing empty graphs. + choices=["java", "ruby", "csharp", "node", "python"], help="Filter results by language (only include drivers for this language)", ) parser.add_argument( diff --git a/scripts/generate_interactive_graphs.py b/scripts/generate_interactive_graphs.py index 3523185..34af16b 100644 --- a/scripts/generate_interactive_graphs.py +++ b/scripts/generate_interactive_graphs.py @@ -680,6 +680,11 @@ def load_cpu_data(results_dir, rps_outlier_map=None): # C# (.NET) drivers โ€” cyan/pink "stackexchange-redis": "#00ACC1", # cyan 600 "valkey-glide-csharp": "#D81B60", # pink 600 + + # Node.js drivers โ€” yellows/browns + "valkey-glide-node": "#F9A825", # yellow 800 + "ioredis": "#6D4C41", # brown 600 + "iovalkey": "#546E7A", # blue grey 600 } DRIVER_FAMILIES = { @@ -694,15 +699,19 @@ def load_cpu_data(results_dir, rps_outlier_map=None): "redisson": "low-level", "stackexchange-redis": "csharp", "valkey-glide-csharp": "csharp", + "valkey-glide-node": "node", + "ioredis": "node", + "iovalkey": "node", } -FAMILY_ORDER = ["spring-data-valkey", "spring-data-redis", "low-level", "csharp"] +FAMILY_ORDER = ["spring-data-valkey", "spring-data-redis", "low-level", "csharp", "node"] FAMILY_LABELS = { "spring-data-valkey": "Spring Data Valkey", "spring-data-redis": "Spring Data Redis", "low-level": "Low-Level Java Drivers", "csharp": "C# (.NET) Drivers", + "node": "Node.js Drivers", } # Fallback color for unknown drivers diff --git a/scripts/requirements.txt b/scripts/requirements.txt new file mode 100644 index 0000000..1b94803 --- /dev/null +++ b/scripts/requirements.txt @@ -0,0 +1,11 @@ +# Python dependencies for scripts. +# +# python -m pip install -r scripts/requirements.txt +# +# Direct dependencies are pinned exactly so each run resolves the same versions; +# transitive dependencies still float (fully locking them needs a compiled lockfile). +# +# numpy is held below 2.5 because 2.5.x requires Python >= 3.12 and +# .github/workflows/benchmark.yml runs Python 3.11. +matplotlib==3.11.1 +numpy==2.4.6 diff --git a/scripts/run_benchmark_matrix.py b/scripts/run_benchmark_matrix.py index 62c8a2c..7af1150 100644 --- a/scripts/run_benchmark_matrix.py +++ b/scripts/run_benchmark_matrix.py @@ -21,13 +21,21 @@ One dimension is designated as the X axis (typically "connections"). All other free dimensions form the series (one line per unique combo). -Output is a flat directory with one NDJSON file per series label: - /