From 8f2ec957b31f05c73b646fcfd0ad03403de13244 Mon Sep 17 00:00:00 2001 From: James Xin Date: Fri, 28 Aug 2026 15:05:03 -0700 Subject: [PATCH 1/8] Fix broken Ruby gem dependency and stale configs, pin build inputs (#5) * Fix broken Ruby gem dependency and stale configs, pin build inputs Signed-off-by: James Xin * Resolve config paths without rewriting them; finish the gem rename Addresses review findings on the initial commit. Rewriting driver_config/workload_template to absolute paths made matrix expansion depend on where the repo sits on disk: those strings double as identity, since DimensionSpec.matches_driver() fnmatches applies_to globs against the whole string and write_manifest() records it. A checkout under any directory whose name contains "glide" made applies_to ["*glide*"] match every driver, applying GLIDE-only thread env vars to Jedis, Lettuce and Redisson. Validation and the runtime opens now share one resolve_config_path() helper, so they agree by construction while the config strings survive byte-for-byte. The new guard tests chdir off the repo root, without which they passed against the buggy source. Also: reject $binding values for driver_config at parse time, fold a non-string path into the aggregated ValueError, drop a false comment about detect_engine_for_driver() swallowing errors (generate_driver_config() raises outside the guarded region, so a bad path crashes loudly), caveat the requirements.txt pinning claim since transitives float, and finish the gem rename in ruby/README.md and ValkeyGlideClient#driver_version, which still looked up a Bundler spec name that repo never published. Signed-off-by: James Xin * address comment: trim over-verbose dependency comments Reviewer noted the AI-authored dep comments were too verbose/narrowly scoped. Trimmed them across the Ruby Gemfile, gemspec, README, the driver_version comment, and scripts/requirements.txt, keeping only the non-obvious bits (the require "valkey" name mismatch and the numpy<2.5 CI constraint). Lockfile follow-up tracked in #20. Comment/doc-only; no logic changes. Signed-off-by: James Xin --------- Signed-off-by: James Xin --- .github/benchmark-config/drivers.json | 21 ---- .github/benchmark-config/workloads.json | 3 - .github/workflows/benchmark.yml | 2 +- .../valkey-glide-basic-multiclients.json | 2 +- ruby/Gemfile | 4 +- ruby/README.md | 4 +- .../client/impl/valkey_glide_client.rb | 10 +- ruby/resp_bench.gemspec | 2 +- scripts/requirements.txt | 11 ++ scripts/run_benchmark_matrix.py | 67 +++++++++++- scripts/tests/test_matrix_config.py | 100 ++++++++++++++++++ 11 files changed, 185 insertions(+), 41 deletions(-) delete mode 100644 .github/benchmark-config/drivers.json delete mode 100644 .github/benchmark-config/workloads.json create mode 100644 scripts/requirements.txt diff --git a/.github/benchmark-config/drivers.json b/.github/benchmark-config/drivers.json deleted file mode 100644 index 74c36f9..0000000 --- a/.github/benchmark-config/drivers.json +++ /dev/null @@ -1,21 +0,0 @@ -{ - "java": [ - "configs/drivers/default/jedis.json", - "configs/drivers/default/lettuce.json", - "configs/drivers/default/redisson.json", - "configs/drivers/default/spring-data-redis-jedis.json", - "configs/drivers/default/spring-data-redis-lettuce.json", - "configs/drivers/default/spring-data-valkey-glide-standalone.json", - "configs/drivers/default/spring-data-valkey-jedis-standalone.json", - "configs/drivers/default/spring-data-valkey-lettuce-standalone.json", - "configs/drivers/default/valkey-glide-standalone.json" - ], - "ruby": [ - "configs/drivers/default/redis-rb.json", - "configs/drivers/default/valkey-glide-ruby.json" - ], - "csharp": [ - "configs/drivers/default/stackexchange-redis.json", - "configs/drivers/default/valkey-glide-csharp.json" - ] -} diff --git a/.github/benchmark-config/workloads.json b/.github/benchmark-config/workloads.json deleted file mode 100644 index 3be6e36..0000000 --- a/.github/benchmark-config/workloads.json +++ /dev/null @@ -1,3 +0,0 @@ -[ - "configs/workloads/reference/basic-standalone-single-client-1M-reqs.json" -] diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml index 66147cd..8aebd34 100644 --- a/.github/workflows/benchmark.yml +++ b/.github/workflows/benchmark.yml @@ -169,7 +169,7 @@ jobs: - name: Install dependencies run: | - pip install matplotlib numpy + pip install -r scripts/requirements.txt - name: Download all artifacts uses: actions/download-artifact@v4 diff --git a/configs/matrices/valkey-glide-basic-multiclients.json b/configs/matrices/valkey-glide-basic-multiclients.json index 4993982..22ff78a 100644 --- a/configs/matrices/valkey-glide-basic-multiclients.json +++ b/configs/matrices/valkey-glide-basic-multiclients.json @@ -1,7 +1,7 @@ { "description": "Valkey-Glide basic example for small connection scaling", "x_axis": "connections", - "workload_template": "configs/workloads/reference/basic-standalone-single-client-10M-secs.json", + "workload_template": "configs/workloads/reference/basic-standalone-single-client-10-secs.json", "iterations": 5, "dimensions": { "connections": [ diff --git a/ruby/Gemfile b/ruby/Gemfile index 1bc8b8d..61151c6 100644 --- a/ruby/Gemfile +++ b/ruby/Gemfile @@ -10,7 +10,9 @@ gem "HDRHistogram", "~> 0.1" # HdrHistogram for latency metrics gem "oj", "~> 3.16" # Fast JSON serialization gem "concurrent-ruby", "~> 1.2" # Thread-safe data structures -gem "valkey", github: "valkey-io/valkey-glide-ruby", branch: "main" # valkey-glide-ruby client (drop-in replacement for redis-rb) +# valkey-glide-ruby client; supplies the `Valkey` class via `require "valkey"`. +# Pinned to an exact released version for reproducible benchmark runs. +gem "valkey-glide-rb", "1.0.0" group :development, :test do gem "minitest", "~> 5.0" diff --git a/ruby/README.md b/ruby/README.md index 58815d0..0e34a5f 100644 --- a/ruby/README.md +++ b/ruby/README.md @@ -7,7 +7,7 @@ Ruby implementation of the resp-bench benchmark suite for Redis/Valkey compatibl | Driver ID | Gem | Description | |-----------|-----|-------------| | `redis-rb` | `redis` | Standard Redis client for Ruby | -| `valkey-glide-ruby` | `valkey` | Valkey GLIDE client for Ruby ([GitHub](https://github.com/valkey-io/valkey-glide-ruby)) | +| `valkey-glide-ruby` | `valkey-glide-rb` | Valkey GLIDE client for Ruby, loaded via `require "valkey"` ([GitHub](https://github.com/valkey-io/valkey-glide-ruby)) | ## Installation @@ -223,7 +223,7 @@ The Ruby engine produces NDJSON output compatible with all other language engine ## Dependencies - `redis` (~> 5.0) - redis-rb client -- `valkey` (~> 0.1) - valkey-glide-ruby client +- `valkey-glide-rb` (1.0.0) - valkey-glide-ruby client - `concurrent-ruby` - Thread-safe data structures - `hdrhistogram` - Latency histograms - `oj` - Fast JSON serialization diff --git a/ruby/lib/resp_bench/client/impl/valkey_glide_client.rb b/ruby/lib/resp_bench/client/impl/valkey_glide_client.rb index 52c7ceb..19d7e7e 100644 --- a/ruby/lib/resp_bench/client/impl/valkey_glide_client.rb +++ b/ruby/lib/resp_bench/client/impl/valkey_glide_client.rb @@ -76,15 +76,7 @@ def close end def driver_version - # For GitHub-sourced gems, report the git commit SHA instead of - # the hardcoded VERSION constant (which may lag behind main). - spec = Bundler.load.specs.find { |s| s.name == "valkey" } - if spec&.source.is_a?(Bundler::Source::Git) - spec.source.revision - else - Valkey::VERSION - end - rescue StandardError + # The Gemfile pins an exact released version of valkey-glide-rb. Valkey::VERSION end diff --git a/ruby/resp_bench.gemspec b/ruby/resp_bench.gemspec index 16ec38d..3af640d 100644 --- a/ruby/resp_bench.gemspec +++ b/ruby/resp_bench.gemspec @@ -25,7 +25,7 @@ Gem::Specification.new do |spec| spec.require_paths = ["lib"] spec.add_dependency "redis", "~> 5.0" - spec.add_dependency "valkey", "~> 0.1" + spec.add_dependency "valkey-glide-rb", "~> 1.0" spec.add_dependency "async", "~> 2.6" spec.add_dependency "async-redis", "~> 0.8" spec.add_dependency "HDRHistogram", "~> 0.1" diff --git a/scripts/requirements.txt b/scripts/requirements.txt new file mode 100644 index 0000000..1b94803 --- /dev/null +++ b/scripts/requirements.txt @@ -0,0 +1,11 @@ +# Python dependencies for scripts. +# +# python -m pip install -r scripts/requirements.txt +# +# Direct dependencies are pinned exactly so each run resolves the same versions; +# transitive dependencies still float (fully locking them needs a compiled lockfile). +# +# numpy is held below 2.5 because 2.5.x requires Python >= 3.12 and +# .github/workflows/benchmark.yml runs Python 3.11. +matplotlib==3.11.1 +numpy==2.4.6 diff --git a/scripts/run_benchmark_matrix.py b/scripts/run_benchmark_matrix.py index 62c8a2c..00da267 100644 --- a/scripts/run_benchmark_matrix.py +++ b/scripts/run_benchmark_matrix.py @@ -63,6 +63,8 @@ SYSTEM_MONITOR_INTERVAL = 0.5 +REPO_ROOT = Path(__file__).resolve().parent.parent + # Well-known dimension names with special handling DIM_CONNECTIONS = "connections" DIM_DRIVER_CONFIG = "driver_config" @@ -92,6 +94,41 @@ } +def resolve_config_path(path_str): + """Resolve a path referenced by a matrix config, or None if it does not exist. + + Matrix configs spell `workload_template` and `driver_config` relative to the + repository root, but the runner may be invoked from anywhere, so try the path + as written (i.e. relative to the CWD) and then relative to the repo root. + + Both parse_matrix_config()'s validation and the runtime opens go through this + one function, so a path that validates is by construction a path the run can + open. Note it returns a resolved path *without* mutating the caller's string: + the config values double as identity — applies_to globs + (DimensionSpec.matches_driver) and _manifest.json both consume them — so + rewriting them would make matrix expansion and manifest contents depend on + where the repository happens to be checked out. + """ + as_written = Path(path_str) + if as_written.is_file(): + return as_written + if not as_written.is_absolute(): + from_repo_root = REPO_ROOT / as_written + if from_repo_root.is_file(): + return from_repo_root + return None + + +def open_config_path(path_str): + """Resolve a matrix-referenced config path for reading, or raise.""" + resolved = resolve_config_path(path_str) + if resolved is None: + raise FileNotFoundError( + f"config file referenced by the matrix does not exist: {path_str}" + ) + return resolved + + def detect_engine_for_driver(driver_config_path): """Detect the correct engine (make target) for a driver config file. @@ -205,6 +242,32 @@ def parse_matrix_config(matrix_path): if not config["workload_template"]: raise ValueError("'workload_template' is required in matrix config") + # Referenced config files must exist. Without this a typo'd path surfaces + # only mid-run — after the server is up and the engine has been built — + # because --dry-run never opens these files. Checking them through the same + # resolve_config_path() that generate_workload() and generate_driver_config() + # use means anything accepted here is openable by the run itself. + missing = [] + + workload_template = config["workload_template"] + if not isinstance(workload_template, str) or resolve_config_path(workload_template) is None: + missing.append(f"workload_template: {workload_template!r}") + + for value in dimensions[DIM_DRIVER_CONFIG].values: + # Every value is checked, including "$binding" strings. Unlike other + # dimensions, driver_config is never a binding target — + # generate_series_combos() excludes it from the series dimensions, so + # resolve_binding() never rewrites it and a "$foo" here would reach + # open() verbatim. + if not isinstance(value, str) or resolve_config_path(value) is None: + missing.append(f"{DIM_DRIVER_CONFIG}: {value!r}") + + if missing: + raise ValueError( + f"matrix config '{matrix_path}' references files that do not exist:\n " + + "\n ".join(missing) + ) + return config @@ -343,7 +406,7 @@ def generate_series_combos(config): def generate_workload(template_path, connections): """Generate a workload JSON with the given connection count.""" - with open(template_path) as f: + with open(open_config_path(template_path)) as f: workload = json.load(f) for phase in workload.get("phases", []): @@ -358,7 +421,7 @@ def generate_driver_config(base_config_path, overrides): overrides is a dict of specific_driver_config keys to set, e.g. {"pool_size": 32, "use_pooling": true}. """ - with open(base_config_path) as f: + with open(open_config_path(base_config_path)) as f: config = json.load(f) if overrides: diff --git a/scripts/tests/test_matrix_config.py b/scripts/tests/test_matrix_config.py index e3c25f3..f78c2f5 100644 --- a/scripts/tests/test_matrix_config.py +++ b/scripts/tests/test_matrix_config.py @@ -175,6 +175,106 @@ def test_missing_x_axis_raises(self, tmp_path): with pytest.raises(ValueError, match="nonexistent"): parse_matrix_config(p) + def test_missing_workload_template_file_raises(self, tmp_path): + config = { + "x_axis": "connections", + "workload_template": "configs/workloads/reference/no-such-workload.json", + "dimensions": { + "connections": [1], + "driver_config": ["configs/drivers/high-throughput/jedis.json"], + }, + } + p = tmp_path / "bad.json" + p.write_text(json.dumps(config)) + with pytest.raises(ValueError, match="no-such-workload.json"): + parse_matrix_config(p) + + def test_missing_driver_config_file_raises(self, tmp_path): + config = { + "x_axis": "connections", + "workload_template": "configs/workloads/reference/basic-standalone-single-client-1M-reqs.json", + "dimensions": { + "connections": [1], + "driver_config": ["configs/drivers/high-throughput/no-such-driver.json"], + }, + } + p = tmp_path / "bad.json" + p.write_text(json.dumps(config)) + with pytest.raises(ValueError, match="no-such-driver.json"): + parse_matrix_config(p) + + def test_referenced_paths_are_openable_after_parse(self, simple_matrix, monkeypatch): + # A path that survives parse_matrix_config() must be one the run can + # actually open — validating against a different base than the runtime + # uses would prove nothing. Assert through the real consumers rather + # than re-implementing resolution here. chdir away from the repo root so + # the repo-root fallback is what is under test; run from the root and + # the paths resolve as written and prove nothing. + monkeypatch.chdir(simple_matrix.parent) + config = parse_matrix_config(simple_matrix) + assert generate_workload(config["workload_template"], 4)["phases"] + for driver in config["dimensions"]["driver_config"].values: + assert generate_driver_config(driver, {}) + + def test_parse_leaves_referenced_paths_verbatim(self, simple_matrix, monkeypatch): + # Validation must not rewrite the paths. These strings double as + # identity: DimensionSpec.matches_driver() fnmatches applies_to globs + # against the whole driver_config string, and write_manifest() records + # it. Normalizing to an absolute path would fold the checkout location + # into both — e.g. a clone under a directory whose name contains + # "glide" would make an `applies_to: ["*glide*"]` dimension match every + # driver. + # + # chdir is essential: rewriting only ever happened on the repo-root + # fallback, so from the repo root this test passes even against the + # rewriting version and catches nothing. + monkeypatch.chdir(simple_matrix.parent) + config = parse_matrix_config(simple_matrix) + assert config["workload_template"] == ( + "configs/workloads/reference/basic-standalone-single-client-1M-reqs.json" + ) + assert config["dimensions"]["driver_config"].values == [ + "configs/drivers/high-throughput/jedis.json" + ] + + def test_driver_config_binding_is_rejected(self, tmp_path): + # driver_config is excluded from generate_series_combos()' series + # dimensions, so resolve_binding() never rewrites it and a "$binding" + # would reach open() verbatim mid-run. Reject it at parse time. + config = { + "x_axis": "connections", + "workload_template": "configs/workloads/reference/basic-standalone-single-client-1M-reqs.json", + "dimensions": { + "connections": [1], + "driver_config": ["$connections"], + }, + } + p = tmp_path / "bad.json" + p.write_text(json.dumps(config)) + with pytest.raises(ValueError, match=r"\$connections"): + parse_matrix_config(p) + + @pytest.mark.parametrize("key", ["workload_template", "driver_config"]) + def test_non_string_path_raises_value_error(self, tmp_path, key): + # A non-string here must join the aggregated ValueError like any other + # bad path, not escape as a bare TypeError out of pathlib. + config = { + "x_axis": "connections", + "workload_template": "configs/workloads/reference/basic-standalone-single-client-1M-reqs.json", + "dimensions": { + "connections": [1], + "driver_config": ["configs/drivers/high-throughput/jedis.json"], + }, + } + if key == "workload_template": + config["workload_template"] = 5 + else: + config["dimensions"]["driver_config"] = [5] + p = tmp_path / "bad.json" + p.write_text(json.dumps(config)) + with pytest.raises(ValueError, match=key): + parse_matrix_config(p) + class TestSeriesComboGeneration: def test_single_driver_no_series_dims(self, simple_matrix): From df6ebf043db96166b4a40836bfcf86ecbbf9b197 Mon Sep 17 00:00:00 2001 From: James Xin Date: Fri, 28 Aug 2026 16:09:19 -0700 Subject: [PATCH 2/8] Matrix runner: fail loudly, isolate runs, report status honestly (#6) Three failure-visibility problems made unattended sweeps unsafe: an unrelated precondition aborted the whole run, separate runs silently merged into the same files, and partial failure reported as success. - Preflight resolves a server CLI (honouring $RESP_BENCH_CLI and $SERVER_PROJECT) and PINGs the endpoint with bounded retry before any benchmark work, passing auth/TLS flags from the driver config. The per-cell FLUSHALL moved inside the guarded region, so a transient failure costs one cell rather than the sweep. Recording-only matrices skip both the probe and the flush. - Results are written under //, with --run-id defaulting to a UTC timestamp; a populated run directory is refused unless --resume or --overwrite is given. The orchestrator maintains an /latest symlink. - _manifest.json records per-cell outcome (status, error summary, duration, records written) plus a planned/attempted/succeeded/failed summary, and the unconditional success message is gone. Exit codes are now a contract: 0 = every attempted cell succeeded, 1 = the sweep finished with at least one failed cell, 2 = preflight failed and nothing ran. make benchmark-matrix-graphs is updated in the same change so main is never left pointing the graph generator at the base output dir: it now reads $(OUTPUT_DIR)/$(RUN_ID) with RUN_ID?=latest. Signed-off-by: James Xin --- Makefile | 8 +- README.md | 12 +- docs/BENCHMARK_MATRIX.md | 87 ++- scripts/run_benchmark_matrix.py | 587 ++++++++++++++- scripts/tests/test_e2e_pipeline.py | 59 +- .../tests/test_matrix_failure_visibility.py | 667 ++++++++++++++++++ 6 files changed, 1356 insertions(+), 64 deletions(-) create mode 100644 scripts/tests/test_matrix_failure_visibility.py diff --git a/Makefile b/Makefile index eb1e22e..db7d5d2 100644 --- a/Makefile +++ b/Makefile @@ -403,6 +403,10 @@ config-editor-dev: MATRIX?=configs/matrices/driver-comparison-high-tps.json GRAPHS_DIR?=graphs/interactive/ +# Results live in $(OUTPUT_DIR)//; the orchestrator points 'latest' at +# the most recent run. Override RUN_ID to graph a specific run. +RUN_ID?=latest +MATRIX_RESULTS_DIR=$(OUTPUT_DIR)/$(RUN_ID) benchmark-matrix: java-build python scripts/run_benchmark_matrix.py \ @@ -417,8 +421,10 @@ benchmark-matrix-dry-run: --dry-run benchmark-matrix-graphs: + @test -n "$(OUTPUT_DIR)" || { echo "ERROR: OUTPUT_DIR is required, e.g. make benchmark-matrix-graphs OUTPUT_DIR=results/my-run" >&2; exit 1; } + @test -d "$(MATRIX_RESULTS_DIR)" || { echo "ERROR: $(MATRIX_RESULTS_DIR) is not a directory — no run has completed in $(OUTPUT_DIR), or 'latest' is stale. Run 'make benchmark-matrix OUTPUT_DIR=$(OUTPUT_DIR)' or pass RUN_ID=." >&2; exit 1; } python scripts/generate_interactive_graphs.py \ - $(OUTPUT_DIR) \ + $(MATRIX_RESULTS_DIR) \ --output $(GRAPHS_DIR) # ============================================================================ diff --git a/README.md b/README.md index 94cf01f..92e3af7 100644 --- a/README.md +++ b/README.md @@ -10,6 +10,11 @@ A multi-language benchmark suite for RESP protocol (Redis/Valkey) compatible dat - Python 3.8+, Java 21+, Maven - Make +- A server CLI (`valkey-cli`) for the matrix runner's readiness probe and per-cell + FLUSHALL — the Makefile's `server-*` targets build one into + `work//bin/`, or any `valkey-cli`/`redis-cli` on `PATH` is used. + Set `RESP_BENCH_CLI` to point at a specific binary. Not needed for matrices that + only use the serverless `recording` driver. ### 1. Run a Benchmark Matrix @@ -25,16 +30,21 @@ make server-standalone-start python scripts/run_benchmark_matrix.py \ --matrix configs/matrices/driver-comparison-high-tps.json \ --output-dir results/my-run \ + --run-id first-try \ --server-host localhost +# Results land in results/my-run// (--run-id defaults to a UTC timestamp). +# Exit code: 0 = all cells ran, 1 = some cell failed, 2 = preflight failed. ``` ### 2. Generate Interactive Graphs ```bash python scripts/generate_interactive_graphs.py \ - results/my-run/ \ + results/my-run/first-try/ \ --output graphs/interactive/my-run/ \ --title "My Benchmark Run" +# Or, for whichever run finished most recently: +# make benchmark-matrix-graphs OUTPUT_DIR=results/my-run # Open graphs/interactive/my-run/scalability_and_delta.html in a browser ``` diff --git a/docs/BENCHMARK_MATRIX.md b/docs/BENCHMARK_MATRIX.md index 6bddd7a..ca0bb16 100644 --- a/docs/BENCHMARK_MATRIX.md +++ b/docs/BENCHMARK_MATRIX.md @@ -13,15 +13,16 @@ python scripts/run_benchmark_matrix.py \ --output-dir results/valkey-glide-sweep \ --dry-run -# Run the matrix benchmark +# Run the matrix benchmark (results land in results/valkey-glide-sweep//) python scripts/run_benchmark_matrix.py \ --matrix configs/matrices/valkey-glide-thread-sweep.json \ --output-dir results/valkey-glide-sweep \ + --run-id first-try \ --server-host 10.0.0.5 -# Generate interactive graphs from results +# Generate interactive graphs from one run's results python scripts/generate_interactive_graphs.py \ - results/valkey-glide-sweep/ \ + results/valkey-glide-sweep/first-try/ \ --output graphs/interactive/valkey-glide-sweep/ ``` @@ -30,6 +31,11 @@ Or via Makefile: make benchmark-matrix-dry-run MATRIX=configs/matrices/valkey-glide-thread-sweep.json make benchmark-matrix MATRIX=configs/matrices/valkey-glide-thread-sweep.json \ OUTPUT_DIR=results/glide-sweep SERVER_HOST=10.0.0.5 + +# Graphs for the run that just finished (follows OUTPUT_DIR/latest) +make benchmark-matrix-graphs OUTPUT_DIR=results/glide-sweep +# ...or for a specific run +make benchmark-matrix-graphs OUTPUT_DIR=results/glide-sweep RUN_ID=20260321T140322Z ``` ## Matrix Config Format @@ -136,19 +142,77 @@ Non-matching drivers skip the dimension entirely, avoiding wasted benchmark time ## Output Format -The matrix runner produces a **flat directory**: +The matrix runner produces a **flat directory per run**, under `//`: ``` results/glide-sweep/ - spring-data-valkey-glide@cb=8,tw=8,pool_size=connections.ndjson - spring-data-valkey-glide@cb=16,tw=16,pool_size=connections.ndjson - *.cpu.ndjson # CPU samples per variant - _manifest.json # Maps labels → config metadata + 20260321T140322Z/ + spring-data-valkey-glide@cb=8,tw=8,pool_size=connections.ndjson + spring-data-valkey-glide@cb=16,tw=16,pool_size=connections.ndjson + *.cpu.ndjson # CPU samples per variant + _manifest.json # Maps labels → config metadata + per-cell outcomes + latest -> 20260321T140322Z # symlink to the most recent successful start ``` +The run id defaults to a UTC timestamp, so two runs into the same `--output-dir` +never merge into the same NDJSON files. Pass `--run-id` to name a run yourself; +if that run directory already holds results, the run is refused unless you pass +`--resume` (append deliberately) or `--overwrite` (discard them first). Point the +graph generator at the run directory, not at `--output-dir`. + +Once preflight passes, the orchestrator repoints `/latest` at the +current run, so tooling can find the newest results without knowing the run id +(`--resume` repoints it at the run being appended to). A failed preflight leaves +the link on the previous run, and a `latest` that is a real directory rather than +a symlink is never touched. + Each `.ndjson` file contains STEADY phase records for ALL connection counts (multiple iterations each). The NDJSON format is identical to what the benchmark engine produces — no changes to the output schema. -The `_manifest.json` records the full configuration for each variant, enabling the graph generator to build rich legend labels. +The `_manifest.json` records the full configuration for each variant, enabling the graph generator to build rich legend labels. It also records what actually ran: + +```json +{ + "run_id": "20260321T140322Z", + "variants": { "...": {} }, + "summary": {"planned": 6, "attempted": 6, "succeeded": 5, "failed": 1}, + "cells": [ + { + "iteration": 1, "x_axis": "connections", "x_value": 4, + "label": "jedis", "driver_config": "configs/drivers/default/jedis.json", + "engine": "java", "metrics_output": "jedis.ndjson", + "started_at": "2026-03-21T14:03:22Z", "status": "ok", + "records_written": 1, "duration_seconds": 41.2 + } + ] +} +``` + +A cell counts as failed if the engine exits non-zero, if the pre-cell FLUSHALL +fails, or if the engine exits 0 but writes no new metrics record. + +## Server Preconditions + +Before the first benchmark runs, the orchestrator resolves a CLI binary and +PINGs the server with bounded retry, so an unreachable endpoint fails up front +instead of aborting mid-sweep. The CLI is resolved in this order: + +1. `$RESP_BENCH_CLI` +2. `work//bin/-cli` — the binary the Makefile builds, where + `` is `$SERVER_PROJECT` (default `valkey`) +3. `-cli`, then `valkey-cli`, then `redis-cli` on `PATH` + +Auth and TLS settings from the driver config (`auth.username`, `auth.password`, +`tls.*`) are passed to the probe and to the per-cell FLUSHALL. Matrices built +only from serverless drivers (`driver_id: "recording"`) skip both the probe and +the flush entirely. + +## Exit Codes + +| Code | Meaning | +|------|---------| +| 0 | Every attempted cell succeeded | +| 1 | At least one cell failed (the sweep still ran to the end) | +| 2 | Preflight failed and nothing ran: server unreachable, no CLI binary, populated run directory, or a matrix with no cells | ## CLI Reference @@ -156,7 +220,10 @@ The `_manifest.json` records the full configuration for each variant, enabling t python scripts/run_benchmark_matrix.py --help --matrix, -m Path to matrix configuration JSON file (required) - --output-dir, -o Directory to write benchmark results (required) + --output-dir, -o Base directory for results; results land in // (required) + --run-id Name of this run's subdirectory (default: UTC timestamp) + --resume Allow appending into a run directory that already has results + --overwrite Delete existing results in the run directory first --server-host Server hostname (overrides matrix config) --port Server port (overrides matrix config) --iterations Override iterations from matrix config diff --git a/scripts/run_benchmark_matrix.py b/scripts/run_benchmark_matrix.py index 00da267..4bfcf9d 100644 --- a/scripts/run_benchmark_matrix.py +++ b/scripts/run_benchmark_matrix.py @@ -21,13 +21,21 @@ One dimension is designated as the X axis (typically "connections"). All other free dimensions form the series (one line per unique combo). -Output is a flat directory with one NDJSON file per series label: - /