Add Python benchmark engine (async valkey-glide, redis-py, valkey-py)

.github/workflows/benchmark.yml

@@ -152,8 +152,77 @@ jobs:
           path: ${{ steps.names.outputs.result_file }}
           retention-days: 30
 
+  benchmark-python:
+    runs-on: ubuntu-latest
+
+    strategy:
+      fail-fast: false
+      matrix:
+        driver:
+          - configs/drivers/default/redis-py.json
+          - configs/drivers/default/valkey-glide-python.json
+        workload:
+          - configs/workloads/reference/basic-standalone-single-client-1M-reqs.json
+
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: '3.12'
+
+      - name: Install build dependencies
+        run: |
+          sudo apt-get update
+          sudo apt-get install -y build-essential
+
+      - name: Install Python engine
+        run: cd python && pip install -e .
+
+      - name: Start Valkey server
+        run: |
+          # Use Makefile target which builds from source and configures with persistence disabled
+          make server-standalone-start
+          # Wait for server to be ready
+          sleep 2
+          # Verify server is up and persistence is disabled
+          work/valkey/bin/valkey-cli ping
+          work/valkey/bin/valkey-cli CONFIG GET save
+
+      - name: Extract names for result file
+        id: names
+        run: |
+          DRIVER_NAME=$(basename ${{ matrix.driver }} .json)
+          WORKLOAD_NAME=$(basename ${{ matrix.workload }} .json)
+          echo "driver_name=$DRIVER_NAME" >> $GITHUB_OUTPUT
+          echo "workload_name=$WORKLOAD_NAME" >> $GITHUB_OUTPUT
+          echo "result_file=results/github-runner/reference/${DRIVER_NAME}-${WORKLOAD_NAME}.ndjson" >> $GITHUB_OUTPUT
+
+      - name: Run benchmark
+        run: |
+          mkdir -p results/github-runner/reference
+          python -m resp_bench \
+            --server localhost:6379 \
+            --driver ${{ matrix.driver }} \
+            --workload ${{ matrix.workload }} \
+            --metrics ${{ steps.names.outputs.result_file }} \
+            --commit-id ${{ github.sha }}
+
+      - name: Stop Valkey server
+        if: always()
+        run: |
+          make server-standalone-stop || true
+
+      - name: Upload results
+        uses: actions/upload-artifact@v4
+        with:
+          name: benchmark-python-${{ steps.names.outputs.driver_name }}-${{ steps.names.outputs.workload_name }}
+          path: ${{ steps.names.outputs.result_file }}
+          retention-days: 30
+
   generate-graphs:
-    needs: [benchmark-java, benchmark-ruby]
+    needs: [benchmark-java, benchmark-ruby, benchmark-python]
     runs-on: ubuntu-latest
     permissions:
       contents: write

Makefile

@@ -31,7 +31,7 @@ WORK_DIR=$(shell pwd)/work
 	server-sentinel-start server-sentinel-stop \
 	server-start server-stop \
 	java-build java-test java-run java-clean \
-	python-build python-test python-run python-clean \
+	python-build python-test python-run python-clean python-info \
 	ruby-build ruby-test ruby-run ruby-clean ruby-info \
 	csharp-build csharp-test csharp-run csharp-clean csharp-info \
 	config-editor-build config-editor-dev
@@ -56,10 +56,11 @@ help:
 	@echo "  make java-run               Run Java benchmark (requires DRIVER and WORKLOAD)"
 	@echo "  make java-clean             Clean Java build artifacts"
 	@echo ""
-	@echo "Python Engine (placeholder):"
-	@echo "  make python-build           Build Python benchmark engine"
+	@echo "Python Engine:"
+	@echo "  make python-build           Install Python benchmark engine (pip install -e .)"
 	@echo "  make python-test            Run Python tests"
-	@echo "  make python-run             Run Python benchmark"
+	@echo "  make python-run             Run Python benchmark (requires DRIVER and WORKLOAD)"
+	@echo "  make python-clean           Clean Python build artifacts"
 	@echo ""
 	@echo "Ruby Engine:"
 	@echo "  make ruby-build             Install Ruby dependencies"
@@ -305,24 +306,27 @@ java-info: java-build
 	java -jar $(JAVA_JAR) --info
 
 # ============================================================================
-# Python Engine (Placeholder)
+# Python Engine
 # ============================================================================
 
 python-build:
-	@echo "Python engine not yet implemented"
-	@echo "Placeholder for: cd python && pip install -e ."
+	cd python && pip install -e .
 
 python-test:
-	@echo "Python engine not yet implemented"
-	@echo "Placeholder for: cd python && pytest"
+	cd python && python -m pytest
 
-python-run:
-	@echo "Python engine not yet implemented"
-	@echo "Placeholder for: python -m resp_bench --server $(SERVER) --driver $(DRIVER) --workload $(WORKLOAD)"
+python-run: python-build
+	python -m resp_bench \
+		--server $(SERVER) \
+		--driver $(DRIVER) \
+		--workload $(WORKLOAD) \
+		--metrics $(METRICS_OUTPUT)
+
+python-info: python-build
+	python -m resp_bench --info
 
 python-clean:
-	@echo "Python engine not yet implemented"
-	@echo "Placeholder for: cd python && rm -rf __pycache__ *.egg-info dist build"
+	cd python && rm -rf .venv .pytest_cache __pycache__ dist build *.egg-info src/*.egg-info
 
 # ============================================================================
 # Ruby Engine

configs/drivers/default/redis-py.json

@@ -0,0 +1,8 @@
+{
+    "schema_version": "1.0",
+    "description": "redis-py async client - default configuration",
+    "driver_id": "redis-py",
+    "mode": "standalone",
+    "command_timeout_ms": 5000,
+    "specific_driver_config": {}
+}

configs/drivers/default/valkey-glide-python.json

@@ -0,0 +1,8 @@
+{
+    "schema_version": "1.0",
+    "description": "valkey-glide Python async client - default configuration",
+    "driver_id": "valkey-glide-python",
+    "mode": "standalone",
+    "command_timeout_ms": 5000,
+    "specific_driver_config": {}
+}

configs/drivers/example-redis-py-standalone.json

@@ -0,0 +1,8 @@
+{
+  "schema_version": "1.0",
+  "description": "redis-py async client - standalone mode",
+  "driver_id": "redis-py",
+  "mode": "standalone",
+  "command_timeout_ms": 5000,
+  "specific_driver_config": {}
+}

configs/drivers/example-valkey-glide-python-standalone.json

@@ -0,0 +1,8 @@
+{
+  "schema_version": "1.0",
+  "description": "valkey-glide Python async client - standalone mode",
+  "driver_id": "valkey-glide-python",
+  "mode": "standalone",
+  "command_timeout_ms": 5000,
+  "specific_driver_config": {}
+}

configs/drivers/high-throughput/redis-py.json

@@ -0,0 +1,8 @@
+{
+    "schema_version": "1.0",
+    "description": "redis-py async client - high-throughput configuration",
+    "driver_id": "redis-py",
+    "mode": "standalone",
+    "specific_driver_config": {},
+    "command_timeout_ms": 10000
+}

configs/drivers/high-throughput/valkey-glide-python.json

@@ -0,0 +1,8 @@
+{
+    "schema_version": "1.0",
+    "description": "valkey-glide Python async client - high-throughput configuration",
+    "driver_id": "valkey-glide-python",
+    "mode": "standalone",
+    "specific_driver_config": {},
+    "command_timeout_ms": 10000
+}

docs/ADDING_LANGUAGE.md

@@ -178,9 +178,10 @@ class MetricsCollector:
 
     def record(self, command: str, latency_us: int, success: bool) -> None:
         if command not in self.command_metrics:
-            # 1µs to 1 hour, 3 significant figures
+            # 1µs to 600s, 3 significant figures (must match the other engines:
+            # Java/C#/Ruby all use a max of 600_000_000µs, not 1 hour)
             self.command_metrics[command] = CommandMetrics(
-                histogram=HdrHistogram(1, 3600000000, 3)
+                histogram=HdrHistogram(1, 600000000, 3)
             )
 
         metrics = self.command_metrics[command]

python/.gitignore

@@ -0,0 +1,7 @@
+.venv/
+__pycache__/
+*.egg-info/
+*.pyc
+dist/
+build/
+.pytest_cache/

python/README.md

@@ -1,67 +1,79 @@
 # resp-bench Python Engine
 
-🚧 **This engine is planned but not yet implemented.**
+Python implementation of the resp-bench benchmark suite, at parity with the
+Java (reference), Ruby, and C# engines.
 
-## Overview
+## Supported Drivers
 
-Python implementation of the resp-bench benchmark suite.
+| Driver | `driver_id` | Package | Notes |
+|--------|-------------|---------|-------|
+| Valkey GLIDE | `valkey-glide-python` | `valkey-glide` (`import glide`) | Async client |
+| redis-py | `redis-py` | `redis` (`redis.asyncio`) | Async client, RESP3, retries disabled |
+| Recording | `recording` | — | In-memory; for server-free tests |
 
-## Planned Drivers
+Peer drivers are kept deliberately few so each one's configuration can be held
+equivalent (same RESP version, same retry policy, same command timeout). A
+`valkey-py` driver is a planned follow-up.
 
-| Driver | Package | Status |
-|--------|---------|--------|
-| redis-py | `redis` | 📋 Planned |
-| redis-py-async | `redis[hiredis]` | 📋 Planned |
-| valkey-glide | `valkey-glide` | 📋 Planned |
+> The GLIDE `driver_id` is `valkey-glide-python` (not the bare `valkey-glide`,
+> which is the Java driver) — matching the `valkey-glide-ruby` /
+> `valkey-glide-csharp` convention.
 
-## Planned Features
+## Execution model
 
-- Full parity with Java engine
-- Async/await based execution using `asyncio`
-- HdrHistogram for latency collection
-- NDJSON metrics output
+The engine is asyncio-based. For a phase with `connections = N`, it creates
+**N client instances** (one client per connection — the `client == connection`
+invariant shared by every engine) and runs **N worker coroutines** concurrently
+on a single event loop. Each worker awaits one command at a time, i.e.
+`pipeline_depth = 1` — the faithful async analogue of the Java/Ruby
+"one in-flight request per connection" model, keeping results comparable across
+engines.
 
-## Contributing
+This one-client-per-connection mapping is this engine's baseline; it is not a
+property of the whole suite (among other engines' drivers, `lettuce` and
+`redis-rb` are 1:1, but `jedis`/`redisson` pool, `spring-data-*` share a
+template, and `stackexchange-redis` multiplexes). Sharing a single multiplexing client across
+workers was proposed and declined upstream
+([ikolomi/resp-bench#11](https://github.com/ikolomi/resp-bench/issues/11)) in
+favour of keeping this baseline.
 
-We welcome contributions to implement the Python engine! Please see:
-- [Architecture Documentation](../docs/ARCHITECTURE.md)
-- [Adding a Language Guide](../docs/ADDING_LANGUAGE.md)
+### Known limits
 
-## Directory Structure (Planned)
+- **`pipeline_depth > 1`** (multiple in-flight requests per connection) is not
+  implemented; such a phase runs at depth 1 and logs a warning.
+- **Single event loop.** Above ~128 connections the event loop, not the driver,
+  becomes the bottleneck, and loop queuing delay is attributed to the driver in
+  the reported latency. The engine warns past that threshold. The Java engine hit
+  the same ceiling with one command-issuing thread and added multiple issuer
+  threads; this engine has no equivalent yet, so high-connection-count Python
+  numbers are not directly comparable to other engines.
 
-```
-python/
-├── README.md
-├── pyproject.toml
-├── requirements.txt
-└── src/
-    └── resp_bench/
-        ├── __init__.py
-        ├── __main__.py
-        ├── client/
-        │   ├── __init__.py
-        │   ├── interface.py
-        │   └── impl/
-        │       └── redis_py.py
-        ├── command/
-        ├── config/
-        ├── engine/
-        └── metrics/
-```
-
-## Usage (Future)
+## Installation
 
 ```bash
-# Install
 pip install -e .
+# with test tooling:
+pip install -e ".[dev]"
+```
+
+## Usage
 
-# Run benchmark
+```bash
 python -m resp_bench \
   --server localhost:6379 \
-  --driver ../configs/drivers/example-redis-py-standalone.json \
+  --driver ../configs/drivers/default/redis-py.json \
   --workload ../configs/workloads/example-workload.json \
   --metrics output.ndjson
 
-# Show supported drivers
+# Show supported drivers and commands
 python -m resp_bench --info
 ```
+
+## Testing
+
+```bash
+pytest                      # unit + recording-driver integration (no server needed)
+```
+
+See [../docs/ADDING_LANGUAGE.md](../docs/ADDING_LANGUAGE.md) and
+[../docs/ARCHITECTURE.md](../docs/ARCHITECTURE.md) for the shared contracts.

python/pyproject.toml

@@ -0,0 +1,40 @@
+[build-system]
+requires = ["setuptools>=68"]
+build-backend = "setuptools.build_meta"
+
+[project]
+name = "resp-bench-python"
+version = "0.1.0"
+description = "Python benchmark engine for resp-bench (async valkey-glide and redis-py)"
+readme = "README.md"
+requires-python = ">=3.10"
+license = { text = "Apache-2.0" }
+# Pinned to exact released versions for reproducible benchmark runs, matching the
+# Ruby/Java/C# engines. Client-library defaults that affect throughput (RESP
+# version, pool size, socket timeouts, retry counts) have changed between minor
+# releases, so a range would make results depend on the resolution date.
+# hiredis is pinned in deliberately so redis-py always uses its compiled parser
+# (GLIDE parses in Rust); the parser actually in use is recorded in the metrics
+# metadata.
+dependencies = [
+    "valkey-glide==2.5.2",
+    "redis==8.1.0",
+    "hiredis==3.4.1",
+    "hdrhistogram==0.10.7",
+]
+
+[project.optional-dependencies]
+dev = [
+    "pytest>=8.0",
+    "pytest-asyncio>=0.23",
+]
+
+[project.scripts]
+resp-bench = "resp_bench.cli:main"
+
+[tool.setuptools.packages.find]
+where = ["src"]
+
+[tool.pytest.ini_options]
+asyncio_mode = "auto"
+testpaths = ["tests"]

python/src/resp_bench/__init__.py

@@ -0,0 +1,11 @@
+"""resp-bench Python engine.
+
+Async implementation of the resp-bench benchmark suite, at parity with the
+Java (reference), Ruby, and C# engines. Drives async clients (valkey-glide,
+redis-py asyncio) on a single asyncio event loop with one client per
+connection.
+"""
+
+from .version import VERSION
+
+__all__ = ["VERSION"]