Add Reson8 turns STT service + median-of-N WER scoring

README.md

@@ -23,6 +23,7 @@ Benchmark results on 1000 samples from the `pipecat-ai/smart-turn-data-v3.1-trai
 | NVIDIA | Nemotron 3.0 ASR (en) | 100.0% | 76.1% | 1.90% | 1.95% | 221ms | 238ms | 252ms |
 | NVIDIA | Nemotron 3.5 ASR (multilingual) | 99.6% | 62.0% | 4.54% | 4.58% | 236ms | 253ms | 266ms |
 | OpenAI | gpt-4o-transcribe | 99.3% | 75.9% | 3.24% | 3.06% | 637ms | 965ms | 1655ms |
+| Reson8 | Resonant-1 Turns | 99.9% | 85.6% | 1.39% | 1.10% | 326ms | 639ms | 1156ms |
 | Smallest AI | pulse | 100.0% | 72.4% | 2.30% | 2.37% | 398ms | 533ms | 1593ms |
 | Soniox | stt-rt-v4 | 99.8% | 84.1% | 1.25% | 1.29% | 249ms | 281ms | 310ms |
 | Speechmatics | N/A | 99.7% | 83.2% | 1.40% | 1.07% | 495ms | 676ms | 736ms |

env.example

@@ -52,6 +52,9 @@ SAGEMAKER_ASR_ENDPOINT_NAME=
 # OpenAI
 OPENAI_API_KEY=
 
+# Reson8
+RESON8_API_KEY=
+
 # Sarvam
 SARVAM_API_KEY=
 

src/stt_benchmark/cli/wer.py

@@ -59,6 +59,11 @@ def calculate_wer(
         "-t",
         help="Use test database (test_results.db) instead of main database",
     ),
+    passes: int = typer.Option(
+        1,
+        "--passes",
+        help="Score each sample this many times and keep the median (suppresses scorer non-determinism). Use an odd value.",
+    ),
 ):
     """Calculate semantic WER metrics for transcription results.
 
@@ -131,7 +136,7 @@ async def run():
 
         console.print(f"Ground truth coverage: {gt_count}/{sample_count} samples\n")
 
-        evaluator = SemanticWEREvaluator(db_path=db_path)
+        evaluator = SemanticWEREvaluator(db_path=db_path, num_passes=passes)
 
         console.print("Warming prompt cache...", end=" ")
         try:

src/stt_benchmark/config.py

@@ -41,6 +41,7 @@ class BenchmarkConfig(BaseSettings):
     mistral_api_key: str = Field(default="", alias="MISTRAL_API_KEY")
     nvidia_api_key: str = Field(default="", alias="NVIDIA_API_KEY")
     openai_api_key: str = Field(default="", alias="OPENAI_API_KEY")
+    reson8_api_key: str = Field(default="", alias="RESON8_API_KEY")
     sagemaker_asr_endpoint_name: str = Field(default="", alias="SAGEMAKER_ASR_ENDPOINT_NAME")
     sarvam_api_key: str = Field(default="", alias="SARVAM_API_KEY")
     smallest_api_key: str = Field(default="", alias="SMALLEST_API_KEY")

src/stt_benchmark/evaluation/semantic_wer.py

@@ -364,11 +364,17 @@ def __init__(
         model: str = "claude-sonnet-4-5-20250929",
         db_path: Path | None = None,
         max_concurrency: int = 50,
+        num_passes: int = 1,
     ):
         self.config = get_config()
         self.model = model
         self.db = Database(db_path=db_path)
         self._api_semaphore = asyncio.Semaphore(max_concurrency)
+        # The scorer is non-deterministic even at temperature 0 (Claude self-reports the
+        # counts, and its normalization occasionally drops/duplicates a word). With
+        # num_passes > 1 each sample is scored that many times and the median-by-total-errors
+        # pass is kept, suppressing those outlier passes. Use an odd value for a clean median.
+        self.num_passes = max(1, num_passes)
 
         if not self.config.anthropic_api_key:
             raise ValueError("ANTHROPIC_API_KEY not set in environment")
@@ -800,6 +806,34 @@ async def evaluate_with_retry(
 
         return None
 
+    async def evaluate_median(
+        self,
+        reference: str,
+        hypothesis: str,
+        filename: str = "",
+    ) -> tuple[dict, SemanticWERTrace] | None:
+        """Score ``self.num_passes`` times and return the median-by-total-errors pass.
+
+        Passes run sequentially so each still holds a single API-semaphore slot (the
+        caller acquires it once per sample). The returned (result, trace) is a real,
+        self-consistent pass — the one whose total error count is the median — rather
+        than a synthetic blend of per-component medians.
+        """
+        if self.num_passes <= 1:
+            return await self.evaluate_with_retry(reference, hypothesis, filename=filename)
+
+        evals: list[tuple[dict, SemanticWERTrace]] = []
+        for _ in range(self.num_passes):
+            pair = await self.evaluate_with_retry(reference, hypothesis, filename=filename)
+            if pair is not None:
+                evals.append(pair)
+
+        if not evals:
+            return None
+
+        evals.sort(key=lambda er: er[0]["substitutions"] + er[0]["deletions"] + er[0]["insertions"])
+        return evals[(len(evals) - 1) // 2]
+
     async def evaluate_service(
         self,
         service_name: ServiceName,
@@ -847,8 +881,8 @@ async def _eval_sample(sample):
                     logger.warning(f"No ground truth for sample {sample.sample_id}")
                     return
 
-                # Evaluate with Claude (with retry and timeout)
-                eval_pair = await self.evaluate_with_retry(
+                # Evaluate with Claude (median of num_passes, each with retry and timeout)
+                eval_pair = await self.evaluate_median(
                     gt.text, result.transcription, filename=sample.sample_id
                 )
 

src/stt_benchmark/models.py

@@ -34,6 +34,7 @@ class ServiceName(str, Enum):
     NVIDIA_SAGEMAKER = "nvidia_sagemaker"
     OPENAI = "openai"
     OPENAI_REALTIME = "openai_realtime"
+    RESON8 = "reson8"
     SARVAM = "sarvam"
     SARVAM_SAARAS_V3 = "sarvam_saaras_v3"
     SMALLEST = "smallest"

src/stt_benchmark/services.py

@@ -357,6 +357,36 @@ def create_openai_realtime() -> FrameProcessor:
     )
 
 
+def create_reson8() -> FrameProcessor:
+    # Reson8 is not bundled with Pipecat; the service lives in this repo.
+    from stt_benchmark.services_custom.reson8_stt import DEFAULT_URL, Reson8STTService
+
+    # When targeting a local piano build (RESON8_URL set to ws://localhost:...),
+    # there is no gateway to validate the API key and inject the Tuba auth
+    # headers, so pass them directly. Any UUIDs work — billing/vocab degrade
+    # gracefully on the dev server.
+    url = os.getenv("RESON8_URL", DEFAULT_URL)
+    extra_headers: dict[str, str] = {}
+    customer_id = os.getenv("RESON8_CUSTOMER_ID")
+    if customer_id:
+        extra_headers = {
+            "X-Customer-Id": customer_id,
+            "X-Organization-Id": os.getenv("RESON8_ORGANIZATION_ID", customer_id),
+            "X-Client-Id": os.getenv("RESON8_CLIENT_ID", customer_id),
+        }
+
+    return Reson8STTService(
+        api_key=_get_env("RESON8_API_KEY"),
+        url=url,
+        extra_headers=extra_headers,
+        min_patience_seconds=0.2,
+        max_patience_seconds=0.2,
+        settings=Reson8STTService.Settings(
+            language=Language.EN,
+        ),
+    )
+
+
 def create_sarvam() -> FrameProcessor:
     from pipecat.services.sarvam.stt import SarvamSTTService
 
@@ -565,6 +595,12 @@ def create_xai() -> FrameProcessor:
         model_label="gpt-4o-transcribe",
         required_env_vars=["OPENAI_API_KEY"],
     ),
+    "reson8": ServiceDefinition(
+        factory=create_reson8,
+        vendor="Reson8",
+        model_label="Resonant-1 Turns",
+        required_env_vars=["RESON8_API_KEY"],
+    ),
     "sarvam": ServiceDefinition(
         factory=create_sarvam,
         vendor="Sarvam",

src/stt_benchmark/services_custom/__init__.py

@@ -0,0 +1 @@
+"""Custom Pipecat STT services for vendors not yet bundled with Pipecat."""