[Ground Truth] Adding Scribe v2 as an additional ground truth source

README.md

@@ -2,6 +2,8 @@
 
 A framework for benchmarking Speech-to-Text services with TTFS (Time To Final Segment) latency and Semantic WER (Word Error Rate) accuracy measurement.
 
+> **Note:** The results below use ground truth that was human-reviewed after batch transcription. If you reproduce the benchmark yourself, the generated ground truth (via Gemini Flash or ElevenLabs Scribe v2) will not have human review and may contain inaccuracies that affect WER scores. Results should be interpreted as relative comparisons between services rather than absolute accuracy metrics.
+
 ## Results Summary
 
 Benchmark results on 1000 samples from the `pipecat-ai/smart-turn-data-v3.1-train` dataset.
@@ -65,8 +67,9 @@ uv run stt-benchmark download --num-samples 100
 # Run benchmarks
 uv run stt-benchmark run --services deepgram,openai
 
-# Generate ground truth (Gemini)
-uv run stt-benchmark ground-truth
+# Generate ground truth
+uv run stt-benchmark ground-truth                  # Gemini (default)
+uv run stt-benchmark ground-truth -p scribe        # ElevenLabs Scribe v2
 
 # Calculate semantic WER (Claude)
 uv run stt-benchmark wer
@@ -166,9 +169,14 @@ uv run stt-benchmark run --services deepgram --limit 50 --vad-stop-secs 0.3
 ### Generating Ground Truth
 
 ```bash
-# Generate ground truth for all samples
+# Generate ground truth using Gemini (default)
 uv run stt-benchmark ground-truth
 
+# Generate ground truth using ElevenLabs Scribe v2
+uv run stt-benchmark ground-truth -p scribe
+
+# Both providers can coexist — they are stored independently per model
+
 # Interactive review with audio playback
 uv run stt-benchmark ground-truth review <run_id>
 ```
@@ -181,6 +189,9 @@ uv run stt-benchmark wer
 
 # Force recalculate
 uv run stt-benchmark wer --services deepgram --force-recalculate
+
+# Evaluate against a specific ground truth model and store separately
+uv run stt-benchmark wer -s elevenlabs -m v2 --gt-model scribe_v2 --wer-label v2-scribe-gt
 ```
 
 ### Viewing Reports
@@ -215,7 +226,7 @@ stt_benchmark_data/
 |-------|-------------|
 | `samples` | Audio sample metadata |
 | `benchmark_results` | TTFS and transcription results |
-| `ground_truths` | Reference transcriptions (Gemini) |
+| `ground_truth` | Reference transcriptions (keyed by sample + model) |
 | `wer_metrics` | Semantic WER calculations |
 | `semantic_wer_traces` | Full Claude reasoning traces |
 
@@ -246,7 +257,7 @@ The benchmark dataset (audio samples and ground truth transcriptions) is publicl
 
 **[pipecat-ai/stt-benchmark-data](https://huggingface.co/datasets/pipecat-ai/stt-benchmark-data)**
 
-Audio samples are sourced from the `pipecat-ai/smart-turn-data-v3.1-train` dataset. Ground truth transcriptions are generated with Gemini and human-reviewed.
+Audio samples are sourced from the `pipecat-ai/smart-turn-data-v3.1-train` dataset. Ground truth transcriptions are generated via batch transcription using Gemini or ElevenLabs Scribe v2.
 
 ## Documentation
 

src/stt_benchmark/cli/ground_truth.py

@@ -9,6 +9,7 @@
 
 from stt_benchmark.config import get_config
 from stt_benchmark.ground_truth.gemini_transcriber import GeminiTranscriber
+from stt_benchmark.ground_truth.scribe_transcriber import ScribeTranscriber
 from stt_benchmark.storage.database import Database
 
 app = typer.Typer()
@@ -24,11 +25,17 @@ def generate_ground_truth(
         "-n",
         help="Limit number of samples to transcribe",
     ),
-    model: str = typer.Option(
-        "gemini-3-flash-preview",
+    provider: str = typer.Option(
+        "scribe",
+        "--provider",
+        "-p",
+        help="Transcription provider: 'scribe' or 'gemini'",
+    ),
+    model: str | None = typer.Option(
+        None,
         "--model",
         "-m",
-        help="Gemini model to use for transcription",
+        help="Model to use (default: gemini-3-flash-preview for gemini, scribe_v2 for scribe)",
     ),
     force: bool = typer.Option(
         False,
@@ -37,32 +44,45 @@ def generate_ground_truth(
         help="Re-transcribe samples that already have ground truth",
     ),
 ):
-    """Generate ground truth transcriptions using Gemini.
+    """Generate ground truth transcriptions using Gemini or ElevenLabs Scribe.
 
-    Uses Google's Gemini model to transcribe audio samples,
-    creating reference transcriptions for WER calculation.
+    Uses Google's Gemini model or ElevenLabs Scribe v2 to transcribe audio
+    samples, creating reference transcriptions for WER calculation.
 
-    Requires GOOGLE_API_KEY environment variable.
+    Requires GOOGLE_API_KEY (for gemini) or ELEVENLABS_API_KEY (for scribe).
 
     Subcommands:
       iterate  - Run a repeatable transcription iteration (saves to JSONL)
       list     - List available transcription runs
       review   - Interactive review of transcription runs with audio playback
     """
-    # If a subcommand was invoked, don't run the default behavior
     if ctx.invoked_subcommand is not None:
         return
 
     config = get_config()
+    provider = provider.lower()
 
-    if not config.google_api_key:
-        console.print("[red]Error: GOOGLE_API_KEY not set[/red]")
-        console.print("\nSet the environment variable:")
-        console.print("  export GOOGLE_API_KEY='your-api-key'")
+    if provider == "scribe":
+        if not config.elevenlabs_api_key:
+            console.print("[red]Error: ELEVENLABS_API_KEY not set[/red]")
+            console.print("\nSet the environment variable:")
+            console.print("  export ELEVENLABS_API_KEY='your-api-key'")
+            raise typer.Exit(1)
+        effective_model = model or "scribe_v2"
+    elif provider == "gemini":
+        if not config.google_api_key:
+            console.print("[red]Error: GOOGLE_API_KEY not set[/red]")
+            console.print("\nSet the environment variable:")
+            console.print("  export GOOGLE_API_KEY='your-api-key'")
+            raise typer.Exit(1)
+        effective_model = model or "gemini-3-flash-preview"
+    else:
+        console.print(f"[red]Error: Unknown provider '{provider}'. Use 'gemini' or 'scribe'.[/red]")
         raise typer.Exit(1)
 
     console.print("\n[bold blue]STT Benchmark - Generate Ground Truth[/bold blue]\n")
-    console.print(f"Model: {model}")
+    console.print(f"Provider: {provider}")
+    console.print(f"Model: {effective_model}")
     if limit:
         console.print(f"Limit: {limit}")
     console.print(f"Force re-transcribe: {force}")
@@ -71,11 +91,10 @@ async def run():
         db = Database()
         await db.initialize()
 
-        # Get samples
         if force:
             samples = await db.get_all_samples()
         else:
-            samples = await db.get_samples_without_ground_truth()
+            samples = await db.get_samples_without_ground_truth(model_used=effective_model)
 
         if not samples:
             console.print("\n[green]All samples already have ground truth![/green]")
@@ -86,8 +105,10 @@ async def run():
 
         console.print(f"Samples to transcribe: {len(samples)}\n")
 
-        # Create transcriber
-        transcriber = GeminiTranscriber(model_name=model)
+        if provider == "scribe":
+            transcriber = ScribeTranscriber(model_name=effective_model)
+        else:
+            transcriber = GeminiTranscriber(model_name=effective_model)
 
         with Progress(
             TextColumn("[progress.description]{task.description}"),
@@ -110,13 +131,11 @@ def callback(current, total, sample_id):
 
             progress.update(task, completed=len(samples))
 
-        # Summary
         console.print(f"\n[green]✓ Generated {len(results)} ground truth transcriptions[/green]")
 
-        # Show stats
-        gt_count = await db.get_ground_truth_count()
+        gt_count = await db.get_ground_truth_count(model_used=effective_model)
         sample_count = await db.get_sample_count()
-        console.print(f"\nGround truth coverage: {gt_count}/{sample_count} samples")
+        console.print(f"\nGround truth coverage ({effective_model}): {gt_count}/{sample_count} samples")
 
         await db.close()
 

src/stt_benchmark/cli/main.py

@@ -18,7 +18,7 @@
 # Add subcommands
 app.add_typer(download_app, name="download", help="Download and prepare audio samples")
 app.add_typer(benchmark_app, name="run", help="Run STT benchmarks")
-app.add_typer(ground_truth_app, name="ground-truth", help="Generate ground truth using Gemini")
+app.add_typer(ground_truth_app, name="ground-truth", help="Generate ground truth transcriptions")
 app.add_typer(wer_app, name="wer", help="Calculate semantic WER metrics")
 app.add_typer(report_app, name="report", help="Generate reports and compare services")
 app.add_typer(export_app, name="export", help="Export data for a specific service")

src/stt_benchmark/cli/wer.py

@@ -45,7 +45,18 @@ def calculate_wer(
         None,
         "--model",
         "-m",
-        help="Model name filter",
+        help="Model name filter (for the STT service result)",
+    ),
+    gt_model: str | None = typer.Option(
+        None,
+        "--gt-model",
+        help="Ground truth model to evaluate against (e.g. 'scribe_v2', 'gemini-3-flash-preview')",
+    ),
+    wer_label: str | None = typer.Option(
+        None,
+        "--wer-label",
+        help="Label for WER results (stored as model_name in wer_metrics). "
+        "Use to avoid overwriting existing WER results.",
     ),
     force_recalculate: bool = typer.Option(
         False,
@@ -103,6 +114,10 @@ def calculate_wer(
     console.print(f"Services: {', '.join(s.value for s in service_list)}")
     if model:
         console.print(f"Model filter: {model}")
+    if gt_model:
+        console.print(f"Ground truth model: {gt_model}")
+    if wer_label:
+        console.print(f"WER label: {wer_label}")
     if force_recalculate:
         console.print("[yellow]Force recalculate: ON[/yellow]")
 
@@ -142,24 +157,31 @@ async def run():
 
         all_stats = []
 
+        # When wer_label is set but model is not, default to empty string
+        # so we only process the default model results, not all models
+        effective_model = model if model is not None else ("" if wer_label else None)
+        effective_wer_label = wer_label or effective_model
+
         for service_name in service_list:
             console.print(f"\n[bold]Evaluating semantic WER for {service_name.value}...[/bold]")
 
             # Delete existing WER metrics if force recalculate
             if force_recalculate:
-                await db.delete_wer_metrics_for_service(service_name, model)
-                await db.delete_semantic_wer_traces_for_service(service_name, model)
+                await db.delete_wer_metrics_for_service(service_name, effective_wer_label)
+                await db.delete_semantic_wer_traces_for_service(service_name, effective_wer_label)
                 console.print("  [yellow]Deleted existing WER metrics and traces[/yellow]")
 
             # Get samples that need WER calculation
-            pending = await db.get_samples_without_wer(service_name, model)
+            pending = await db.get_samples_without_wer(
+                service_name, effective_model, wer_label=effective_wer_label
+            )
 
             if not pending:
                 console.print("  All samples already have WER metrics")
                 # Still show existing stats
-                metrics = await db.get_wer_metrics_for_service(service_name, model)
+                metrics = await db.get_wer_metrics_for_service(service_name, effective_wer_label)
                 if metrics:
-                    stats = compute_wer_stats(service_name, metrics)
+                    stats = compute_wer_stats(service_name, metrics, effective_wer_label)
                     all_stats.append(stats)
                     console.print(f"  Mean WER: {stats['wer_mean']:.2%}")
                 continue
@@ -180,16 +202,18 @@ def callback(current, total, sample_id, task_id=progress_task):
 
                 metrics = await evaluator.evaluate_service(
                     service_name,
-                    model_name=model,
+                    model_name=effective_model,
+                    gt_model=gt_model,
+                    wer_label=wer_label,
                     progress_callback=callback,
                 )
 
                 progress.update(progress_task, completed=len(pending))
 
             # Get all metrics for stats
-            all_metrics = await db.get_wer_metrics_for_service(service_name, model)
+            all_metrics = await db.get_wer_metrics_for_service(service_name, effective_wer_label)
             if all_metrics:
-                stats = compute_wer_stats(service_name, all_metrics)
+                stats = compute_wer_stats(service_name, all_metrics, effective_wer_label)
                 all_stats.append(stats)
                 console.print(f"  [green]Completed: {len(metrics)} samples[/green]")
                 console.print(f"  Mean Semantic WER: {stats['wer_mean']:.2%}")
@@ -205,17 +229,19 @@ def callback(current, total, sample_id, task_id=progress_task):
     asyncio.run(run())
 
 
-def compute_wer_stats(service_name: ServiceName, metrics: list) -> dict:
+def compute_wer_stats(
+    service_name: ServiceName, metrics: list, model_name: str | None = None
+) -> dict:
     """Compute aggregate semantic WER statistics."""
     wer_values = [m.wer for m in metrics if m.wer < float("inf")]
 
-    # Compute pooled WER
     total_errors = sum(m.substitutions + m.deletions + m.insertions for m in metrics)
     total_ref_words = sum(m.reference_words for m in metrics)
     pooled_wer = total_errors / total_ref_words if total_ref_words > 0 else 0.0
 
     return {
         "service_name": service_name,
+        "model_name": model_name or "",
         "num_samples": len(metrics),
         "wer_mean": statistics.mean(wer_values) if wer_values else 0.0,
         "wer_median": statistics.median(wer_values) if wer_values else 0.0,
@@ -226,24 +252,34 @@ def compute_wer_stats(service_name: ServiceName, metrics: list) -> dict:
     }
 
 
+def _format_service_label(stats: dict) -> str:
+    """Format service name with model for display."""
+    name = stats["service_name"].value
+    model = stats.get("model_name", "")
+    if model:
+        return f"{name} ({model})"
+    return name
+
+
 def print_wer_summary(stats_list: list[dict]):
     """Print semantic WER summary table."""
     table = Table(title="Semantic WER Summary")
 
     table.add_column("Service", style="cyan", no_wrap=True)
+    table.add_column("Model", style="dim")
     table.add_column("Samples", justify="right")
     table.add_column("WER Mean", justify="right")
     table.add_column("WER Median", justify="right")
     table.add_column("WER Min", justify="right")
     table.add_column("WER Max", justify="right")
     table.add_column("Pooled WER", justify="right")
 
-    # Sort by mean WER
     sorted_stats = sorted(stats_list, key=lambda x: x["wer_mean"])
 
     for stats in sorted_stats:
         table.add_row(
             stats["service_name"].value,
+            stats.get("model_name", ""),
             str(stats["num_samples"]),
             f"{stats['wer_mean']:.2%}",
             f"{stats['wer_median']:.2%}",
@@ -254,8 +290,8 @@ def print_wer_summary(stats_list: list[dict]):
 
     console.print(table)
 
-    # Rankings
     console.print("\n[bold]Rankings (by mean semantic WER, lower is better):[/bold]")
     for i, stats in enumerate(sorted_stats, 1):
+        label = _format_service_label(stats)
         medal = "🥇" if i == 1 else "🥈" if i == 2 else "🥉" if i == 3 else f"{i}."
-        console.print(f"  {medal} {stats['service_name'].value}: {stats['wer_mean']:.2%}")
+        console.print(f"  {medal} {label}: {stats['wer_mean']:.2%}")