From 0281d2e357f186814ddb47f65bb92dc17c2cf168 Mon Sep 17 00:00:00 2001 From: Cooper Date: Wed, 5 Aug 2026 10:29:23 -0700 Subject: [PATCH 1/4] [BENCH-623] Persist the TTFA roundtrip / leading-silence split per run MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit TTFA = network roundtrip + leading silence, and the runner already computed both parts per sample before discarding the split. Persist the components as two new metric rows (TTFARoundtrip, TTFALeadingSilence), written only when both are known so they always sum back to the TTFA row — a transport-gated or arrival-only TTFA writes neither. Because metric_type is a plain string dimension everywhere downstream, the matviews, series rollups and aggregates API carry the new metrics with no schema or migration changes; every run's real split lands in the pipeline from the next deploy onward. Runner-only on purpose: the dashboard reads none of this yet. The visualization lands separately once the split has accumulated, so it can be built against real served data. Full runner suite 1249 passed; ruff format and check clean. --- runner/src/coval_bench/providers/base.py | 3 + .../src/coval_bench/providers/tts/_common.py | 1 + runner/src/coval_bench/registries/metrics.py | 19 +++ runner/src/coval_bench/runner/orchestrator.py | 25 ++++ runner/tests/unit/test_metric_registry.py | 7 +- runner/tests/unit/test_orchestrator.py | 111 ++++++++++++++++++ runner/tests/unit/test_tts_common.py | 3 + 7 files changed, 168 insertions(+), 1 deletion(-) diff --git a/runner/src/coval_bench/providers/base.py b/runner/src/coval_bench/providers/base.py index 737e01b5..68f41113 100644 --- a/runner/src/coval_bench/providers/base.py +++ b/runner/src/coval_bench/providers/base.py @@ -67,6 +67,9 @@ class TTSResult: http_version: str | None = None submit_to_headers_ms: float | None = None connection_reused: bool | None = None + # The leading-silence part of ttfa_ms; None when offset detection didn't + # run or failed, in which case ttfa_ms is arrival-only and has no split. + leading_silence_ms: float | None = None # --------------------------------------------------------------------------- diff --git a/runner/src/coval_bench/providers/tts/_common.py b/runner/src/coval_bench/providers/tts/_common.py index b2fbf3d7..a86738cf 100644 --- a/runner/src/coval_bench/providers/tts/_common.py +++ b/runner/src/coval_bench/providers/tts/_common.py @@ -131,6 +131,7 @@ def finalize_tts_result( http_version=http_version, submit_to_headers_ms=submit_to_headers_ms, connection_reused=connection_reused, + leading_silence_ms=offset_ms if ttfa_ms is not None else None, ) diff --git a/runner/src/coval_bench/registries/metrics.py b/runner/src/coval_bench/registries/metrics.py index 4a82ef9c..24510dac 100644 --- a/runner/src/coval_bench/registries/metrics.py +++ b/runner/src/coval_bench/registries/metrics.py @@ -24,6 +24,8 @@ class Metric(StrEnum): TTFT = "TTFT" TTFS = "TTFS" TTFA = "TTFA" + TTFA_ROUNDTRIP = "TTFARoundtrip" + TTFA_LEADING_SILENCE = "TTFALeadingSilence" RTF = "RTF" AUDIO_TO_FINAL = "AudioToFinal" V2V = "V2V" @@ -78,6 +80,23 @@ class MetricSpec(BaseModel, frozen=True): decimals=0, benchmarks=frozenset({Benchmark.TTS}), ), + # Perceived TTFA split: roundtrip (send → first chunk) + leading silence + # (stream start → first audible sample). Written only when both are known, + # so the two rows always sum back to the TTFA row. + Metric.TTFA_ROUNDTRIP: MetricSpec( + display_name="TTFA Network Roundtrip", + units="milliseconds", + direction=MetricDirection.LOWER_IS_BETTER, + decimals=0, + benchmarks=frozenset({Benchmark.TTS}), + ), + Metric.TTFA_LEADING_SILENCE: MetricSpec( + display_name="TTFA Leading Silence", + units="milliseconds", + direction=MetricDirection.LOWER_IS_BETTER, + decimals=0, + benchmarks=frozenset({Benchmark.TTS}), + ), Metric.RTF: MetricSpec( display_name="Real-Time Factor", units="ratio", diff --git a/runner/src/coval_bench/runner/orchestrator.py b/runner/src/coval_bench/runner/orchestrator.py index 6d870777..69d459ac 100644 --- a/runner/src/coval_bench/runner/orchestrator.py +++ b/runner/src/coval_bench/runner/orchestrator.py @@ -775,6 +775,31 @@ async def _run_tts_item( ) ) + # TTFA component rows, only when the split is known so the two + # always sum back to the TTFA row. A nulled TTFA (transport gate) + # or an arrival-only TTFA (offset detection failed) writes neither. + leading_silence_ms = tts_result.leading_silence_ms if tts_result else None + if ttfa_value is not None and leading_silence_ms is not None: + for component, value in ( + (Metric.TTFA_ROUNDTRIP, ttfa_value - leading_silence_ms), + (Metric.TTFA_LEADING_SILENCE, leading_silence_ms), + ): + results.append( + Result( + run_id=run_id, + provider=entry.provider, + model=entry.model, + voice=voice, + benchmark=Benchmark.TTS, + metric_type=component, + metric_value=value, + metric_units=METRIC_SPECS[component].units, + transcript=transcript, + status=ResultStatus.SUCCESS, + error=None, + ) + ) + # 2. WER via Whisper transcription of synthesized audio (skip when synth errored) if item_error is None and audio_path is not None and audio_path.exists(): # Whisper transcription is our measurement instrument, not the provider under diff --git a/runner/tests/unit/test_metric_registry.py b/runner/tests/unit/test_metric_registry.py index 6750384d..eb762f65 100644 --- a/runner/tests/unit/test_metric_registry.py +++ b/runner/tests/unit/test_metric_registry.py @@ -19,6 +19,8 @@ def test_metric_values_match_stored_strings() -> None: "TTFT", "TTFS", "TTFA", + "TTFARoundtrip", + "TTFALeadingSilence", "RTF", "AudioToFinal", "V2V", @@ -33,6 +35,8 @@ def test_units_match_stored_strings() -> None: Metric.TTFT: "seconds", Metric.TTFS: "seconds", Metric.TTFA: "milliseconds", + Metric.TTFA_ROUNDTRIP: "milliseconds", + Metric.TTFA_LEADING_SILENCE: "milliseconds", Metric.RTF: "ratio", Metric.AUDIO_TO_FINAL: "seconds", Metric.V2V: "milliseconds", @@ -43,7 +47,8 @@ def test_units_match_stored_strings() -> None: def test_benchmark_coverage() -> None: assert METRIC_SPECS[Metric.WER].benchmarks == {Benchmark.STT, Benchmark.TTS} - assert METRIC_SPECS[Metric.TTFA].benchmarks == {Benchmark.TTS} + for metric in (Metric.TTFA, Metric.TTFA_ROUNDTRIP, Metric.TTFA_LEADING_SILENCE): + assert METRIC_SPECS[metric].benchmarks == {Benchmark.TTS} for metric in (Metric.TTFT, Metric.TTFS, Metric.RTF, Metric.AUDIO_TO_FINAL): assert METRIC_SPECS[metric].benchmarks == {Benchmark.STT} diff --git a/runner/tests/unit/test_orchestrator.py b/runner/tests/unit/test_orchestrator.py index f50a4441..725093dc 100644 --- a/runner/tests/unit/test_orchestrator.py +++ b/runner/tests/unit/test_orchestrator.py @@ -864,6 +864,7 @@ async def test_tts_http1_downgrade_nulls_ttfa_row(settings: Settings) -> None: error=None, http_version="HTTP/1.1", submit_to_headers_ms=210.0, + leading_silence_ms=45.0, ) provider_inst = MagicMock() @@ -944,6 +945,11 @@ def _fake_get_db_symbols() -> tuple[Any, Any, Any, Any]: assert "HTTP/1.1" in ttfa_rows[0].error assert ttfa_rows[0].http_version == "HTTP/1.1" + # A nulled TTFA writes no component rows either. + assert not [ + r for r in recorded if r.metric_type.startswith("TTFA") and r.metric_type != "TTFA" + ] + assert len(wer_rows) == 1 assert wer_rows[0].status == ResultStatus.SUCCESS @@ -1053,6 +1059,111 @@ def _fake_get_db_symbols() -> tuple[Any, Any, Any, Any]: assert writer.finish_run.call_args.kwargs["status"] == RunStatus.SUCCEEDED +@pytest.mark.asyncio +async def test_tts_ttfa_component_rows_reconcile(settings: Settings) -> None: + """A comparable TTFA with a known split writes component rows that sum back to it.""" + with tempfile.TemporaryDirectory() as tmpdir: + audio_path = Path(tmpdir) / "synth.wav" + audio_path.write_bytes(b"\x00" * 512) + + tts_result = TTSResult( + provider="elevenlabs", + model="eleven_flash_v2_5", + voice="IKne3meq5aSn9XLyUdCD", + ttfa_ms=120.0, + audio_path=audio_path, + error=None, + http_version="HTTP/2", + submit_to_headers_ms=90.0, + connection_reused=True, + leading_silence_ms=45.0, + ) + + provider_inst = MagicMock() + provider_inst.synthesize = AsyncMock(return_value=tts_result) + provider_cls = MagicMock(return_value=provider_inst) + provider_cls.warmup = AsyncMock(return_value=None) + + tts_providers = {"elevenlabs": provider_cls} + + run = _make_run() + writer = _make_stub_writer(run) + + tts_dataset = MagicMock() + tts_dataset.items = [_make_tts_item("hello world")] + + def _load( + dataset_id: str, *, settings: Any, sample_size: int | None = None, rng: Any = None + ) -> Any: + return tts_dataset + + fake_pool = MagicMock() + + @contextlib.asynccontextmanager + async def _fake_pool(s: Any) -> AsyncIterator[MagicMock]: + yield fake_pool + + models_mod = MagicMock() + models_mod.Benchmark = Benchmark + models_mod.Result = Result + models_mod.ResultStatus = ResultStatus + models_mod.RunStatus = RunStatus + + def _fake_get_db_symbols() -> tuple[Any, Any, Any, Any]: + return _fake_pool, MagicMock(return_value=writer), RunStatus, models_mod + + compute_wer_real = __import__("coval_bench.metrics", fromlist=["compute_wer"]).compute_wer + compute_rtf_real = __import__("coval_bench.metrics", fromlist=["compute_rtf"]).compute_rtf + + matrix = [ + *_paused_registry(Benchmark.TTS), + _tts_entry("elevenlabs", "eleven_flash_v2_5", "IKne3meq5aSn9XLyUdCD"), + ] + + with ( + patch( + "coval_bench.runner.orchestrator._get_db_symbols", + side_effect=_fake_get_db_symbols, + ), + patch("coval_bench.runner.orchestrator._get_stt_providers", return_value={}), + patch( + "coval_bench.runner.orchestrator._get_tts_providers", + return_value=tts_providers, + ), + patch("coval_bench.runner.orchestrator._get_load_dataset", return_value=_load), + patch( + "coval_bench.runner.orchestrator._get_metrics", + return_value=(compute_wer_real, compute_rtf_real), + ), + patch( + "coval_bench.runner.orchestrator._transcribe_with_whisper", + return_value="hello world", + ), + ): + await run_benchmarks( + settings=settings, + benchmark_kind="tts", + smoke=True, + matrix_overrides=matrix, + ) + + recorded = [r for call in writer.record_results.call_args_list for r in call.args[0]] + ttfa = [r for r in recorded if r.metric_type == "TTFA"] + roundtrip = [r for r in recorded if r.metric_type == "TTFARoundtrip"] + silence = [r for r in recorded if r.metric_type == "TTFALeadingSilence"] + + assert len(ttfa) == len(roundtrip) == len(silence) == 1 + assert ttfa[0].metric_value == 120.0 + assert roundtrip[0].metric_value == 75.0 + assert silence[0].metric_value == 45.0 + for row in (roundtrip[0], silence[0]): + assert row.status == ResultStatus.SUCCESS + assert row.metric_units == "milliseconds" + assert row.voice == "IKne3meq5aSn9XLyUdCD" + + assert writer.finish_run.call_args.kwargs["status"] == RunStatus.SUCCEEDED + + # --------------------------------------------------------------------------- # 9. test_matrix_overrides # --------------------------------------------------------------------------- diff --git a/runner/tests/unit/test_tts_common.py b/runner/tests/unit/test_tts_common.py index 7c7a54d6..310ca5a3 100644 --- a/runner/tests/unit/test_tts_common.py +++ b/runner/tests/unit/test_tts_common.py @@ -50,6 +50,8 @@ def test_finalize_adds_leading_silence_offset() -> None: assert result.ttfa_ms is not None assert result.ttfa_ms > arrival_ms assert result.ttfa_ms == pytest.approx(arrival_ms + lead_ms, abs=12.0) + assert result.leading_silence_ms is not None + assert result.ttfa_ms == arrival_ms + result.leading_silence_ms assert result.audio_path is not None assert result.audio_path.exists() @@ -155,6 +157,7 @@ def test_finalize_offset_failure_falls_back_and_writes_wav() -> None: ) assert result.ttfa_ms == pytest.approx(arrival_ms) # arrival only, offset dropped + assert result.leading_silence_ms is None # unknown split, not 0.0 assert result.error is None assert result.audio_path is not None assert result.audio_path.exists() From 989cca097eda49aeb6878da216cd6e60fd1dba19 Mon Sep 17 00:00:00 2001 From: Cooper Date: Wed, 5 Aug 2026 13:02:41 -0700 Subject: [PATCH 2/4] [BENCH-623] Move the TTFA breakdown to Latency Variation as stacked bars MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The Latency Variation card gains a Distribution / Breakdown toggle on TTS once the split metrics are served. Breakdown reuses the shared QualityMetricBars chassis (the accuracy chart's): every model as one stacked bar in its own palette color — the same color the Filters sidebar and every other chart use — with texture carrying the split: solid is the network roundtrip, hatched the leading silence. Ranked fastest first with the same frozen-axis horizontal scroll, so it reads identically at three models or thirty. The headline flips to the field's leading-silence share; tooltips give each part's ms and share. All values are window-aggregate averages served by the API: the runner writes the component pair over the same samples, so avg roundtrip + avg silence equals the split runs' average TTFA exactly — labeled "Average TTFA" throughout so the breakdown never reads against the distribution view's medians. Nothing is derived or estimated, and the per-run timeline stays untouched from main. The chassis grows optional stackSegments (fills resolvable per model), svgDefs for the hatch patterns, and tickFormatter; stacked rows must not carry a fill key (recharts prefers row fill over segment fill), and stacked mode keeps mobile tap-to-inspect tooltips since those bars have no click-to-compare action. Verified against a local API seeded with all 29 active registry TTS models: bars, labels and tooltips reconcile, the scroll matches the accuracy chart, and mobile keeps 44px+ targets. Web 108 passed, typecheck and lint clean. --- web/components/charts/QualityMetricBars.tsx | 94 +++++-- .../charts/tooltips/BarTooltip.test.tsx | 26 +- web/components/charts/tooltips/BarTooltip.tsx | 51 +++- web/components/dashboard/BoxPlotSection.tsx | 253 +++++++++++++++--- web/hooks/useChartData.ts | 32 ++- web/hooks/useDashboardState.tsx | 6 +- web/lib/config/metrics.ts | 10 +- web/types/benchmark.types.ts | 14 + 8 files changed, 416 insertions(+), 70 deletions(-) diff --git a/web/components/charts/QualityMetricBars.tsx b/web/components/charts/QualityMetricBars.tsx index 1f30614d..4b9862d5 100644 --- a/web/components/charts/QualityMetricBars.tsx +++ b/web/components/charts/QualityMetricBars.tsx @@ -7,6 +7,7 @@ import React, { type CSSProperties, type ReactElement, type ReactNode, type RefO import { BarChart, Bar, + Cell, XAxis, YAxis, CartesianGrid, @@ -25,12 +26,13 @@ const CHART_BOTTOM_MARGIN = 80; // A row plotted as one bar. Callers pass the numeric value under `valueKey` // (e.g. "averageWER" or "instructionScore"); the extra keys ride along for the -// tooltip/label without the chassis needing to know them. +// tooltip/label without the chassis needing to know them. Stacked rows must +// NOT carry `fill` — recharts prefers a row's own fill over the segment's. type QualityBarRow = { model: string; provider: string; - fill: string; - fillOpacity: number; + fill?: string; + fillOpacity?: number; } & Record; interface QualityMetricBarsProps { @@ -38,6 +40,21 @@ interface QualityMetricBarsProps { data: readonly QualityBarRow[]; /** Which numeric field is the bar height (e.g. "averageWER"). */ valueKey: string; + /** + * Stacked segments drawn instead of the single `valueKey` bar (whose key + * then only sizes the frozen y-axis). Order is bottom-up; the last segment + * gets the rounded top and the caller's `barLabel`. A function fill is + * resolved per row's model, so segments can follow the model palette. + */ + stackSegments?: readonly { + dataKey: string; + fill: string | ((model: string) => string); + fillOpacity?: number; + }[]; + /** Extra SVG defs (e.g. per-model hatch patterns for segment fills). */ + svgDefs?: ReactNode; + /** Y-axis tick formatter; defaults to whole percent. */ + tickFormatter?: (value: number) => string; /** Rotated y-axis caption, e.g. "WER % · lower is better". */ yAxisLabel: string; /** Value tooltip element (caller owns its content/formatting). */ @@ -45,7 +62,7 @@ interface QualityMetricBarsProps { isMobile: boolean; getProviderForModel: (model: string) => string; /** The per-bar list — callers own fill/interaction/aria per bar. */ - children: ReactNode; + children?: ReactNode; /** Bar-top value labels; caller-provided so WER can add its markers. */ barLabel?: BarProps["label"]; onBarClick?: (bar: BarRectangleItem) => void; @@ -67,6 +84,9 @@ interface QualityMetricBarsProps { const QualityMetricBars: React.FC = ({ data, valueKey, + stackSegments, + svgDefs, + tickFormatter = (value) => `${value}%`, yAxisLabel, tooltip, isMobile, @@ -82,7 +102,10 @@ const QualityMetricBars: React.FC = ({ overlay, }) => { const themeColors = useThemeColors(); - const activeOverride = tooltipActive ?? (isMobile ? false : undefined); + // Single-bar charts pair mobile taps with click-to-compare, so their tooltip + // is off there; stacked bars have no click action, so tap-to-inspect stays. + const activeOverride = + tooltipActive ?? (isMobile && !stackSegments ? false : undefined); // Room the diagonal tick labels need left of the first bar, measured on the // providers actually on show since their line is never ellipsized. Shared by // both charts, so the instruction bars get the same exact padding as WER. @@ -115,7 +138,7 @@ const QualityMetricBars: React.FC = ({ axisLine={false} tickLine={false} tick={{ fill: themeColors.axisText, fontSize: 12 }} - tickFormatter={(value) => `${value}%`} + tickFormatter={tickFormatter} label={{ value: yAxisLabel, angle: -90, @@ -170,20 +193,51 @@ const QualityMetricBars: React.FC = ({ isAnimationActive={false} wrapperStyle={{ pointerEvents: "auto" }} /> - onBarClick(bar) - : undefined - } - label={barLabel} - style={barStyle} - > - {children} - + {svgDefs} + {stackSegments ? ( + stackSegments.map((segment, idx) => ( + + {typeof segment.fill === "function" && + data.map((row) => ( + string)( + row.model + )} + /> + ))} + + )) + ) : ( + onBarClick(bar) + : undefined + } + label={barLabel} + style={barStyle} + > + {children} + + )} diff --git a/web/components/charts/tooltips/BarTooltip.test.tsx b/web/components/charts/tooltips/BarTooltip.test.tsx index 35c58452..2475f51f 100644 --- a/web/components/charts/tooltips/BarTooltip.test.tsx +++ b/web/components/charts/tooltips/BarTooltip.test.tsx @@ -3,8 +3,8 @@ import { renderToStaticMarkup } from "react-dom/server"; import { describe, expect, it } from "vitest"; -import type { BarDataPoint } from "../../../types/benchmark.types"; -import CustomBarTooltip from "./BarTooltip"; +import type { BarDataPoint, TtfaBreakdownBar } from "../../../types/benchmark.types"; +import CustomBarTooltip, { TtfaBreakdownTooltip } from "./BarTooltip"; const point: BarDataPoint = { model: "soniox/stt-rt-v5", @@ -39,3 +39,25 @@ describe("CustomBarTooltip", () => { expect(html).not.toContain("Insertions"); }); }); + +describe("TtfaBreakdownTooltip", () => { + it("leads with the total and gives each segment its share", () => { + const bar: TtfaBreakdownBar = { + model: "elevenlabs:eleven_flash_v2_5", + provider: "elevenlabs", + roundtrip: 160, + silence: 40, + ttfa: 200, + }; + const html = renderToStaticMarkup( + + ); + expect(html).toContain("Avg TTFA: 200 ms"); + expect(html).toContain("Network roundtrip: 160 ms (80%)"); + expect(html).toContain("Leading silence: 40 ms (20%)"); + }); +}); diff --git a/web/components/charts/tooltips/BarTooltip.tsx b/web/components/charts/tooltips/BarTooltip.tsx index ea37c186..6a6d53b7 100644 --- a/web/components/charts/tooltips/BarTooltip.tsx +++ b/web/components/charts/tooltips/BarTooltip.tsx @@ -7,7 +7,7 @@ import { DedicatedBadge } from "@/components/shared/DedicatedInferenceInfo"; import { RegionBadge } from "@/components/shared/InferenceRegionInfo"; import { normalizeModelName } from "@/lib/utils/formatters"; import { WER_BREAKDOWN_LABELS } from "@/lib/utils/werBreakdown"; -import type { BarDataPoint } from "@/types/benchmark.types"; +import type { BarDataPoint, TtfaBreakdownBar } from "@/types/benchmark.types"; interface CustomBarTooltipProps extends Partial, @@ -82,4 +82,53 @@ const CustomBarTooltip: React.FC = ({ return null; }; +// Tooltip for the stacked TTFA breakdown bars: the total leads, then the two +// segments with their share of it, so the composition reads off the tooltip +// the same way it reads off the bar. +export const TtfaBreakdownTooltip: React.FC< + Partial, "active" | "payload" | "label">> & { + getProviderForModel?: (model: string) => string; + } +> = ({ active, payload, label, getProviderForModel }) => { + const row = payload?.[0]?.payload as TtfaBreakdownBar | undefined; + if (!active || !row) return null; + const modelKey = String(label ?? ""); + const provider = getProviderForModel?.(modelKey); + const modelLabel = provider + ? `${provider} ${normalizeModelName(modelKey)}` + : normalizeModelName(modelKey); + const parts: [string, number][] = [ + ["Network roundtrip", row.roundtrip], + ["Leading silence", row.silence], + ]; + return ( +
+

{`Model: ${modelLabel}`}

+

+ {`Avg TTFA: ${Math.round(row.ttfa)} ms`} +

+ {parts.map(([text, value]) => ( +

{`${text}: ${Math.round(value)} ms (${Math.round((value / row.ttfa) * 100)}%)`}

+ ))} +
+ ); +}; + export default CustomBarTooltip; diff --git a/web/components/dashboard/BoxPlotSection.tsx b/web/components/dashboard/BoxPlotSection.tsx index c6e50c52..e355ca1c 100644 --- a/web/components/dashboard/BoxPlotSection.tsx +++ b/web/components/dashboard/BoxPlotSection.tsx @@ -3,21 +3,38 @@ "use client"; -import React, { useMemo } from "react"; +import React, { useCallback, useMemo, useState } from "react"; +import { type LabelProps } from "recharts"; import { getModelColor } from "@/lib/utils/colors"; import { normalizeModelName, parseModelKey } from "@/lib/utils/formatters"; import BoxPlot from "@/components/charts/d3/BoxPlot"; +import QualityMetricBars from "@/components/charts/QualityMetricBars"; +import { TtfaBreakdownTooltip } from "@/components/charts/tooltips/BarTooltip"; import Card from "@/components/shared/Card"; import SectionHeader from "@/components/shared/SectionHeader"; import MetricInfo from "@/components/shared/MetricInfo"; import MetricToggle, { useMetricTab } from "@/components/dashboard/MetricToggle"; import { metricAboutNote } from "@/lib/config/metrics"; import { useDashboard } from "@/contexts/DashboardContext"; +import { useThemeColors } from "@/hooks/useThemeColors"; import { useChartHoverTracking } from "@/hooks/useChartHoverTracking"; +// Segments follow each model's palette color (the same one the Filters +// sidebar and every other chart use); texture carries the split — solid is +// the network roundtrip, hatched the leading silence. +function silencePatternId(model: string): string { + return `ttfa-var-silence-${model.replace(/[^a-zA-Z0-9_-]/g, "-")}`; +} + +const TTFA_SEGMENTS = [ + { dataKey: "roundtrip", fill: (model: string) => getModelColor(model) }, + { dataKey: "silence", fill: (model: string) => `url(#${silencePatternId(model)})` }, +] as const; + const BoxPlotSection: React.FC = () => { const { - boxPlotDescription: description, + page, + boxPlotDescription, latencyLabel, getBoxPlotData, getProviderForModel, @@ -25,9 +42,18 @@ const BoxPlotSection: React.FC = () => { crossRegionModels, isMobile, activeMetric, + ttfaBreakdownBars, } = useDashboard(); const trackChartHover = useChartHoverTracking("box_plot"); const metricTab = useMetricTab(); + const themeColors = useThemeColors(); + + // TTS gains a second view: the distribution boxes, or the same models as + // stacked bars splitting average TTFA into roundtrip + leading silence. + // The toggle appears only once component rows exist in the served window. + const [view, setView] = useState<"distribution" | "breakdown">("distribution"); + const hasBreakdown = page === "tts" && ttfaBreakdownBars.length > 0; + const showBreakdown = hasBreakdown && view === "breakdown"; const boxPlotData = useMemo( () => getBoxPlotData(activeMetric), @@ -45,60 +71,203 @@ const BoxPlotSection: React.FC = () => { : undefined; }, [boxPlotData]); + // Headline for the breakdown view: how much of the field's average TTFA is + // leading silence — the share a listener waits through after bytes arrive. + const silenceShare = useMemo(() => { + const totals = ttfaBreakdownBars.reduce( + (acc, bar) => ({ silence: acc.silence + bar.silence, ttfa: acc.ttfa + bar.ttfa }), + { silence: 0, ttfa: 0 } + ); + return totals.ttfa > 0 ? (totals.silence / totals.ttfa) * 100 : undefined; + }, [ttfaBreakdownBars]); + + // Total-TTFA labels ride the top of each stack; thin bars stay unlabeled + // like the WER chart's, the tooltip still carrying the values. + const totalBarLabel = useCallback( + ({ x = 0, y = 0, width = 0, index = 0 }: LabelProps) => { + const entry = ttfaBreakdownBars[index]; + if (!entry || Number(width) < 34) return ; + return ( + + {`${Math.round(entry.ttfa)}`} + + ); + }, + [ttfaBreakdownBars, themeColors.label] + ); + return (
- boxPlotData.data.map(({ model, quartiles, stats }) => ({ - model: parseModelKey(model).model, - provider: getProviderForModel(model), - metric: activeMetric, - whisker_low_ms: quartiles.min, - q1_ms: quartiles.q1, - median_ms: quartiles.median, - q3_ms: quartiles.q3, - whisker_high_ms: quartiles.max, - iqr_ms: quartiles.q3 - quartiles.q1, - iqr_pct_of_median: - quartiles.median > 0 - ? ((quartiles.q3 - quartiles.q1) / quartiles.median) * 100 - : undefined, - mean_ms: stats.mean, - std_dev_ms: stats.std, - p95_ms: stats.p95, - max_ms: stats.max, - runs: stats.count, - })) + showBreakdown + ? ttfaBreakdownBars.map(({ model, roundtrip, silence, ttfa }) => ({ + model: parseModelKey(model).model, + provider: getProviderForModel(model), + avg_ttfa_ms: ttfa, + roundtrip_ms: roundtrip, + leading_silence_ms: silence, + })) + : boxPlotData.data.map(({ model, quartiles, stats }) => ({ + model: parseModelKey(model).model, + provider: getProviderForModel(model), + metric: activeMetric, + whisker_low_ms: quartiles.min, + q1_ms: quartiles.q1, + median_ms: quartiles.median, + q3_ms: quartiles.q3, + whisker_high_ms: quartiles.max, + iqr_ms: quartiles.q3 - quartiles.q1, + iqr_pct_of_median: + quartiles.median > 0 + ? ((quartiles.q3 - quartiles.q1) / quartiles.median) * 100 + : undefined, + mean_ms: stats.mean, + std_dev_ms: stats.std, + p95_ms: stats.p95, + max_ms: stats.max, + runs: stats.count, + })) } stat={ - avgIqrMs === undefined - ? undefined - : { - label: ( - {`Average ${latencyLabel} IQR`} - ), - value: `${avgIqrMs.toFixed(0)} ms`, - } + showBreakdown + ? silenceShare === undefined + ? undefined + : { + label: "Leading silence share of TTFA", + value: `${silenceShare.toFixed(0)}%`, + } + : avgIqrMs === undefined + ? undefined + : { + label: ( + {`Average ${latencyLabel} IQR`} + ), + value: `${avgIqrMs.toFixed(0)} ms`, + } } /> + {hasBreakdown && ( +
+
+ {( + [ + ["distribution", "Distribution"], + ["breakdown", "Breakdown"], + ] as const + ).map(([key, text]) => ( + + ))} +
+ {showBreakdown && ( +
+ + + Network roundtrip + + + + Leading silence + +
+ )} +
+ )} - + {showBreakdown ? ( + + {ttfaBreakdownBars.map(({ model }) => ( + + + + + ))} + + } + tickFormatter={(value) => `${Math.round(value)}`} + yAxisLabel="Avg TTFA ms · lower is better" + tooltip={ + + } + isMobile={isMobile} + getProviderForModel={getProviderForModel} + barLabel={totalBarLabel} + onHover={trackChartHover} + /> + ) : ( + + )}
); diff --git a/web/hooks/useChartData.ts b/web/hooks/useChartData.ts index b98333d6..4e28e158 100644 --- a/web/hooks/useChartData.ts +++ b/web/hooks/useChartData.ts @@ -14,16 +14,19 @@ import type { ModelHeatmapData, BarDataPoint, InstructionBarDataPoint, - LatencyPercentile + LatencyPercentile, + TtfaBreakdownBar } from "@/types/benchmark.types"; import type { SeriesPoint } from "@/lib/api/client"; import { latencyToMs, normalizeModelName, normalizeProviderNameForTab, toModelKey, parseModelKey } from "@/lib/utils/formatters"; import { WINDOW_MS, type TimeWindow } from "@/lib/config/timeWindows"; import { werBreakdownOf } from "@/lib/utils/werBreakdown"; +import { TTFA_LEADING_SILENCE, TTFA_ROUNDTRIP } from "@/lib/config/metrics"; // Latency metrics share every chart's number machinery (box plot, scatter, // comparison table). Membership gates the builders so a new one (V2V for S2S) -// is never silently skipped. +// is never silently skipped. The TTFA component metrics are deliberately +// absent: they feed only the breakdown bars and the comparison-table split. const LATENCY_METRICS: readonly string[] = ["TTFS", "TTFT", "TTFA", "V2V"]; interface UseChartDataParams { @@ -299,6 +302,30 @@ export function useChartData({ [scatterByMetricModel, selectedModels] ); + // Stacked-bar rows for the TTFA breakdown view, fastest total first. Rows + // exist only where both component metrics are served; the runner writes the + // pair together, so the parts sum to the split runs' average TTFA exactly. + const ttfaBreakdownBars = useMemo( + () => + selectedModels + .flatMap((model) => { + const roundtrip = getStat(model, TTFA_ROUNDTRIP); + const silence = getStat(model, TTFA_LEADING_SILENCE)?.avg_value; + if (!roundtrip || silence === undefined) return []; + return [ + { + model, + provider: roundtrip.provider, + roundtrip: roundtrip.avg_value, + silence, + ttfa: roundtrip.avg_value + silence + } + ]; + }) + .sort((a, b) => a.ttfa - b.ttfa), + [selectedModels, getStat] + ); + // Comparison rows for a given latency metric: the full latency percentile // ladder straight from the SQL stats, plus avg WER and sample counts. const getHeatmapData = useCallback( @@ -509,6 +536,7 @@ export function useChartData({ getScatterData, getHeatmapData, getWERBarData, + ttfaBreakdownBars, getInstructionBarData, getCurrentTimeWindow, getTimelineTicks, diff --git a/web/hooks/useDashboardState.tsx b/web/hooks/useDashboardState.tsx index cd07a9ab..66e571f6 100644 --- a/web/hooks/useDashboardState.tsx +++ b/web/hooks/useDashboardState.tsx @@ -198,6 +198,7 @@ export function useDashboardState(page: "tts" | "stt" | "s2s") { [aggregatesQuery.data] ); + const allModelsByProvider = useMemo( () => buildModelsByProvider(modelStats, benchmarkParam, providersQuery.data), [providersQuery.data, modelStats, benchmarkParam] @@ -438,7 +439,7 @@ export function useDashboardState(page: "tts" | "stt" | "s2s") { }); // Calculate metrics - const { getStat, getHeatmapData } = chartData; + const { getStat, getHeatmapData, ttfaBreakdownBars } = chartData; // Run-weighted average latency across selected models, in display units: // Σ(avg·runs) / Σ(runs). Backs the box plot and timeline headlines. @@ -635,7 +636,7 @@ export function useDashboardState(page: "tts" | "stt" | "s2s") { // Pre-computed key metrics for display const primaryKeyMetric = { - label: `Lowest Median ${activeMetric}`, + label: `Lowest Median ${latencyLabel}`, displayValue: `${fastestPrimary.fastestMs.toFixed(0)} ms`, subtitle: fastestPrimary.fastestModel ? { @@ -706,6 +707,7 @@ export function useDashboardState(page: "tts" | "stt" | "s2s") { sttMetric, setSttMetric, activeMetric, + ttfaBreakdownBars, // WER dataset pin (STT comparison card) werDataset: activeWerDataset, diff --git a/web/lib/config/metrics.ts b/web/lib/config/metrics.ts index 658c471d..0a3d14a5 100644 --- a/web/lib/config/metrics.ts +++ b/web/lib/config/metrics.ts @@ -1,10 +1,16 @@ // Copyright 2026 The Coval Benchmarks Authors // SPDX-License-Identifier: Apache-2.0 +// Canonical results.metric_type strings for the perceived-TTFA split. The +// runner writes the two component rows only when both are known, so they +// always sum back to the TTFA row they sit under. +export const TTFA_ROUNDTRIP = "TTFARoundtrip"; +export const TTFA_LEADING_SILENCE = "TTFALeadingSilence"; + // Definition of the active metric, shown as a bolded block inside the // "About this benchmark" tooltip on cards that carry the TTFS/TTFT toggle — // the toggle buttons themselves carry no tooltips. Metrics without a tooltip -// (TTFA, V2V) resolve to undefined and add nothing. +// (V2V) resolve to undefined and add nothing. export const metricAboutNote = ( metric: string ): { term: string; text: string } | undefined => { @@ -18,6 +24,8 @@ export const metricAboutNote = ( export const metricDescriptions = { ttfa: { short: "Time to First Audio", + tooltip: + "Time from sending the text until the listener would first hear speech: network roundtrip plus any leading silence the provider front-loads. Lower is better.", detailed: "Delivering natural and responsive voice agents requires both speed and consistency. At Coval, we understand that latency is critical for realistic conversations, which is why we go beyond average measurements to track comprehensive percentile metrics with continuous 15-minute evaluation cycles. This rigorous approach ensures your voice AI maintains the reliable performance necessary for engaging user experiences." }, diff --git a/web/types/benchmark.types.ts b/web/types/benchmark.types.ts index 0f2c77ff..33ccfd7b 100644 --- a/web/types/benchmark.types.ts +++ b/web/types/benchmark.types.ts @@ -48,6 +48,20 @@ export interface ModelHeatmapData { sampleCount: number; } +/** + * One stacked bar of the TTFA breakdown view. All three values come straight + * from served averages over the same rows, so roundtrip + silence = ttfa + * exactly. A type alias (not an interface) so rows satisfy the bar chassis's + * Record constraint via an implicit index signature. + */ +export type TtfaBreakdownBar = { + model: string; + provider: string; + roundtrip: number; + silence: number; + ttfa: number; +}; + export interface BarDataPoint { model: string; averageWER: number; From fb94d6ba62f1350daf9c959cd0aa498957c74ce6 Mon Sep 17 00:00:00 2001 From: Cooper Date: Wed, 5 Aug 2026 13:22:59 -0700 Subject: [PATCH 3/4] [BENCH-623] Keep the TTFA split out of the series rollup MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The component metrics are consumed as window aggregates only (the Latency Variation breakdown reads model_stats); nothing reads them per bucket. TTS runs ~48x a day, so letting them flow into results_by_bucket would double the 30d aggregates response — 66k series rows / 16.9 MB on prod today — for rows no chart uses. Exclude them in refresh_bucket via SERIES_EXCLUDED_METRICS, keeping every aggregates payload byte-identical to today; the raw result rows still feed the stats matviews. Remove a metric from the set if a per-run surface ever ships for it. --- runner/src/coval_bench/db/writer.py | 11 +++- runner/src/coval_bench/registries/__init__.py | 2 + runner/src/coval_bench/registries/metrics.py | 11 ++++ runner/tests/unit/test_db_writer.py | 58 +++++++++++++++++++ 4 files changed, 80 insertions(+), 2 deletions(-) diff --git a/runner/src/coval_bench/db/writer.py b/runner/src/coval_bench/db/writer.py index 46135561..63a40027 100644 --- a/runner/src/coval_bench/db/writer.py +++ b/runner/src/coval_bench/db/writer.py @@ -27,7 +27,7 @@ from psycopg_pool import AsyncConnectionPool from coval_bench.db.models import Result, Run, RunStatus -from coval_bench.registries import Metric +from coval_bench.registries import SERIES_EXCLUDED_METRICS, Metric STATS_MATVIEWS: tuple[str, ...] = ("results_24h", "results_7d", "results_30d") @@ -171,7 +171,13 @@ async def refresh_bucket(self, run_id: int, *, period_seconds: int) -> None: # interleave such that the staler recompute commits and the # fresher one aborts on the primary key, dropping a run from # the slot. Released on commit/abort. - params = {"bucket": bucket_at, "period": period_seconds} + params = { + "bucket": bucket_at, + "period": period_seconds, + # Window-aggregate-only metrics stay out of the series + # rollup; see SERIES_EXCLUDED_METRICS for the why. + "series_excluded": [str(m) for m in SERIES_EXCLUDED_METRICS], + } await cur.execute( "SELECT pg_advisory_xact_lock(hashtextextended('results_by_bucket'," " extract(epoch FROM %(bucket)s::timestamptz)::bigint))", @@ -211,6 +217,7 @@ async def refresh_bucket(self, run_id: int, *, period_seconds: int) -> None: WHERE r.status = 'success' AND rn.status IN ('succeeded', 'partial') AND r.metric_value IS NOT NULL + AND r.metric_type != ALL(%(series_excluded)s) AND ( rn.scheduled_at = %(bucket)s OR ( diff --git a/runner/src/coval_bench/registries/__init__.py b/runner/src/coval_bench/registries/__init__.py index 214a33cc..3a70fa0d 100644 --- a/runner/src/coval_bench/registries/__init__.py +++ b/runner/src/coval_bench/registries/__init__.py @@ -10,6 +10,7 @@ from coval_bench.registries.metrics import ( METRIC_EXCLUSIONS, METRIC_SPECS, + SERIES_EXCLUDED_METRICS, Metric, MetricDirection, MetricSpec, @@ -35,6 +36,7 @@ "Benchmark", "METRIC_EXCLUSIONS", "METRIC_SPECS", + "SERIES_EXCLUDED_METRICS", "MODEL_REGISTRY", "Metric", "MetricDirection", diff --git a/runner/src/coval_bench/registries/metrics.py b/runner/src/coval_bench/registries/metrics.py index 24510dac..b93c4207 100644 --- a/runner/src/coval_bench/registries/metrics.py +++ b/runner/src/coval_bench/registries/metrics.py @@ -134,6 +134,17 @@ class MetricSpec(BaseModel, frozen=True): raise RuntimeError(f"METRIC_SPECS is missing specs for: {_missing}") +# Metrics kept out of the per-bucket series rollup (results_by_bucket) and +# therefore out of every aggregates response's `series` array. The TTFA +# components are consumed as window aggregates only (the Latency Variation +# breakdown); TTS runs ~48x/day, so carrying them per bucket would double an +# already multi-MB 30d series payload for rows nothing reads. Remove a metric +# here if a per-run surface ever ships for it. +SERIES_EXCLUDED_METRICS: frozenset[Metric] = frozenset( + {Metric.TTFA_ROUNDTRIP, Metric.TTFA_LEADING_SILENCE} +) + + # (provider, model) pairs whose metric is not comparable with the cohort: # TTFT gated by buffering rather than engine speed, TTFS acked without # finalizing. The orchestrator skips writing these rows; the API hides diff --git a/runner/tests/unit/test_db_writer.py b/runner/tests/unit/test_db_writer.py index 7e0cad49..84c337b0 100644 --- a/runner/tests/unit/test_db_writer.py +++ b/runner/tests/unit/test_db_writer.py @@ -685,6 +685,64 @@ async def _run() -> None: assert float(row["p50"]) == pytest.approx(3.0) +def test_refresh_bucket_excludes_series_excluded_metrics( + pg_conn: psycopg.Connection[Any], +) -> None: + """TTFA component rows stay out of the series rollup: they are consumed as + window aggregates only, and at ~48 TTS runs/day they would double an + already multi-MB 30d series payload for rows nothing reads.""" + _apply_migrations(pg_conn) + scheduled = datetime.now(UTC).replace(microsecond=0) - timedelta(hours=1) + + def _tts_result(run_id: int, metric_type: str, value: float) -> Result: + return Result( + run_id=run_id, + provider="elevenlabs", + model="eleven_flash_v2_5", + benchmark=Benchmark.TTS, + metric_type=metric_type, + metric_value=value, + metric_units="milliseconds", + status=ResultStatus.SUCCESS, + ) + + async def _run() -> None: + pool = await _make_pool(pg_conn) + try: + writer = RunWriter(pool) + run = await writer.start_run( + runner_sha="abc123", + dataset_id="tts-v1", + dataset_sha256="deadbeef", + scheduled_at=scheduled, + ) + assert run.id is not None + await writer.record_results( + [ + _tts_result(run.id, "TTFA", 170.0), + _tts_result(run.id, "TTFARoundtrip", 140.0), + _tts_result(run.id, "TTFALeadingSilence", 30.0), + ] + ) + await writer.finish_run(run.id, status=RunStatus.SUCCEEDED) + await writer.refresh_bucket(run.id, period_seconds=1800) + finally: + await pool.close() + + asyncio.run(_run()) + + pg_conn.autocommit = True + with pg_conn.cursor() as cur: + cur.execute("SELECT DISTINCT metric_type FROM benchmarks_v2.results_by_bucket") + bucket_metrics = {row[0] for row in cur.fetchall()} + # The raw rows keep the split for the stats matviews. + cur.execute("SELECT DISTINCT metric_type FROM benchmarks_v2.results") + result_metrics = {row[0] for row in cur.fetchall()} + + assert bucket_metrics == {"TTFA"} + assert result_metrics == {"TTFA", "TTFARoundtrip", "TTFALeadingSilence"} + + def test_refresh_bucket_splits_datasets(pg_conn: psycopg.Connection[Any]) -> None: """Two runs on different datasets in one bucket: per-dataset rows split, the pooled '__all__' row spans both.""" From 95247eab500775be68ac9965608d34a833888732 Mon Sep 17 00:00:00 2001 From: Cooper Date: Wed, 5 Aug 2026 14:03:13 -0700 Subject: [PATCH 4/4] [BENCH-623] Address review: gate component rows on TTFA success, guard zero-total shares MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A provider error can arrive after audio has streamed, leaving ttfa_ms and the silence offset populated on a FAILED row — the component gate now requires the TTFA row's SUCCESS status, so failed samples never leak into the breakdown aggregates (regression assertion added to the provider-error test). The breakdown tooltip also skips the percent share when the total is zero instead of rendering NaN%. --- runner/src/coval_bench/runner/orchestrator.py | 14 +++++++++---- runner/tests/unit/test_orchestrator.py | 4 ++++ .../charts/tooltips/BarTooltip.test.tsx | 20 +++++++++++++++++++ web/components/charts/tooltips/BarTooltip.tsx | 4 +++- 4 files changed, 37 insertions(+), 5 deletions(-) diff --git a/runner/src/coval_bench/runner/orchestrator.py b/runner/src/coval_bench/runner/orchestrator.py index 69d459ac..9f8b529c 100644 --- a/runner/src/coval_bench/runner/orchestrator.py +++ b/runner/src/coval_bench/runner/orchestrator.py @@ -775,11 +775,17 @@ async def _run_tts_item( ) ) - # TTFA component rows, only when the split is known so the two - # always sum back to the TTFA row. A nulled TTFA (transport gate) - # or an arrival-only TTFA (offset detection failed) writes neither. + # TTFA component rows, only for a successful TTFA with a known + # split, so the two always sum back to the TTFA row. A failed row + # (provider error can arrive after audio, leaving ttfa_ms set), a + # nulled TTFA (transport gate) or an arrival-only TTFA (offset + # detection failed) writes neither. leading_silence_ms = tts_result.leading_silence_ms if tts_result else None - if ttfa_value is not None and leading_silence_ms is not None: + if ( + ttfa_status is ResultStatus.SUCCESS + and ttfa_value is not None + and leading_silence_ms is not None + ): for component, value in ( (Metric.TTFA_ROUNDTRIP, ttfa_value - leading_silence_ms), (Metric.TTFA_LEADING_SILENCE, leading_silence_ms), diff --git a/runner/tests/unit/test_orchestrator.py b/runner/tests/unit/test_orchestrator.py index 725093dc..7219801c 100644 --- a/runner/tests/unit/test_orchestrator.py +++ b/runner/tests/unit/test_orchestrator.py @@ -1977,6 +1977,7 @@ async def test_tts_provider_error_wins_over_contamination( audio_path=None, error="synth stream closed early", http_version="HTTP/1.1", # would otherwise trigger the contamination message + leading_silence_ms=45.0, # split present — a failed row must not emit components ) provider_inst = MagicMock() @@ -2012,6 +2013,9 @@ async def test_tts_provider_error_wins_over_contamination( assert "HTTP/1.1" not in (ttfa.error or "") # contamination message must not win assert ttfa.http_version == "HTTP/1.1" # diagnostic still recorded assert "WER" not in by_metric # errored synth → no WER scoring + # A failed TTFA writes no component rows even though its split was known. + assert "TTFARoundtrip" not in by_metric + assert "TTFALeadingSilence" not in by_metric assert summary.success_count == 0 diff --git a/web/components/charts/tooltips/BarTooltip.test.tsx b/web/components/charts/tooltips/BarTooltip.test.tsx index 2475f51f..da012bd9 100644 --- a/web/components/charts/tooltips/BarTooltip.test.tsx +++ b/web/components/charts/tooltips/BarTooltip.test.tsx @@ -60,4 +60,24 @@ describe("TtfaBreakdownTooltip", () => { expect(html).toContain("Network roundtrip: 160 ms (80%)"); expect(html).toContain("Leading silence: 40 ms (20%)"); }); + + it("omits shares when the total is zero instead of rendering NaN%", () => { + const bar: TtfaBreakdownBar = { + model: "elevenlabs:eleven_flash_v2_5", + provider: "elevenlabs", + roundtrip: 0, + silence: 0, + ttfa: 0, + }; + const html = renderToStaticMarkup( + + ); + expect(html).toContain("Network roundtrip: 0 ms"); + expect(html).not.toContain("NaN"); + expect(html).not.toContain("%"); + }); }); diff --git a/web/components/charts/tooltips/BarTooltip.tsx b/web/components/charts/tooltips/BarTooltip.tsx index 6a6d53b7..37adbab9 100644 --- a/web/components/charts/tooltips/BarTooltip.tsx +++ b/web/components/charts/tooltips/BarTooltip.tsx @@ -125,7 +125,9 @@ export const TtfaBreakdownTooltip: React.FC< color: "var(--color-text-on-tooltip)", opacity: 0.8, }} - >{`${text}: ${Math.round(value)} ms (${Math.round((value / row.ttfa) * 100)}%)`}

+ >{`${text}: ${Math.round(value)} ms${ + row.ttfa > 0 ? ` (${Math.round((value / row.ttfa) * 100)}%)` : "" + }`}

))} );