VALUE-04A: distinguish decision-grade device evidence

frontend/src/api/campaign-types.ts

@@ -27,6 +27,7 @@ export interface CampaignDimensionReadModel extends UIModelIdentity {
   dimension: CampaignComparisonDimension;
   comparable: boolean;
   evidence_available: boolean;
+  evidence_note?: string | null;
   reasons: CampaignCompatibilityReasonReadModel[];
 }
 

src/performance_lab/application/campaign_models.py

@@ -26,6 +26,7 @@ class CampaignDimensionReadModel(UIModel):
     dimension: ComparisonDimension
     comparable: bool
     evidence_available: bool
+    evidence_note: str | None = None
     reasons: tuple[CampaignCompatibilityReasonReadModel, ...] = ()
 
 

src/performance_lab/application/campaign_queries.py

@@ -35,6 +35,7 @@
     decision_policy_read_model,
     recommend_strict_quality_dominance,
 )
+from .device_evidence import resource_measurement_is_decision_eligible
 from .evidence_models import SampleEvidenceDetailReadModel
 from .ui_models import RunDetailReadModel
 from .ui_queries import CompletedRunReader
@@ -428,14 +429,15 @@ def _matching_samples(
 def _compatibility(
     candidates: tuple[tuple[str, Run], ...],
 ) -> tuple[CampaignDimensionReadModel, ...]:
+    runs = tuple(run for _, run in candidates)
     if len(candidates) < 2:
         return tuple(
             CampaignDimensionReadModel(
                 dimension=dimension,
                 comparable=False,
-                evidence_available=all(
-                    _has_dimension_evidence(run, dimension) for _, run in candidates
-                ),
+                evidence_available=bool(runs)
+                and all(_has_dimension_evidence(run, dimension) for run in runs),
+                evidence_note=_dimension_evidence_note(runs, dimension),
                 reasons=(),
             )
             for dimension in ComparisonDimension
@@ -465,15 +467,39 @@ def _compatibility(
             CampaignDimensionReadModel(
                 dimension=dimension,
                 comparable=not reasons,
-                evidence_available=all(
-                    _has_dimension_evidence(run, dimension) for _, run in candidates
-                ),
+                evidence_available=all(_has_dimension_evidence(run, dimension) for run in runs),
+                evidence_note=_dimension_evidence_note(runs, dimension),
                 reasons=tuple(reasons),
             )
         )
     return tuple(dimensions)
 
 
+def _dimension_evidence_note(
+    runs: tuple[Run, ...],
+    dimension: ComparisonDimension,
+) -> str | None:
+    if runs and all(_has_dimension_evidence(run, dimension) for run in runs):
+        return None
+    if not runs:
+        return "No completed candidate Run evidence is available for this dimension."
+    if dimension == ComparisonDimension.CAPABILITY:
+        return "Comparable aggregate quality evidence is unavailable for one or more candidates."
+    if dimension == ComparisonDimension.RUNTIME:
+        return "Black-box request performance evidence is unavailable for one or more candidates."
+    contextual_resource_telemetry = any(
+        item.provenance in {MeasurementProvenance.HOST, MeasurementProvenance.RUNTIME}
+        for run in runs
+        for item in run.aggregate_measurements
+    )
+    if contextual_resource_telemetry:
+        return (
+            "Contextual host/runtime telemetry is retained, but no explicitly attributable "
+            "model-resource metric is decision-eligible for every candidate."
+        )
+    return "No explicitly attributable model-resource evidence is retained for every candidate."
+
+
 def _has_dimension_evidence(run: Run, dimension: ComparisonDimension) -> bool:
     if dimension == ComparisonDimension.CAPABILITY:
         return bool(run.aggregate_scores)
@@ -482,6 +508,5 @@ def _has_dimension_evidence(run: Run, dimension: ComparisonDimension) -> bool:
             item.provenance == MeasurementProvenance.CLIENT for item in run.aggregate_measurements
         )
     return any(
-        item.provenance in {MeasurementProvenance.HOST, MeasurementProvenance.RUNTIME}
-        for item in run.aggregate_measurements
+        resource_measurement_is_decision_eligible(item) for item in run.aggregate_measurements
     )

src/performance_lab/application/device_evidence.py

@@ -0,0 +1,85 @@
+"""Policy-eligibility classification for retained device/performance measurements.
+
+Performance Lab retains more telemetry than it is allowed to use for a best-fit decision.
+This module is the canonical application boundary that prevents contextual host/runtime
+observations from silently becoming model-resource evidence.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+from enum import StrEnum
+
+from performance_lab.domain import Measurement, MeasurementProvenance
+
+
+class DecisionEvidenceRole(StrEnum):
+    """Whether a retained measurement may participate in a decision after comparability."""
+
+    DECISION_ELIGIBLE = "decision_eligible"
+    CONTEXT_ONLY = "context_only"
+
+
+@dataclass(frozen=True, slots=True)
+class DecisionEvidenceClassification:
+    role: DecisionEvidenceRole
+    reason: str
+
+
+# Resource measurements require explicit opt-in by the owning runtime/collector contract.
+# The current host-stdlib collector measures the Performance Lab process/host, while
+# local-llm-server-status-v1 reports runtime activity/concurrency rather than attributable
+# model resource consumption. Therefore no current HOST/RUNTIME measurement is registered.
+_RESOURCE_DECISION_KEYS: frozenset[tuple[str, str, str, str]] = frozenset()
+
+
+def classify_measurement(measurement: Measurement) -> DecisionEvidenceClassification:
+    """Classify retained measurement truth without inferring missing ownership semantics."""
+
+    if measurement.provenance == MeasurementProvenance.CLIENT:
+        return DecisionEvidenceClassification(
+            role=DecisionEvidenceRole.DECISION_ELIGIBLE,
+            reason=(
+                "Black-box request performance is observed by Performance Lab for the frozen "
+                "execution and may participate after runtime comparability is established."
+            ),
+        )
+
+    key = (
+        measurement.provenance.value,
+        measurement.protocol_version,
+        measurement.name,
+        measurement.unit,
+    )
+    if key in _RESOURCE_DECISION_KEYS:
+        return DecisionEvidenceClassification(
+            role=DecisionEvidenceRole.DECISION_ELIGIBLE,
+            reason="The owning resource telemetry contract explicitly marks this metric eligible.",
+        )
+
+    if measurement.provenance == MeasurementProvenance.HOST:
+        return DecisionEvidenceClassification(
+            role=DecisionEvidenceRole.CONTEXT_ONLY,
+            reason=(
+                "Host telemetry is retained as execution context; the current collector does not "
+                "establish attributable model-server resource consumption."
+            ),
+        )
+    return DecisionEvidenceClassification(
+        role=DecisionEvidenceRole.CONTEXT_ONLY,
+        reason=(
+            "Runtime telemetry is retained as context unless its owning versioned contract "
+            "explicitly declares an attributable resource metric policy-eligible."
+        ),
+    )
+
+
+def resource_measurement_is_decision_eligible(measurement: Measurement) -> bool:
+    """Return true only for explicitly contracted HOST/RUNTIME model-resource evidence."""
+
+    if measurement.provenance not in {
+        MeasurementProvenance.HOST,
+        MeasurementProvenance.RUNTIME,
+    }:
+        return False
+    return classify_measurement(measurement).role == DecisionEvidenceRole.DECISION_ELIGIBLE

tests/test_campaign_queries.py

@@ -13,6 +13,9 @@
     ExecutionFingerprint,
     HardwareIdentity,
     LoadProfile,
+    Measurement,
+    MeasurementProvenance,
+    MeasurementScope,
     ModelIdentity,
     Run,
     RunStatus,
@@ -22,7 +25,12 @@
 from performance_lab.storage import SQLiteCampaignStore, SQLiteRunStore
 
 
-def _run(model_id: str, value: float) -> Run:
+def _run(
+    model_id: str,
+    value: float,
+    *,
+    measurements: tuple[Measurement, ...] = (),
+) -> Run:
     bundle = build_general_starter_suite()
     evaluator = EvaluatorRef(evaluator_id="quality", version="1")
     now = datetime.now(UTC)
@@ -54,21 +62,13 @@ def _run(model_id: str, value: float) -> Run:
                 higher_is_better=True,
             ),
         ),
+        aggregate_measurements=measurements,
     )
 
 
-def test_campaign_results_show_policy_before_a_strict_dominance_recommendation(
-    tmp_path: Path,
-) -> None:
-    run_store = SQLiteRunStore(tmp_path / "runs.sqlite3")
-    first = _run("model-a", 1.0)
-    second = _run("model-b", 0.5)
-    run_store.publish(first)
-    run_store.publish(second)
-
+def _save_campaign(store: SQLiteCampaignStore, first: Run, second: Run) -> None:
     now = datetime.now(UTC)
-    campaign_store = SQLiteCampaignStore(run_store.path)
-    campaign_store.save(
+    store.save(
         Campaign(
             campaign_id="campaign-a",
             plan_digest="a" * 64,
@@ -105,6 +105,19 @@ def test_campaign_results_show_policy_before_a_strict_dominance_recommendation(
             ),
         )
     )
+
+
+def test_campaign_results_show_policy_before_a_strict_dominance_recommendation(
+    tmp_path: Path,
+) -> None:
+    run_store = SQLiteRunStore(tmp_path / "runs.sqlite3")
+    first = _run("model-a", 1.0)
+    second = _run("model-b", 0.5)
+    run_store.publish(first)
+    run_store.publish(second)
+
+    campaign_store = SQLiteCampaignStore(run_store.path)
+    _save_campaign(campaign_store, first, second)
     run_queries = UIQueryService(run_store)
 
     result = CampaignQueryService(campaign_store, run_queries).get("campaign-a")
@@ -116,5 +129,45 @@ def test_campaign_results_show_policy_before_a_strict_dominance_recommendation(
     )
     assert capability.comparable
     assert capability.evidence_available
+    assert capability.evidence_note is None
     assert result.results.recommendation is not None
     assert result.results.recommendation.model_id == "model-a"
+
+
+def test_campaign_does_not_promote_context_telemetry_to_resource_evidence(
+    tmp_path: Path,
+) -> None:
+    host_context = Measurement(
+        name="process_peak_rss_bytes",
+        value=1024.0,
+        unit="bytes",
+        scope=MeasurementScope.RUN,
+        provenance=MeasurementProvenance.HOST,
+        protocol_version="host-stdlib-v1",
+    )
+    runtime_context = Measurement(
+        name="peak_active_requests",
+        value=1.0,
+        unit="count",
+        scope=MeasurementScope.RUN,
+        provenance=MeasurementProvenance.RUNTIME,
+        protocol_version="local-llm-server-status-v1",
+    )
+    run_store = SQLiteRunStore(tmp_path / "runs.sqlite3")
+    first = _run("model-a", 1.0, measurements=(host_context, runtime_context))
+    second = _run("model-b", 0.5, measurements=(host_context, runtime_context))
+    run_store.publish(first)
+    run_store.publish(second)
+
+    campaign_store = SQLiteCampaignStore(run_store.path)
+    _save_campaign(campaign_store, first, second)
+
+    result = CampaignQueryService(campaign_store, UIQueryService(run_store)).get("campaign-a")
+    resource = next(
+        item for item in result.results.compatibility if item.dimension.value == "resource"
+    )
+
+    assert resource.comparable
+    assert not resource.evidence_available
+    assert resource.evidence_note is not None
+    assert "Contextual host/runtime telemetry is retained" in resource.evidence_note

tests/test_device_evidence.py

@@ -0,0 +1,71 @@
+from performance_lab.application.device_evidence import (
+    DecisionEvidenceRole,
+    classify_measurement,
+    resource_measurement_is_decision_eligible,
+)
+from performance_lab.domain import (
+    Measurement,
+    MeasurementProvenance,
+    MeasurementScope,
+)
+
+
+def _measurement(
+    *,
+    name: str,
+    unit: str,
+    provenance: MeasurementProvenance,
+    protocol_version: str,
+) -> Measurement:
+    return Measurement(
+        name=name,
+        value=1.0,
+        unit=unit,
+        scope=MeasurementScope.RUN,
+        provenance=provenance,
+        protocol_version=protocol_version,
+    )
+
+
+def test_client_performance_is_attributable_decision_evidence() -> None:
+    measurement = _measurement(
+        name="request_latency_ms",
+        unit="ms",
+        provenance=MeasurementProvenance.CLIENT,
+        protocol_version="openai-compatible-v1",
+    )
+
+    classification = classify_measurement(measurement)
+
+    assert classification.role == DecisionEvidenceRole.DECISION_ELIGIBLE
+    assert not resource_measurement_is_decision_eligible(measurement)
+
+
+def test_host_process_telemetry_stays_context_only() -> None:
+    measurement = _measurement(
+        name="process_peak_rss_bytes",
+        unit="bytes",
+        provenance=MeasurementProvenance.HOST,
+        protocol_version="host-stdlib-v1",
+    )
+
+    classification = classify_measurement(measurement)
+
+    assert classification.role == DecisionEvidenceRole.CONTEXT_ONLY
+    assert "attributable model-server" in classification.reason
+    assert not resource_measurement_is_decision_eligible(measurement)
+
+
+def test_local_llm_server_activity_telemetry_stays_context_only() -> None:
+    measurement = _measurement(
+        name="peak_active_requests",
+        unit="count",
+        provenance=MeasurementProvenance.RUNTIME,
+        protocol_version="local-llm-server-status-v1",
+    )
+
+    classification = classify_measurement(measurement)
+
+    assert classification.role == DecisionEvidenceRole.CONTEXT_ONLY
+    assert "owning versioned contract" in classification.reason
+    assert not resource_measurement_is_decision_eligible(measurement)