From 3d9adee236b707178a86c545ab5d29d1a041ede4 Mon Sep 17 00:00:00 2001 From: Daniele21 Date: Thu, 3 Sep 2026 14:10:35 +0200 Subject: [PATCH 1/7] VALUE-04A: classify decision-eligible device evidence --- .../application/device_evidence.py | 85 +++++++++++++++++++ 1 file changed, 85 insertions(+) create mode 100644 src/performance_lab/application/device_evidence.py diff --git a/src/performance_lab/application/device_evidence.py b/src/performance_lab/application/device_evidence.py new file mode 100644 index 00000000..210de9ca --- /dev/null +++ b/src/performance_lab/application/device_evidence.py @@ -0,0 +1,85 @@ +"""Policy-eligibility classification for retained device/performance measurements. + +Performance Lab retains more telemetry than it is allowed to use for a best-fit decision. +This module is the canonical application boundary that prevents contextual host/runtime +observations from silently becoming model-resource evidence. +""" + +from __future__ import annotations + +from dataclasses import dataclass +from enum import StrEnum + +from performance_lab.domain import Measurement, MeasurementProvenance + + +class DecisionEvidenceRole(StrEnum): + """Whether a retained measurement may participate in a decision after comparability.""" + + DECISION_ELIGIBLE = "decision_eligible" + CONTEXT_ONLY = "context_only" + + +@dataclass(frozen=True, slots=True) +class DecisionEvidenceClassification: + role: DecisionEvidenceRole + reason: str + + +# Resource measurements require explicit opt-in by the owning runtime/collector contract. +# The current host-stdlib collector measures the Performance Lab process/host, while +# local-llm-server-status-v1 reports runtime activity/concurrency rather than attributable +# model resource consumption. Therefore no current HOST/RUNTIME measurement is registered. +_RESOURCE_DECISION_KEYS: frozenset[tuple[str, str, str, str]] = frozenset() + + +def classify_measurement(measurement: Measurement) -> DecisionEvidenceClassification: + """Classify retained measurement truth without inferring missing ownership semantics.""" + + if measurement.provenance == MeasurementProvenance.CLIENT: + return DecisionEvidenceClassification( + role=DecisionEvidenceRole.DECISION_ELIGIBLE, + reason=( + "Black-box request performance is observed by Performance Lab for the frozen " + "execution and may participate after runtime comparability is established." + ), + ) + + key = ( + measurement.provenance.value, + measurement.protocol_version, + measurement.name, + measurement.unit, + ) + if key in _RESOURCE_DECISION_KEYS: + return DecisionEvidenceClassification( + role=DecisionEvidenceRole.DECISION_ELIGIBLE, + reason="The owning resource telemetry contract explicitly marks this metric eligible.", + ) + + if measurement.provenance == MeasurementProvenance.HOST: + return DecisionEvidenceClassification( + role=DecisionEvidenceRole.CONTEXT_ONLY, + reason=( + "Host telemetry is retained as execution context; the current collector does not " + "establish attributable model-server resource consumption." + ), + ) + return DecisionEvidenceClassification( + role=DecisionEvidenceRole.CONTEXT_ONLY, + reason=( + "Runtime telemetry is retained as context unless its owning versioned contract " + "explicitly declares an attributable resource metric policy-eligible." + ), + ) + + +def resource_measurement_is_decision_eligible(measurement: Measurement) -> bool: + """Return true only for explicitly contracted HOST/RUNTIME model-resource evidence.""" + + if measurement.provenance not in { + MeasurementProvenance.HOST, + MeasurementProvenance.RUNTIME, + }: + return False + return classify_measurement(measurement).role == DecisionEvidenceRole.DECISION_ELIGIBLE From 00d07568e78d8efd309f13bd3f4ef1429863eb57 Mon Sep 17 00:00:00 2001 From: Daniele21 Date: Thu, 3 Sep 2026 14:11:07 +0200 Subject: [PATCH 2/7] VALUE-04A: expose dimension evidence notes --- src/performance_lab/application/campaign_models.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/performance_lab/application/campaign_models.py b/src/performance_lab/application/campaign_models.py index 79945713..99fff785 100644 --- a/src/performance_lab/application/campaign_models.py +++ b/src/performance_lab/application/campaign_models.py @@ -26,6 +26,7 @@ class CampaignDimensionReadModel(UIModel): dimension: ComparisonDimension comparable: bool evidence_available: bool + evidence_note: str | None = None reasons: tuple[CampaignCompatibilityReasonReadModel, ...] = () From 89ab2a836217a62c648c86906a8ecbfdcee4f337 Mon Sep 17 00:00:00 2001 From: Daniele21 Date: Thu, 3 Sep 2026 14:12:22 +0200 Subject: [PATCH 3/7] VALUE-04A: require attributable resource evidence --- .../application/campaign_queries.py | 39 ++++++++++++++++--- 1 file changed, 33 insertions(+), 6 deletions(-) diff --git a/src/performance_lab/application/campaign_queries.py b/src/performance_lab/application/campaign_queries.py index 9bd8b162..a8d82943 100644 --- a/src/performance_lab/application/campaign_queries.py +++ b/src/performance_lab/application/campaign_queries.py @@ -35,6 +35,7 @@ decision_policy_read_model, recommend_strict_quality_dominance, ) +from .device_evidence import resource_measurement_is_decision_eligible from .evidence_models import SampleEvidenceDetailReadModel from .ui_models import RunDetailReadModel from .ui_queries import CompletedRunReader @@ -428,14 +429,15 @@ def _matching_samples( def _compatibility( candidates: tuple[tuple[str, Run], ...], ) -> tuple[CampaignDimensionReadModel, ...]: + runs = tuple(run for _, run in candidates) if len(candidates) < 2: return tuple( CampaignDimensionReadModel( dimension=dimension, comparable=False, - evidence_available=all( - _has_dimension_evidence(run, dimension) for _, run in candidates - ), + evidence_available=bool(runs) + and all(_has_dimension_evidence(run, dimension) for run in runs), + evidence_note=_dimension_evidence_note(runs, dimension), reasons=(), ) for dimension in ComparisonDimension @@ -466,14 +468,40 @@ def _compatibility( dimension=dimension, comparable=not reasons, evidence_available=all( - _has_dimension_evidence(run, dimension) for _, run in candidates + _has_dimension_evidence(run, dimension) for run in runs ), + evidence_note=_dimension_evidence_note(runs, dimension), reasons=tuple(reasons), ) ) return tuple(dimensions) +def _dimension_evidence_note( + runs: tuple[Run, ...], + dimension: ComparisonDimension, +) -> str | None: + if runs and all(_has_dimension_evidence(run, dimension) for run in runs): + return None + if not runs: + return "No completed candidate Run evidence is available for this dimension." + if dimension == ComparisonDimension.CAPABILITY: + return "Comparable aggregate quality evidence is unavailable for one or more candidates." + if dimension == ComparisonDimension.RUNTIME: + return "Black-box request performance evidence is unavailable for one or more candidates." + contextual_resource_telemetry = any( + item.provenance in {MeasurementProvenance.HOST, MeasurementProvenance.RUNTIME} + for run in runs + for item in run.aggregate_measurements + ) + if contextual_resource_telemetry: + return ( + "Contextual host/runtime telemetry is retained, but no explicitly attributable " + "model-resource metric is decision-eligible for every candidate." + ) + return "No explicitly attributable model-resource evidence is retained for every candidate." + + def _has_dimension_evidence(run: Run, dimension: ComparisonDimension) -> bool: if dimension == ComparisonDimension.CAPABILITY: return bool(run.aggregate_scores) @@ -482,6 +510,5 @@ def _has_dimension_evidence(run: Run, dimension: ComparisonDimension) -> bool: item.provenance == MeasurementProvenance.CLIENT for item in run.aggregate_measurements ) return any( - item.provenance in {MeasurementProvenance.HOST, MeasurementProvenance.RUNTIME} - for item in run.aggregate_measurements + resource_measurement_is_decision_eligible(item) for item in run.aggregate_measurements ) From 1a241f3e32a66b18a5244d08cd8fbdae47e1f1c1 Mon Sep 17 00:00:00 2001 From: Daniele21 Date: Thu, 3 Sep 2026 14:12:52 +0200 Subject: [PATCH 4/7] VALUE-04A: mirror evidence note in campaign API type --- frontend/src/api/campaign-types.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/frontend/src/api/campaign-types.ts b/frontend/src/api/campaign-types.ts index c771e956..7599caac 100644 --- a/frontend/src/api/campaign-types.ts +++ b/frontend/src/api/campaign-types.ts @@ -27,6 +27,7 @@ export interface CampaignDimensionReadModel extends UIModelIdentity { dimension: CampaignComparisonDimension; comparable: boolean; evidence_available: boolean; + evidence_note?: string | null; reasons: CampaignCompatibilityReasonReadModel[]; } From 7f5e5d31827ae220b5ac8e247ba73df0a2d89b32 Mon Sep 17 00:00:00 2001 From: Daniele21 Date: Thu, 3 Sep 2026 14:13:12 +0200 Subject: [PATCH 5/7] VALUE-04A: test device evidence classification --- tests/test_device_evidence.py | 71 +++++++++++++++++++++++++++++++++++ 1 file changed, 71 insertions(+) create mode 100644 tests/test_device_evidence.py diff --git a/tests/test_device_evidence.py b/tests/test_device_evidence.py new file mode 100644 index 00000000..6c3fead0 --- /dev/null +++ b/tests/test_device_evidence.py @@ -0,0 +1,71 @@ +from performance_lab.application.device_evidence import ( + DecisionEvidenceRole, + classify_measurement, + resource_measurement_is_decision_eligible, +) +from performance_lab.domain import ( + Measurement, + MeasurementProvenance, + MeasurementScope, +) + + +def _measurement( + *, + name: str, + unit: str, + provenance: MeasurementProvenance, + protocol_version: str, +) -> Measurement: + return Measurement( + name=name, + value=1.0, + unit=unit, + scope=MeasurementScope.RUN, + provenance=provenance, + protocol_version=protocol_version, + ) + + +def test_client_performance_is_attributable_decision_evidence() -> None: + measurement = _measurement( + name="request_latency_ms", + unit="ms", + provenance=MeasurementProvenance.CLIENT, + protocol_version="openai-compatible-v1", + ) + + classification = classify_measurement(measurement) + + assert classification.role == DecisionEvidenceRole.DECISION_ELIGIBLE + assert not resource_measurement_is_decision_eligible(measurement) + + +def test_host_process_telemetry_stays_context_only() -> None: + measurement = _measurement( + name="process_peak_rss_bytes", + unit="bytes", + provenance=MeasurementProvenance.HOST, + protocol_version="host-stdlib-v1", + ) + + classification = classify_measurement(measurement) + + assert classification.role == DecisionEvidenceRole.CONTEXT_ONLY + assert "attributable model-server" in classification.reason + assert not resource_measurement_is_decision_eligible(measurement) + + +def test_local_llm_server_activity_telemetry_stays_context_only() -> None: + measurement = _measurement( + name="peak_active_requests", + unit="count", + provenance=MeasurementProvenance.RUNTIME, + protocol_version="local-llm-server-status-v1", + ) + + classification = classify_measurement(measurement) + + assert classification.role == DecisionEvidenceRole.CONTEXT_ONLY + assert "owning versioned contract" in classification.reason + assert not resource_measurement_is_decision_eligible(measurement) From c6bc29c2f10b4666aaed6f95e978a92390f4afaf Mon Sep 17 00:00:00 2001 From: Daniele21 Date: Thu, 3 Sep 2026 14:13:50 +0200 Subject: [PATCH 6/7] VALUE-04A: test contextual resource evidence stays unavailable --- tests/test_campaign_queries.py | 77 ++++++++++++++++++++++++++++------ 1 file changed, 65 insertions(+), 12 deletions(-) diff --git a/tests/test_campaign_queries.py b/tests/test_campaign_queries.py index 4d6b0ef8..16a6db15 100644 --- a/tests/test_campaign_queries.py +++ b/tests/test_campaign_queries.py @@ -13,6 +13,9 @@ ExecutionFingerprint, HardwareIdentity, LoadProfile, + Measurement, + MeasurementProvenance, + MeasurementScope, ModelIdentity, Run, RunStatus, @@ -22,7 +25,12 @@ from performance_lab.storage import SQLiteCampaignStore, SQLiteRunStore -def _run(model_id: str, value: float) -> Run: +def _run( + model_id: str, + value: float, + *, + measurements: tuple[Measurement, ...] = (), +) -> Run: bundle = build_general_starter_suite() evaluator = EvaluatorRef(evaluator_id="quality", version="1") now = datetime.now(UTC) @@ -54,21 +62,13 @@ def _run(model_id: str, value: float) -> Run: higher_is_better=True, ), ), + aggregate_measurements=measurements, ) -def test_campaign_results_show_policy_before_a_strict_dominance_recommendation( - tmp_path: Path, -) -> None: - run_store = SQLiteRunStore(tmp_path / "runs.sqlite3") - first = _run("model-a", 1.0) - second = _run("model-b", 0.5) - run_store.publish(first) - run_store.publish(second) - +def _save_campaign(store: SQLiteCampaignStore, first: Run, second: Run) -> None: now = datetime.now(UTC) - campaign_store = SQLiteCampaignStore(run_store.path) - campaign_store.save( + store.save( Campaign( campaign_id="campaign-a", plan_digest="a" * 64, @@ -105,6 +105,19 @@ def test_campaign_results_show_policy_before_a_strict_dominance_recommendation( ), ) ) + + +def test_campaign_results_show_policy_before_a_strict_dominance_recommendation( + tmp_path: Path, +) -> None: + run_store = SQLiteRunStore(tmp_path / "runs.sqlite3") + first = _run("model-a", 1.0) + second = _run("model-b", 0.5) + run_store.publish(first) + run_store.publish(second) + + campaign_store = SQLiteCampaignStore(run_store.path) + _save_campaign(campaign_store, first, second) run_queries = UIQueryService(run_store) result = CampaignQueryService(campaign_store, run_queries).get("campaign-a") @@ -116,5 +129,45 @@ def test_campaign_results_show_policy_before_a_strict_dominance_recommendation( ) assert capability.comparable assert capability.evidence_available + assert capability.evidence_note is None assert result.results.recommendation is not None assert result.results.recommendation.model_id == "model-a" + + +def test_campaign_does_not_promote_context_telemetry_to_resource_evidence( + tmp_path: Path, +) -> None: + host_context = Measurement( + name="process_peak_rss_bytes", + value=1024.0, + unit="bytes", + scope=MeasurementScope.RUN, + provenance=MeasurementProvenance.HOST, + protocol_version="host-stdlib-v1", + ) + runtime_context = Measurement( + name="peak_active_requests", + value=1.0, + unit="count", + scope=MeasurementScope.RUN, + provenance=MeasurementProvenance.RUNTIME, + protocol_version="local-llm-server-status-v1", + ) + run_store = SQLiteRunStore(tmp_path / "runs.sqlite3") + first = _run("model-a", 1.0, measurements=(host_context, runtime_context)) + second = _run("model-b", 0.5, measurements=(host_context, runtime_context)) + run_store.publish(first) + run_store.publish(second) + + campaign_store = SQLiteCampaignStore(run_store.path) + _save_campaign(campaign_store, first, second) + + result = CampaignQueryService(campaign_store, UIQueryService(run_store)).get("campaign-a") + resource = next( + item for item in result.results.compatibility if item.dimension.value == "resource" + ) + + assert resource.comparable + assert not resource.evidence_available + assert resource.evidence_note is not None + assert "Contextual host/runtime telemetry is retained" in resource.evidence_note From ba7502430c4b3151e941259a71946bb3d953e630 Mon Sep 17 00:00:00 2001 From: Daniele21 Date: Thu, 3 Sep 2026 14:20:48 +0200 Subject: [PATCH 7/7] VALUE-04A: format campaign evidence projection --- src/performance_lab/application/campaign_queries.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/src/performance_lab/application/campaign_queries.py b/src/performance_lab/application/campaign_queries.py index a8d82943..a53bca99 100644 --- a/src/performance_lab/application/campaign_queries.py +++ b/src/performance_lab/application/campaign_queries.py @@ -467,9 +467,7 @@ def _compatibility( CampaignDimensionReadModel( dimension=dimension, comparable=not reasons, - evidence_available=all( - _has_dimension_evidence(run, dimension) for run in runs - ), + evidence_available=all(_has_dimension_evidence(run, dimension) for run in runs), evidence_note=_dimension_evidence_note(runs, dimension), reasons=tuple(reasons), )