From 1cfd5ee495ca1c8dbb4f6502ffcc4fb1ffe2f440 Mon Sep 17 00:00:00 2001 From: Daniele21 Date: Thu, 3 Sep 2026 15:20:10 +0200 Subject: [PATCH 01/35] feat: carry declared domains into candidate planning --- src/performance_lab/application/planning_models.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/performance_lab/application/planning_models.py b/src/performance_lab/application/planning_models.py index 476e893d..6afaeac8 100644 --- a/src/performance_lab/application/planning_models.py +++ b/src/performance_lab/application/planning_models.py @@ -11,6 +11,7 @@ from .ui_models import ( DatasetSummaryReadModel, + GenerationParameterDomainReadModel, SuiteSummaryReadModel, TargetSummaryReadModel, UIModel, @@ -46,6 +47,7 @@ class CandidateModelReadModel(UIModel): runtime_name: str | None = None runtime_version: str | None = None runtime_config_digest: str | None = None + generation_parameter_domains: tuple[GenerationParameterDomainReadModel, ...] = () source: Literal["configured", "discovered"] From eb568aca21649b76eac2a1dd243498d8848169ea Mon Sep 17 00:00:00 2001 From: Daniele21 Date: Thu, 3 Sep 2026 15:20:58 +0200 Subject: [PATCH 02/35] feat: add declared generation domain read model --- src/performance_lab/application/ui_models.py | 46 ++++++++++++++++++++ 1 file changed, 46 insertions(+) diff --git a/src/performance_lab/application/ui_models.py b/src/performance_lab/application/ui_models.py index a4a7db55..8e3e7175 100644 --- a/src/performance_lab/application/ui_models.py +++ b/src/performance_lab/application/ui_models.py @@ -159,6 +159,51 @@ class CapabilitySupportReadModel(UIModel): detail: str | None = None +class GenerationParameterDomainReadModel(UIModel): + """Validated projection of a backend-declared request-generation domain.""" + + name: str = Field(min_length=1) + kind: Literal["float", "integer", "boolean"] + scope: Literal["request_generation"] = "request_generation" + source: Literal["local_llm_server"] = "local_llm_server" + provenance: Literal["registry_declared"] = "registry_declared" + minimum: int | float | None = None + maximum: int | float | None = None + step: int | float | None = None + values: tuple[bool, ...] = () + + @model_validator(mode="after") + def validate_domain_shape(self) -> GenerationParameterDomainReadModel: + if self.kind == "boolean": + if self.minimum is not None or self.maximum is not None or self.step is not None: + raise ValueError("boolean generation domains cannot declare numeric bounds") + if len(self.values) != 2 or set(self.values) != {False, True}: + raise ValueError("boolean generation domains must contain false and true") + return self + + if self.values: + raise ValueError("numeric generation domains cannot declare boolean values") + if self.minimum is None or self.maximum is None: + raise ValueError("numeric generation domains require minimum and maximum") + if isinstance(self.minimum, bool) or isinstance(self.maximum, bool): + raise ValueError("numeric generation domains require numeric bounds") + if self.minimum >= self.maximum: + raise ValueError("numeric generation domains require minimum < maximum") + if self.kind == "integer" and ( + not isinstance(self.minimum, int) + or not isinstance(self.maximum, int) + or isinstance(self.minimum, bool) + or isinstance(self.maximum, bool) + ): + raise ValueError("integer generation domains require integer bounds") + if self.step is not None: + if isinstance(self.step, bool) or self.step <= 0 or self.step > self.maximum - self.minimum: + raise ValueError("generation domain step must be positive and within the span") + if self.kind == "integer" and not isinstance(self.step, int): + raise ValueError("integer generation domains require an integer step") + return self + + class RuntimeParameterReadModel(UIModel): name: str = Field(min_length=1) scope: Literal["runtime_load"] = "runtime_load" @@ -170,6 +215,7 @@ class RuntimeParameterReadModel(UIModel): class DiscoveredModelReadModel(UIModel): model_id: str = Field(min_length=1) runtime_parameters: tuple[RuntimeParameterReadModel, ...] = () + generation_parameter_domains: tuple[GenerationParameterDomainReadModel, ...] = () class EndpointProbeReadModel(UIModel): From 7e2c4a864951240ce00322d6dc8b5b9a8b033c8c Mon Sep 17 00:00:00 2001 From: Daniele21 Date: Thu, 3 Sep 2026 15:21:35 +0200 Subject: [PATCH 03/35] feat: consume local llm generation domains --- .../application/endpoint_discovery.py | 54 +++++++++++++++---- 1 file changed, 45 insertions(+), 9 deletions(-) diff --git a/src/performance_lab/application/endpoint_discovery.py b/src/performance_lab/application/endpoint_discovery.py index a8c986f1..eb25ac1e 100644 --- a/src/performance_lab/application/endpoint_discovery.py +++ b/src/performance_lab/application/endpoint_discovery.py @@ -6,7 +6,7 @@ from typing import Literal import httpx -from pydantic import HttpUrl +from pydantic import HttpUrl, ValidationError from performance_lab.adapters import OpenAICompatibleAdapter from performance_lab.domain import EndpointProfile @@ -17,6 +17,7 @@ DiscoveredModelReadModel, EndpointConnectionInput, EndpointProbeReadModel, + GenerationParameterDomainReadModel, RuntimeParameterReadModel, ) @@ -66,18 +67,22 @@ async def probe_endpoint_profile( capabilities = _capability_evidence(passive.capabilities, healthy=passive.healthy) runtime_parameters: dict[str, tuple[RuntimeParameterReadModel, ...]] = {} + generation_domains: dict[str, tuple[GenerationParameterDomainReadModel, ...]] = {} warning: str | None = None if ( passive.healthy and local_connection is not None and local_connection.server_type == "local_llm_server" ): - runtime_parameters, warning = await _probe_local_llm_server_registry(local_connection) + runtime_parameters, generation_domains, warning = await _probe_local_llm_server_registry( + local_connection + ) models = tuple( DiscoveredModelReadModel( model_id=model_id, runtime_parameters=runtime_parameters.get(model_id, ()), + generation_parameter_domains=generation_domains.get(model_id, ()), ) for model_id in passive.models ) @@ -131,7 +136,11 @@ def _capability_evidence( async def _probe_local_llm_server_registry( connection: EndpointConnectionInput, -) -> tuple[dict[str, tuple[RuntimeParameterReadModel, ...]], str | None]: +) -> tuple[ + dict[str, tuple[RuntimeParameterReadModel, ...]], + dict[str, tuple[GenerationParameterDomainReadModel, ...]], + str | None, +]: """Best-effort first-party enrichment; generic OpenAI discovery stays authoritative.""" root = str(local_server_root(connection)).rstrip("/") url = f"{root}/api/v1/models/registry" @@ -142,17 +151,20 @@ async def _probe_local_llm_server_registry( payload = response.json() except (httpx.HTTPError, ValueError, TypeError): return ( + {}, {}, "Model discovery succeeded, but Local LLM Server runtime details are unavailable.", ) if not isinstance(payload, Mapping): - return {}, "Local LLM Server returned an invalid runtime registry response." + return {}, {}, "Local LLM Server returned an invalid runtime registry response." raw_models = payload.get("models") if not isinstance(raw_models, list): - return {}, "Local LLM Server runtime registry did not include a model list." + return {}, {}, "Local LLM Server runtime registry did not include a model list." - result: dict[str, tuple[RuntimeParameterReadModel, ...]] = {} + runtime_result: dict[str, tuple[RuntimeParameterReadModel, ...]] = {} + domain_result: dict[str, tuple[GenerationParameterDomainReadModel, ...]] = {} + invalid_domain_metadata = False for raw_model in raw_models: if not isinstance(raw_model, Mapping): continue @@ -163,7 +175,7 @@ async def _probe_local_llm_server_registry( runtime_config = raw_model.get("runtime_config") config = runtime_config if isinstance(runtime_config, Mapping) else {} names = raw_capabilities if isinstance(raw_capabilities, list) else [] - parameters = tuple( + runtime_result[model_id] = tuple( RuntimeParameterReadModel( name=name, current_value=config.get(name), @@ -171,8 +183,32 @@ async def _probe_local_llm_server_registry( for name in names if isinstance(name, str) and name ) - result[model_id] = parameters - return result, None + + raw_domains = raw_model.get("generation_parameter_domains") + if raw_domains is None: + domain_result[model_id] = () + continue + if not isinstance(raw_domains, list): + invalid_domain_metadata = True + domain_result[model_id] = () + continue + domains: list[GenerationParameterDomainReadModel] = [] + for raw_domain in raw_domains: + if not isinstance(raw_domain, Mapping): + invalid_domain_metadata = True + continue + try: + domains.append(GenerationParameterDomainReadModel.model_validate(dict(raw_domain))) + except ValidationError: + invalid_domain_metadata = True + domain_result[model_id] = tuple(sorted(domains, key=lambda item: item.name)) + + warning = ( + "Local LLM Server returned invalid generation-domain metadata; invalid domains were ignored." + if invalid_domain_metadata + else None + ) + return runtime_result, domain_result, warning def _model_id(raw_model: Mapping[object, object]) -> str | None: From 69bb7ec7d9474c4af7db0914f3e5f85a542a3f36 Mon Sep 17 00:00:00 2001 From: Daniele21 Date: Thu, 3 Sep 2026 15:23:36 +0200 Subject: [PATCH 04/35] feat: preserve model scoped generation domains in planning --- src/performance_lab/application/planning_queries.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/performance_lab/application/planning_queries.py b/src/performance_lab/application/planning_queries.py index dff3b831..2205a922 100644 --- a/src/performance_lab/application/planning_queries.py +++ b/src/performance_lab/application/planning_queries.py @@ -41,6 +41,7 @@ DatasetSummaryReadModel, DiscoveredModelReadModel, EndpointConnectionInput, + GenerationParameterDomainReadModel, SuiteSummaryReadModel, TargetSummaryReadModel, ) @@ -166,6 +167,7 @@ def campaign_planning_context(self) -> CampaignPlanningContextReadModel: target.target_id, model.model_id, source="discovered", + generation_parameter_domains=model.generation_parameter_domains, ) ) candidates = list({item.candidate_id: item for item in candidates}.values()) @@ -521,6 +523,7 @@ def _candidate( model_id: str, *, source: Literal["configured", "discovered"], + generation_parameter_domains: tuple[GenerationParameterDomainReadModel, ...] = (), ) -> CandidateModelReadModel: payload = json.dumps( {"target_id": target_id, "model_id": model_id}, @@ -532,6 +535,7 @@ def _candidate( candidate_id=candidate_id, target_id=target_id, model_id=model_id, + generation_parameter_domains=generation_parameter_domains, source=source, ) From 025d8087fe4c0f00b0182ddf639e1ab42b6cb9b4 Mon Sep 17 00:00:00 2001 From: Daniele21 Date: Thu, 3 Sep 2026 15:24:00 +0200 Subject: [PATCH 05/35] feat: export generation domain read model --- src/performance_lab/application/__init__.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/performance_lab/application/__init__.py b/src/performance_lab/application/__init__.py index 22d0fc78..79f7883c 100644 --- a/src/performance_lab/application/__init__.py +++ b/src/performance_lab/application/__init__.py @@ -61,6 +61,7 @@ EvaluatorDefinitionReadModel, EvidenceAvailability, FrozenExecutionPreviewReadModel, + GenerationParameterDomainReadModel, IdentitySummary, MetricDimension, MetricReadModel, @@ -123,6 +124,7 @@ "EvidenceContentState", "ExplanationState", "FrozenExecutionPreviewReadModel", + "GenerationParameterDomainReadModel", "IdentitySummary", "MetricDimension", "MetricReadModel", From c6e1f50a7d39e0af02f846fa433328ff399c1642 Mon Sep 17 00:00:00 2001 From: Daniele21 Date: Thu, 3 Sep 2026 15:24:49 +0200 Subject: [PATCH 06/35] feat: project generation domains to typescript --- frontend/src/api/types.ts | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/frontend/src/api/types.ts b/frontend/src/api/types.ts index a575e658..f7ded817 100644 --- a/frontend/src/api/types.ts +++ b/frontend/src/api/types.ts @@ -104,6 +104,18 @@ export interface CapabilitySupportReadModel extends UIModelIdentity { detail: string | null; } +export interface GenerationParameterDomainReadModel extends UIModelIdentity { + name: string; + kind: "float" | "integer" | "boolean"; + scope: "request_generation"; + source: "local_llm_server"; + provenance: "registry_declared"; + minimum: number | null; + maximum: number | null; + step: number | null; + values: boolean[]; +} + export interface RuntimeParameterReadModel extends UIModelIdentity { name: string; scope: "runtime_load"; @@ -115,6 +127,7 @@ export interface RuntimeParameterReadModel extends UIModelIdentity { export interface DiscoveredModelReadModel extends UIModelIdentity { model_id: string; runtime_parameters: RuntimeParameterReadModel[]; + generation_parameter_domains: GenerationParameterDomainReadModel[]; } export interface EndpointProbeReadModel extends UIModelIdentity { From 4304815f51d742fe428d23d5cd8de126c7c0146b Mon Sep 17 00:00:00 2001 From: Daniele21 Date: Thu, 3 Sep 2026 15:25:09 +0200 Subject: [PATCH 07/35] feat: expose candidate generation domains to frontend --- frontend/src/api/planning-types.ts | 2 ++ 1 file changed, 2 insertions(+) diff --git a/frontend/src/api/planning-types.ts b/frontend/src/api/planning-types.ts index 46a22ba0..0768ac62 100644 --- a/frontend/src/api/planning-types.ts +++ b/frontend/src/api/planning-types.ts @@ -1,5 +1,6 @@ import type { DatasetSummaryReadModel, + GenerationParameterDomainReadModel, SuiteSummaryReadModel, TargetSummaryReadModel, UIModelIdentity, @@ -28,6 +29,7 @@ export interface CandidateModelReadModel extends UIModelIdentity { runtime_name: string | null; runtime_version: string | null; runtime_config_digest: string | null; + generation_parameter_domains: GenerationParameterDomainReadModel[]; source: "configured" | "discovered"; } From 0c987e4c51bfe81f70f560113a90d22799c73104 Mon Sep 17 00:00:00 2001 From: Daniele21 Date: Thu, 3 Sep 2026 15:26:11 +0200 Subject: [PATCH 08/35] test: cover declared generation domain ingestion --- tests/test_generation_parameter_domains.py | 181 +++++++++++++++++++++ 1 file changed, 181 insertions(+) create mode 100644 tests/test_generation_parameter_domains.py diff --git a/tests/test_generation_parameter_domains.py b/tests/test_generation_parameter_domains.py new file mode 100644 index 00000000..f6edde5b --- /dev/null +++ b/tests/test_generation_parameter_domains.py @@ -0,0 +1,181 @@ +from __future__ import annotations + +import asyncio + +from performance_lab.application import ( + CampaignSearchStrategy, + DiscoveredModelReadModel, + EndpointConnectionInput, + GenerationParameterDomainReadModel, + UIQueryService, +) +from performance_lab.application.endpoint_discovery import _probe_local_llm_server_registry +from performance_lab.storage import SQLiteRunStore + + +class _RegistryResponse: + def __init__(self, payload: object) -> None: + self._payload = payload + + def raise_for_status(self) -> None: + return None + + def json(self) -> object: + return self._payload + + +class _RegistryClient: + def __init__(self, payload: object) -> None: + self._payload = payload + + async def __aenter__(self) -> _RegistryClient: + return self + + async def __aexit__(self, *_args: object) -> None: + return None + + async def get(self, _url: str) -> _RegistryResponse: + return _RegistryResponse(self._payload) + + +def _patch_registry(monkeypatch, payload: object) -> None: + monkeypatch.setattr( + "performance_lab.application.endpoint_discovery.httpx.AsyncClient", + lambda **_kwargs: _RegistryClient(payload), + ) + + +def test_local_llm_registry_preserves_declared_model_scoped_domains(monkeypatch) -> None: + _patch_registry( + monkeypatch, + { + "models": [ + { + "key": "demo", + "model_id": "org/demo", + "config_capabilities": ["n_batch"], + "runtime_config": {"n_batch": 512}, + "generation_parameter_domains": [ + { + "name": "temperature", + "kind": "float", + "minimum": 0.0, + "maximum": 0.8, + "step": 0.1, + "provenance": "registry_declared", + }, + { + "name": "enable_thinking", + "kind": "boolean", + "values": [False, True], + "provenance": "registry_declared", + }, + ], + } + ] + }, + ) + + runtime_parameters, domains, warning = asyncio.run( + _probe_local_llm_server_registry( + EndpointConnectionInput( + base_url="http://127.0.0.1:1235/v1", + server_type="local_llm_server", + ) + ) + ) + + assert warning is None + assert runtime_parameters["org/demo"][0].scope == "runtime_load" + assert runtime_parameters["org/demo"][0].name == "n_batch" + assert [domain.name for domain in domains["org/demo"]] == [ + "enable_thinking", + "temperature", + ] + temperature = domains["org/demo"][1] + assert temperature.scope == "request_generation" + assert temperature.source == "local_llm_server" + assert temperature.provenance == "registry_declared" + assert temperature.minimum == 0.0 + assert temperature.maximum == 0.8 + assert temperature.step == 0.1 + + +def test_invalid_domain_metadata_is_ignored_without_losing_registry_model(monkeypatch) -> None: + _patch_registry( + monkeypatch, + { + "models": [ + { + "model_id": "org/demo", + "generation_parameter_domains": [ + { + "name": "temperature", + "kind": "float", + "minimum": 1.0, + "maximum": 0.0, + "provenance": "registry_declared", + } + ], + } + ] + }, + ) + + runtime_parameters, domains, warning = asyncio.run( + _probe_local_llm_server_registry( + EndpointConnectionInput( + base_url="http://127.0.0.1:1235/v1", + server_type="local_llm_server", + ) + ) + ) + + assert runtime_parameters["org/demo"] == () + assert domains["org/demo"] == () + assert warning is not None + assert "invalid generation-domain metadata" in warning + + +def test_planning_preserves_domains_but_does_not_enable_sweeps(tmp_path) -> None: + queries = UIQueryService(SQLiteRunStore(tmp_path / "runs.sqlite3")) + temperature = GenerationParameterDomainReadModel( + name="temperature", + kind="float", + minimum=0.0, + maximum=0.8, + step=0.1, + ) + target = queries.register_session_connection( + EndpointConnectionInput( + display_name="Declared-domain runtime", + base_url="http://127.0.0.1:1235/v1", + server_type="local_llm_server", + ), + discovered_models=( + DiscoveredModelReadModel( + model_id="org/demo", + generation_parameter_domains=(temperature,), + ), + ), + supported_generation_parameters=("temperature",), + ) + + target_context = next( + item + for item in queries.campaign_planning_context().targets + if item.target.target_id == target.target_id + ) + + assert len(target_context.candidates) == 1 + candidate = target_context.candidates[0] + assert candidate.model_id == "org/demo" + assert candidate.generation_parameter_domains == (temperature,) + assert target_context.bounded_generation_parameter_ranges == () + quick = next( + option + for option in target_context.configuration_search_options + if option.strategy == CampaignSearchStrategy.QUICK + ) + assert not quick.available + assert quick.blocked_reason is not None From 965e1ac69549309a497809118b09706ed274a82a Mon Sep 17 00:00:00 2001 From: Daniele21 Date: Thu, 3 Sep 2026 16:06:13 +0200 Subject: [PATCH 09/35] fix(value-03a): export generation domain read model --- frontend/src/api/index.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/frontend/src/api/index.ts b/frontend/src/api/index.ts index b50b3f9a..d9fc7b8c 100644 --- a/frontend/src/api/index.ts +++ b/frontend/src/api/index.ts @@ -80,6 +80,7 @@ export type { EvidenceContentState, ExplanationState, FrozenExecutionPreviewReadModel, + GenerationParameterDomainReadModel, IdentityDifference, IdentitySummary, MetricDelta, From 34dcce8cfd7e409813f436e35f733f2d40f53ee7 Mon Sep 17 00:00:00 2001 From: Daniele21 Date: Thu, 3 Sep 2026 16:06:33 +0200 Subject: [PATCH 10/35] fix(value-03a): update typed campaign fixture --- frontend/src/pages/find-best-setup/FindBestSetupPage.test.tsx | 1 + 1 file changed, 1 insertion(+) diff --git a/frontend/src/pages/find-best-setup/FindBestSetupPage.test.tsx b/frontend/src/pages/find-best-setup/FindBestSetupPage.test.tsx index d59b9492..1a2bb9be 100644 --- a/frontend/src/pages/find-best-setup/FindBestSetupPage.test.tsx +++ b/frontend/src/pages/find-best-setup/FindBestSetupPage.test.tsx @@ -58,6 +58,7 @@ const context: CampaignPlanningContextReadModel = { runtime_name: null, runtime_version: null, runtime_config_digest: null, + generation_parameter_domains: [], source: "configured", }, ], From 5d7ab944cc02a40dd9fe94a5685433e973c6402d Mon Sep 17 00:00:00 2001 From: Daniele21 Date: Thu, 3 Sep 2026 16:07:03 +0200 Subject: [PATCH 11/35] fix(value-03a): update typed endpoint probe fixture --- frontend/src/pages/test-model/TestModelPage.test.tsx | 1 + 1 file changed, 1 insertion(+) diff --git a/frontend/src/pages/test-model/TestModelPage.test.tsx b/frontend/src/pages/test-model/TestModelPage.test.tsx index 04f5f3b7..e451fd93 100644 --- a/frontend/src/pages/test-model/TestModelPage.test.tsx +++ b/frontend/src/pages/test-model/TestModelPage.test.tsx @@ -120,6 +120,7 @@ const probe: EndpointProbeReadModel = { provenance: "local_llm_server", }, ], + generation_parameter_domains: [], }, ], capabilities: [ From ec09939a89859f1b60dcb9efddb91d34255a0483 Mon Sep 17 00:00:00 2001 From: Daniele21 Date: Thu, 3 Sep 2026 16:09:29 +0200 Subject: [PATCH 12/35] fix(value-03a): format generation domain validation --- src/performance_lab/application/ui_models.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/performance_lab/application/ui_models.py b/src/performance_lab/application/ui_models.py index 8e3e7175..033d90af 100644 --- a/src/performance_lab/application/ui_models.py +++ b/src/performance_lab/application/ui_models.py @@ -197,7 +197,11 @@ def validate_domain_shape(self) -> GenerationParameterDomainReadModel: ): raise ValueError("integer generation domains require integer bounds") if self.step is not None: - if isinstance(self.step, bool) or self.step <= 0 or self.step > self.maximum - self.minimum: + if ( + isinstance(self.step, bool) + or self.step <= 0 + or self.step > self.maximum - self.minimum + ): raise ValueError("generation domain step must be positive and within the span") if self.kind == "integer" and not isinstance(self.step, int): raise ValueError("integer generation domains require an integer step") From 85a61e82818cad7ba0609d4dc055f40b12382e7d Mon Sep 17 00:00:00 2001 From: Daniele21 Date: Thu, 3 Sep 2026 16:10:46 +0200 Subject: [PATCH 13/35] fix(value-03a): format endpoint domain enrichment --- src/performance_lab/application/endpoint_discovery.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/src/performance_lab/application/endpoint_discovery.py b/src/performance_lab/application/endpoint_discovery.py index eb25ac1e..eb089147 100644 --- a/src/performance_lab/application/endpoint_discovery.py +++ b/src/performance_lab/application/endpoint_discovery.py @@ -74,8 +74,8 @@ async def probe_endpoint_profile( and local_connection is not None and local_connection.server_type == "local_llm_server" ): - runtime_parameters, generation_domains, warning = await _probe_local_llm_server_registry( - local_connection + runtime_parameters, generation_domains, warning = ( + await _probe_local_llm_server_registry(local_connection) ) models = tuple( @@ -198,7 +198,9 @@ async def _probe_local_llm_server_registry( invalid_domain_metadata = True continue try: - domains.append(GenerationParameterDomainReadModel.model_validate(dict(raw_domain))) + domains.append( + GenerationParameterDomainReadModel.model_validate(dict(raw_domain)) + ) except ValidationError: invalid_domain_metadata = True domain_result[model_id] = tuple(sorted(domains, key=lambda item: item.name)) From 29ea284f0ed58e88fb37e66020ba62d7a74478d9 Mon Sep 17 00:00:00 2001 From: Daniele21 Date: Thu, 3 Sep 2026 16:13:19 +0200 Subject: [PATCH 14/35] fix(value-03a): format generation domain tests --- tests/test_generation_parameter_domains.py | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/tests/test_generation_parameter_domains.py b/tests/test_generation_parameter_domains.py index f6edde5b..0bc1c9d4 100644 --- a/tests/test_generation_parameter_domains.py +++ b/tests/test_generation_parameter_domains.py @@ -9,7 +9,9 @@ GenerationParameterDomainReadModel, UIQueryService, ) -from performance_lab.application.endpoint_discovery import _probe_local_llm_server_registry +from performance_lab.application.endpoint_discovery import ( + _probe_local_llm_server_registry, +) from performance_lab.storage import SQLiteRunStore @@ -45,7 +47,9 @@ def _patch_registry(monkeypatch, payload: object) -> None: ) -def test_local_llm_registry_preserves_declared_model_scoped_domains(monkeypatch) -> None: +def test_local_llm_registry_preserves_declared_model_scoped_domains( + monkeypatch, +) -> None: _patch_registry( monkeypatch, { @@ -101,7 +105,9 @@ def test_local_llm_registry_preserves_declared_model_scoped_domains(monkeypatch) assert temperature.step == 0.1 -def test_invalid_domain_metadata_is_ignored_without_losing_registry_model(monkeypatch) -> None: +def test_invalid_domain_metadata_is_ignored_without_losing_registry_model( + monkeypatch, +) -> None: _patch_registry( monkeypatch, { From 43ec8ed8635723c8564faa6aea3578f7c85a0f49 Mon Sep 17 00:00:00 2001 From: Daniele21 Date: Thu, 3 Sep 2026 16:16:01 +0200 Subject: [PATCH 15/35] fix(value-03a): align endpoint formatting to repo line length --- src/performance_lab/application/endpoint_discovery.py | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/src/performance_lab/application/endpoint_discovery.py b/src/performance_lab/application/endpoint_discovery.py index eb089147..eb25ac1e 100644 --- a/src/performance_lab/application/endpoint_discovery.py +++ b/src/performance_lab/application/endpoint_discovery.py @@ -74,8 +74,8 @@ async def probe_endpoint_profile( and local_connection is not None and local_connection.server_type == "local_llm_server" ): - runtime_parameters, generation_domains, warning = ( - await _probe_local_llm_server_registry(local_connection) + runtime_parameters, generation_domains, warning = await _probe_local_llm_server_registry( + local_connection ) models = tuple( @@ -198,9 +198,7 @@ async def _probe_local_llm_server_registry( invalid_domain_metadata = True continue try: - domains.append( - GenerationParameterDomainReadModel.model_validate(dict(raw_domain)) - ) + domains.append(GenerationParameterDomainReadModel.model_validate(dict(raw_domain))) except ValidationError: invalid_domain_metadata = True domain_result[model_id] = tuple(sorted(domains, key=lambda item: item.name)) From 0ec11807a9e43913d92c191127ce3498a15fd6d2 Mon Sep 17 00:00:00 2001 From: Daniele21 Date: Thu, 3 Sep 2026 16:16:26 +0200 Subject: [PATCH 16/35] fix(value-03a): align tests to repo line length --- tests/test_generation_parameter_domains.py | 12 +++--------- 1 file changed, 3 insertions(+), 9 deletions(-) diff --git a/tests/test_generation_parameter_domains.py b/tests/test_generation_parameter_domains.py index 0bc1c9d4..f6edde5b 100644 --- a/tests/test_generation_parameter_domains.py +++ b/tests/test_generation_parameter_domains.py @@ -9,9 +9,7 @@ GenerationParameterDomainReadModel, UIQueryService, ) -from performance_lab.application.endpoint_discovery import ( - _probe_local_llm_server_registry, -) +from performance_lab.application.endpoint_discovery import _probe_local_llm_server_registry from performance_lab.storage import SQLiteRunStore @@ -47,9 +45,7 @@ def _patch_registry(monkeypatch, payload: object) -> None: ) -def test_local_llm_registry_preserves_declared_model_scoped_domains( - monkeypatch, -) -> None: +def test_local_llm_registry_preserves_declared_model_scoped_domains(monkeypatch) -> None: _patch_registry( monkeypatch, { @@ -105,9 +101,7 @@ def test_local_llm_registry_preserves_declared_model_scoped_domains( assert temperature.step == 0.1 -def test_invalid_domain_metadata_is_ignored_without_losing_registry_model( - monkeypatch, -) -> None: +def test_invalid_domain_metadata_is_ignored_without_losing_registry_model(monkeypatch) -> None: _patch_registry( monkeypatch, { From caed430641d40228e9778155f5b8157daf5238a5 Mon Sep 17 00:00:00 2001 From: Daniele21 Date: Thu, 3 Sep 2026 16:18:46 +0200 Subject: [PATCH 17/35] fix(value-03a): satisfy endpoint lint line length --- src/performance_lab/application/endpoint_discovery.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/performance_lab/application/endpoint_discovery.py b/src/performance_lab/application/endpoint_discovery.py index eb25ac1e..6275deda 100644 --- a/src/performance_lab/application/endpoint_discovery.py +++ b/src/performance_lab/application/endpoint_discovery.py @@ -204,7 +204,8 @@ async def _probe_local_llm_server_registry( domain_result[model_id] = tuple(sorted(domains, key=lambda item: item.name)) warning = ( - "Local LLM Server returned invalid generation-domain metadata; invalid domains were ignored." + "Local LLM Server returned invalid generation-domain metadata; " + "invalid domains were ignored." if invalid_domain_metadata else None ) From 116cebdfd28a91c036924e4502456531a41f9aa6 Mon Sep 17 00:00:00 2001 From: Daniele21 Date: Fri, 4 Sep 2026 16:17:01 +0200 Subject: [PATCH 18/35] VALUE-03B1: freeze generation in run config --- src/performance_lab/run_config.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/performance_lab/run_config.py b/src/performance_lab/run_config.py index 0e772863..5a086a5e 100644 --- a/src/performance_lab/run_config.py +++ b/src/performance_lab/run_config.py @@ -8,7 +8,7 @@ from pydantic import BaseModel, ConfigDict, Field, HttpUrl, ValidationError -from performance_lab.domain import EndpointProfile, EvidenceMode, HardwareIdentity +from performance_lab.domain import EndpointProfile, EvidenceMode, GenerationConfig, HardwareIdentity RUN_CONFIG_VERSION: Literal[1] = 1 @@ -50,6 +50,7 @@ class StarterRunConfig(BaseModel): local_llm_server_telemetry: LocalLLMServerTelemetryConfig | None = None local_llm_server_identity: LocalLLMServerIdentityConfig | None = None hardware: HardwareIdentity = Field(default_factory=HardwareIdentity) + generation: GenerationConfig | None = None suite_id: str = Field(default="general-diagnostic-starter", min_length=1) suite_version: str | None = Field(default=None, min_length=1) From de4f2831bb8a2ed36f4a7bd2675352991c1df672 Mon Sep 17 00:00:00 2001 From: Daniele21 Date: Fri, 4 Sep 2026 16:17:34 +0200 Subject: [PATCH 19/35] VALUE-03B1: execute frozen fingerprint generation --- src/performance_lab/engine/orchestrator.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/performance_lab/engine/orchestrator.py b/src/performance_lab/engine/orchestrator.py index 9fc242b3..b7d483ba 100644 --- a/src/performance_lab/engine/orchestrator.py +++ b/src/performance_lab/engine/orchestrator.py @@ -151,7 +151,7 @@ async def run( task_id=task.task_id, sample_id=record.sample_id, model_id=fingerprint.model.model_id, - generation=suite.generation, + generation=fingerprint.generation, input_value=record.input, expected=record.expected, evaluator=evaluator, From d5ac7797ef7aec2cdd55b9b3f632a53a9fd2a72a Mon Sep 17 00:00:00 2001 From: Daniele21 Date: Fri, 4 Sep 2026 16:18:02 +0200 Subject: [PATCH 20/35] VALUE-03B1: bind runner to frozen generation --- src/performance_lab/runner.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/performance_lab/runner.py b/src/performance_lab/runner.py index 1951f300..c607a05a 100644 --- a/src/performance_lab/runner.py +++ b/src/performance_lab/runner.py @@ -81,6 +81,7 @@ async def execute_starter_run( """ bundle = _resolve_execution_bundle(config) + generation = config.generation or bundle.suite.generation run_id = config.run_id or f"run-{uuid4()}" store = SQLiteRunStore(config.store_path) base_adapter = OpenAICompatibleAdapter(config.endpoint, model=config.model_id) @@ -113,7 +114,7 @@ async def execute_starter_run( model=identity.model, runtime=identity.runtime, hardware=identity.hardware, - generation=bundle.suite.generation, + generation=generation, prompt_template_version="direct-user-v1", dataset_snapshots=snapshots, evaluator_versions=evaluator_versions, From 97d9aefbcf8d3ea40b7b4c49b11460d7da95e420 Mon Sep 17 00:00:00 2001 From: Daniele21 Date: Fri, 4 Sep 2026 16:19:42 +0200 Subject: [PATCH 21/35] VALUE-03B1: identify campaign configurations --- src/performance_lab/domain/campaign.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/src/performance_lab/domain/campaign.py b/src/performance_lab/domain/campaign.py index 0b1fee22..5670ec18 100644 --- a/src/performance_lab/domain/campaign.py +++ b/src/performance_lab/domain/campaign.py @@ -61,6 +61,7 @@ class DecisionPolicyRef(FrozenModel): class CampaignEntry(VersionedModel): entry_id: NonEmptyStr candidate_id: NonEmptyStr + configuration_id: NonEmptyStr = "fixed-1" model_id: NonEmptyStr config_digest: str = Field(pattern=r"^[0-9a-f]{64}$") status: CampaignEntryStatus = CampaignEntryStatus.QUEUED @@ -111,11 +112,13 @@ def require_unique_entries(cls, value: tuple[CampaignEntry, ...]) -> tuple[Campa if not value: raise ValueError("campaign requires at least one entry") entry_ids = [entry.entry_id for entry in value] - candidate_ids = [entry.candidate_id for entry in value] + candidate_configurations = [ + (entry.candidate_id, entry.configuration_id) for entry in value + ] if len(entry_ids) != len(set(entry_ids)): raise ValueError("campaign entry ids must be unique") - if len(candidate_ids) != len(set(candidate_ids)): - raise ValueError("campaign candidate ids must be unique") + if len(candidate_configurations) != len(set(candidate_configurations)): + raise ValueError("campaign candidate/configuration pairs must be unique") return value @model_validator(mode="after") From 7806fd50ccb5ba95df163e3dde5e8bdec6035c28 Mon Sep 17 00:00:00 2001 From: Daniele21 Date: Fri, 4 Sep 2026 16:20:33 +0200 Subject: [PATCH 22/35] VALUE-03B1: carry configuration identity into campaigns --- src/performance_lab/application/campaign_jobs.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/performance_lab/application/campaign_jobs.py b/src/performance_lab/application/campaign_jobs.py index 786e2167..20941fdd 100644 --- a/src/performance_lab/application/campaign_jobs.py +++ b/src/performance_lab/application/campaign_jobs.py @@ -47,6 +47,7 @@ class CampaignRunSpec: candidate_id: str model_id: str config: StarterRunConfig + configuration_id: str = "fixed-1" @dataclass(frozen=True, slots=True) @@ -139,12 +140,14 @@ async def launch(self, plan: CampaignLaunchPlan) -> Campaign: candidate_id=spec.candidate_id, model_id=spec.model_id, config=config, + configuration_id=spec.configuration_id, ) ) entries.append( CampaignEntry( entry_id=f"entry-{index + 1}", candidate_id=spec.candidate_id, + configuration_id=spec.configuration_id, model_id=spec.model_id, config_digest=starter_run_config_digest(config), ) From 1f91a35fd6e822a786d466a75d49c18a00de4583 Mon Sep 17 00:00:00 2001 From: Daniele21 Date: Fri, 4 Sep 2026 16:21:47 +0200 Subject: [PATCH 23/35] VALUE-03B1: expose exact frozen generation configurations --- .../application/planning_models.py | 21 +++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/src/performance_lab/application/planning_models.py b/src/performance_lab/application/planning_models.py index 6afaeac8..2bebddd7 100644 --- a/src/performance_lab/application/planning_models.py +++ b/src/performance_lab/application/planning_models.py @@ -109,14 +109,35 @@ class CampaignPlanIssueReadModel(UIModel): field: str | None = None +class FrozenGenerationConfigurationReadModel(UIModel): + """One exact request-generation configuration reviewed before campaign launch.""" + + configuration_id: str = Field(min_length=1) + generation_digest: str = Field(pattern=r"^[0-9a-f]{64}$") + generation: GenerationConfig + + class ConfigurationSearchPlanReadModel(UIModel): strategy: CampaignSearchStrategy title: str = Field(min_length=1) configuration_count_per_candidate: int = Field(gt=0) base_generation: GenerationConfig + configurations: tuple[FrozenGenerationConfigurationReadModel, ...] bounded_parameter_ranges: tuple[str, ...] = () note: str = Field(min_length=1) + @model_validator(mode="after") + def require_exact_configuration_matrix(self) -> ConfigurationSearchPlanReadModel: + if len(self.configurations) != self.configuration_count_per_candidate: + raise ValueError("configuration count must match the frozen configuration matrix") + configuration_ids = [item.configuration_id for item in self.configurations] + if len(configuration_ids) != len(set(configuration_ids)): + raise ValueError("configuration ids must be unique") + generation_digests = [item.generation_digest for item in self.configurations] + if len(generation_digests) != len(set(generation_digests)): + raise ValueError("frozen generation configurations must be unique") + return self + class BenchmarkPlanReadModel(UIModel): suite: SuiteSummaryReadModel From 9e9d11a4161295e59ae143dd780a7eb030765142 Mon Sep 17 00:00:00 2001 From: Daniele21 Date: Fri, 4 Sep 2026 16:22:54 +0200 Subject: [PATCH 24/35] VALUE-03B1: freeze exact campaign generation matrix --- .../application/planning_queries.py | 41 +++++++++++++++++-- 1 file changed, 37 insertions(+), 4 deletions(-) diff --git a/src/performance_lab/application/planning_queries.py b/src/performance_lab/application/planning_queries.py index 2205a922..c838d64c 100644 --- a/src/performance_lab/application/planning_queries.py +++ b/src/performance_lab/application/planning_queries.py @@ -12,6 +12,7 @@ DecisionPolicyRef, EndpointProfile, EvaluationSuite, + GenerationConfig, HardwareIdentity, Target, ) @@ -35,6 +36,7 @@ ConfigurationSearchOptionReadModel, ConfigurationSearchPlanReadModel, DecisionPolicyReadModel, + FrozenGenerationConfigurationReadModel, UseCaseReadModel, ) from .ui_models import ( @@ -271,11 +273,15 @@ def preview_campaign_plan( else snapshot.sample_count ) + frozen_configurations = ( + _frozen_generation_configuration("fixed-1", suite.generation), + ) configuration = ConfigurationSearchPlanReadModel( strategy=CampaignSearchStrategy.FIXED, title="Fixed benchmark configuration", - configuration_count_per_candidate=1, + configuration_count_per_candidate=len(frozen_configurations), base_generation=suite.generation, + configurations=frozen_configurations, bounded_parameter_ranges=(), note=( "Uses the benchmark suite generation configuration exactly as authored. " @@ -290,10 +296,10 @@ def preview_campaign_plan( evaluator_ids=tuple(evaluator_ids), case_count_per_run=case_count, ) - planned_run_count = len(candidates) + planned_run_count = len(candidates) * len(frozen_configurations) estimate = CampaignEstimateReadModel( candidate_count=len(candidates), - configuration_count_per_candidate=1, + configuration_count_per_candidate=len(frozen_configurations), planned_run_count=planned_run_count, benchmark_case_count_per_run=case_count, estimated_request_count=planned_run_count * case_count, @@ -350,6 +356,7 @@ def prepare_campaign_launch( if ( preview.use_case is None or preview.target is None + or preview.configuration_search is None or preview.benchmark_plan is None or preview.decision_policy is None ): @@ -360,15 +367,18 @@ def prepare_campaign_launch( CampaignRunSpec( candidate_id=candidate.candidate_id, model_id=candidate.model_id, + configuration_id=configuration.configuration_id, config=self._campaign_run_config( target=preview.target, endpoint=endpoint, model_id=candidate.model_id, + generation=configuration.generation, suite_id=preview.benchmark_plan.suite.suite_id, suite_version=preview.benchmark_plan.suite.suite_version, ), ) for candidate in preview.candidates + for configuration in preview.configuration_search.configurations ) return CampaignLaunchPlan( plan_digest=expected_plan_digest, @@ -402,6 +412,7 @@ def _campaign_run_config( target: TargetSummaryReadModel, endpoint: EndpointProfile, model_id: str, + generation: GenerationConfig, suite_id: str, suite_version: str, ) -> StarterRunConfig: @@ -420,6 +431,7 @@ def _campaign_run_config( "endpoint": endpoint, "model_id": model_id, "run_id": None, + "generation": generation, "suite_id": suite_id, "suite_version": suite_version, } @@ -566,6 +578,27 @@ def _search_options() -> tuple[ConfigurationSearchOptionReadModel, ...]: ) +def _frozen_generation_configuration( + configuration_id: str, + generation: GenerationConfig, +) -> FrozenGenerationConfigurationReadModel: + return FrozenGenerationConfigurationReadModel( + configuration_id=configuration_id, + generation_digest=_generation_digest(generation), + generation=generation, + ) + + +def _generation_digest(generation: GenerationConfig) -> str: + canonical = json.dumps( + generation.model_dump(mode="json"), + sort_keys=True, + separators=(",", ":"), + ensure_ascii=False, + ) + return sha256(canonical.encode("utf-8")).hexdigest() + + def _suite_summary(suite: EvaluationSuite) -> SuiteSummaryReadModel: return SuiteSummaryReadModel( suite_id=suite.suite_id, @@ -608,7 +641,7 @@ def _plan_digest( decision_policy: DecisionPolicyReadModel, ) -> str: payload = { - "contract": "campaign-plan-v2", + "contract": "campaign-plan-v3", "use_case": use_case.model_dump(mode="json"), "target": target.model_dump(mode="json"), "candidates": [item.model_dump(mode="json") for item in candidates], From 8e2039646c52e0ac702f58081b77609383e5dc90 Mon Sep 17 00:00:00 2001 From: Daniele21 Date: Fri, 4 Sep 2026 16:23:18 +0200 Subject: [PATCH 25/35] VALUE-03B1: project configuration identity --- src/performance_lab/application/campaign_models.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/performance_lab/application/campaign_models.py b/src/performance_lab/application/campaign_models.py index 79945713..df7b0836 100644 --- a/src/performance_lab/application/campaign_models.py +++ b/src/performance_lab/application/campaign_models.py @@ -47,6 +47,7 @@ class CampaignResultsReadModel(UIModel): class CampaignEntryReadModel(UIModel): entry_id: str = Field(min_length=1) candidate_id: str = Field(min_length=1) + configuration_id: str = Field(min_length=1) model_id: str = Field(min_length=1) config_digest: str = Field(pattern=r"^[0-9a-f]{64}$") status: CampaignEntryStatus @@ -89,6 +90,7 @@ class CampaignCaseSummaryReadModel(UIModel): class CampaignCaseCandidateReadModel(UIModel): entry_id: str = Field(min_length=1) candidate_id: str = Field(min_length=1) + configuration_id: str = Field(min_length=1) model_id: str = Field(min_length=1) config_digest: str = Field(pattern=r"^[0-9a-f]{64}$") entry_status: CampaignEntryStatus From dae65b79585a31b2d4728cc6ecdf50e588a64f4f Mon Sep 17 00:00:00 2001 From: Daniele21 Date: Fri, 4 Sep 2026 16:24:07 +0200 Subject: [PATCH 26/35] VALUE-03B1: type frozen generation configurations --- frontend/src/api/planning-types.ts | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/frontend/src/api/planning-types.ts b/frontend/src/api/planning-types.ts index 0768ac62..ec2f7598 100644 --- a/frontend/src/api/planning-types.ts +++ b/frontend/src/api/planning-types.ts @@ -69,11 +69,18 @@ export interface CampaignPlanIssueReadModel extends UIModelIdentity { field: string | null; } +export interface FrozenGenerationConfigurationReadModel extends UIModelIdentity { + configuration_id: string; + generation_digest: string; + generation: Record; +} + export interface ConfigurationSearchPlanReadModel extends UIModelIdentity { strategy: CampaignSearchStrategy; title: string; configuration_count_per_candidate: number; base_generation: Record; + configurations: FrozenGenerationConfigurationReadModel[]; bounded_parameter_ranges: string[]; note: string; } From eb3002c95a0de98be9d3c455047d11459d77c0e2 Mon Sep 17 00:00:00 2001 From: Daniele21 Date: Fri, 4 Sep 2026 16:24:32 +0200 Subject: [PATCH 27/35] VALUE-03B1: type campaign configuration identity --- frontend/src/api/campaign-types.ts | 2 ++ 1 file changed, 2 insertions(+) diff --git a/frontend/src/api/campaign-types.ts b/frontend/src/api/campaign-types.ts index c771e956..8a0a63e2 100644 --- a/frontend/src/api/campaign-types.ts +++ b/frontend/src/api/campaign-types.ts @@ -48,6 +48,7 @@ export interface CampaignResultsReadModel extends UIModelIdentity { export interface CampaignEntryReadModel extends UIModelIdentity { entry_id: string; candidate_id: string; + configuration_id: string; model_id: string; config_digest: string; status: CampaignEntryStatus; @@ -90,6 +91,7 @@ export interface CampaignCaseSummaryReadModel extends UIModelIdentity { export interface CampaignCaseCandidateReadModel extends UIModelIdentity { entry_id: string; candidate_id: string; + configuration_id: string; model_id: string; config_digest: string; entry_status: CampaignEntryStatus; From 8d93143ec8ee6b8628c3c0c7dc9a1e7ea1e318ee Mon Sep 17 00:00:00 2001 From: Daniele21 Date: Fri, 4 Sep 2026 16:24:52 +0200 Subject: [PATCH 28/35] VALUE-03B1: default fixed configuration projection --- src/performance_lab/application/campaign_models.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/performance_lab/application/campaign_models.py b/src/performance_lab/application/campaign_models.py index df7b0836..d6c5f0f1 100644 --- a/src/performance_lab/application/campaign_models.py +++ b/src/performance_lab/application/campaign_models.py @@ -47,7 +47,7 @@ class CampaignResultsReadModel(UIModel): class CampaignEntryReadModel(UIModel): entry_id: str = Field(min_length=1) candidate_id: str = Field(min_length=1) - configuration_id: str = Field(min_length=1) + configuration_id: str = Field(default="fixed-1", min_length=1) model_id: str = Field(min_length=1) config_digest: str = Field(pattern=r"^[0-9a-f]{64}$") status: CampaignEntryStatus @@ -90,7 +90,7 @@ class CampaignCaseSummaryReadModel(UIModel): class CampaignCaseCandidateReadModel(UIModel): entry_id: str = Field(min_length=1) candidate_id: str = Field(min_length=1) - configuration_id: str = Field(min_length=1) + configuration_id: str = Field(default="fixed-1", min_length=1) model_id: str = Field(min_length=1) config_digest: str = Field(pattern=r"^[0-9a-f]{64}$") entry_status: CampaignEntryStatus From 9c0620748c15804187e525a8384b3ac3b7d640ff Mon Sep 17 00:00:00 2001 From: Daniele21 Date: Fri, 4 Sep 2026 16:25:41 +0200 Subject: [PATCH 29/35] test: prove frozen generation reaches inference requests --- tests/test_orchestrator.py | 33 +++++++++++++++++++++++++++++++-- 1 file changed, 31 insertions(+), 2 deletions(-) diff --git a/tests/test_orchestrator.py b/tests/test_orchestrator.py index cd4a005f..546d4c84 100644 --- a/tests/test_orchestrator.py +++ b/tests/test_orchestrator.py @@ -94,13 +94,17 @@ def suite() -> EvaluationSuite: ) -def fingerprint(snapshot: DatasetSnapshot) -> ExecutionFingerprint: +def fingerprint( + snapshot: DatasetSnapshot, + *, + generation: GenerationConfig | None = None, +) -> ExecutionFingerprint: return ExecutionFingerprint( target_id="target", adapter_type="fake", endpoint_identity="fixture", model=ModelIdentity(model_id="model-a"), - generation=suite().generation, + generation=generation or suite().generation, prompt_template_version="chat-v1", dataset_snapshots=(snapshot,), evaluator_versions=(EvaluatorRef(evaluator_id="simple", version="1"),), @@ -144,6 +148,31 @@ def test_orchestrator_completes_and_emits_content_safe_progress() -> None: assert "ok" not in serialized_events +def test_orchestrator_executes_the_generation_frozen_in_the_fingerprint() -> None: + materialized = dataset((DatasetRecord(sample_id="1", input="hello", expected="ok"),)) + frozen_generation = GenerationConfig( + max_output_tokens=3, + temperature=0.7, + top_p=0.8, + seed=42, + ) + adapter = FakeInferenceAdapter(response_text="ok") + + result = asyncio.run( + EvaluationOrchestrator(adapter, {"simple": SimpleEvaluator()}).run( + run_id="run-frozen-generation", + fingerprint=fingerprint(materialized.snapshot, generation=frozen_generation), + suite=suite(), + datasets={"demo": materialized}, + ) + ) + + assert result.fingerprint.generation == frozen_generation + assert len(adapter.requests) == 1 + assert adapter.requests[0].generation == frozen_generation + assert adapter.requests[0].generation != suite().generation + + def test_partial_endpoint_failure_is_typed_without_aborting_run() -> None: materialized = dataset( ( From 963d8dfd373fab4ed9a1cba8715e66e74443d23a Mon Sep 17 00:00:00 2001 From: Daniele21 Date: Fri, 4 Sep 2026 16:26:17 +0200 Subject: [PATCH 30/35] test: cover frozen campaign generation configuration --- tests/test_campaign_planning.py | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/tests/test_campaign_planning.py b/tests/test_campaign_planning.py index 94aa9ece..2ece069e 100644 --- a/tests/test_campaign_planning.py +++ b/tests/test_campaign_planning.py @@ -112,6 +112,13 @@ def test_fixed_general_plan_is_frozen_deterministic_and_executable(tmp_path: Pat assert first.configuration_search is not None assert first.configuration_search.configuration_count_per_candidate == 1 assert first.configuration_search.bounded_parameter_ranges == () + assert len(first.configuration_search.configurations) == 1 + frozen = first.configuration_search.configurations[0] + assert frozen.configuration_id == "fixed-1" + assert len(frozen.generation_digest) == 64 + assert frozen.generation == first.configuration_search.base_generation + assert second.configuration_search is not None + assert second.configuration_search.configurations == first.configuration_search.configurations assert first.benchmark_plan is not None assert first.benchmark_plan.suite.suite_id == "general-diagnostic-starter" assert first.benchmark_plan.case_count_per_run == 23 @@ -124,6 +131,8 @@ def test_fixed_general_plan_is_frozen_deterministic_and_executable(tmp_path: Pat assert launch.plan_digest == first.plan_digest assert launch.decision_policy.policy_id == "strict-quality-dominance" assert len(launch.runs) == 1 + assert launch.runs[0].configuration_id == frozen.configuration_id + assert launch.runs[0].config.generation == frozen.generation assert launch.runs[0].config.suite_id == "general-diagnostic-starter" assert launch.runs[0].config.suite_version == first.benchmark_plan.suite.suite_version assert launch.runs[0].config.evidence_mode == EvidenceMode.AGGREGATE_SAFE @@ -156,6 +165,8 @@ def test_workload_use_case_maps_to_its_own_versioned_executable_plan(tmp_path: P assert preview.can_plan assert preview.use_case is not None assert preview.use_case.source == "workload_pack" + assert preview.configuration_search is not None + assert len(preview.configuration_search.configurations) == 1 assert preview.benchmark_plan is not None assert preview.benchmark_plan.suite.suite_id == "workload-structured-document-extraction" assert preview.benchmark_plan.case_count_per_run == 12 @@ -164,6 +175,8 @@ def test_workload_use_case_maps_to_its_own_versioned_executable_plan(tmp_path: P assert preview.plan_digest is not None launch = queries.prepare_campaign_launch(request, expected_plan_digest=preview.plan_digest) + assert launch.runs[0].configuration_id == "fixed-1" + assert launch.runs[0].config.generation == preview.configuration_search.configurations[0].generation assert launch.runs[0].config.suite_id == "workload-structured-document-extraction" assert launch.runs[0].config.suite_version == preview.benchmark_plan.suite.suite_version assert launch.runs[0].config.evidence_mode == EvidenceMode.AGGREGATE_SAFE @@ -256,15 +269,19 @@ def test_configured_target_discovery_plans_multiple_models_with_candidate_bound_ preview = queries.preview_campaign_plan(request) assert preview.can_plan + assert preview.configuration_search is not None + assert len(preview.configuration_search.configurations) == 1 assert preview.estimate is not None assert preview.estimate.planned_run_count == 2 assert preview.plan_digest is not None launch = queries.prepare_campaign_launch(request, expected_plan_digest=preview.plan_digest) assert [run.model_id for run in launch.runs] == ["model-a", "model-b"] + assert all(run.configuration_id == "fixed-1" for run in launch.runs) for run in launch.runs: config = run.config assert config.model_id == run.model_id + assert config.generation == preview.configuration_search.configurations[0].generation assert config.local_llm_server_identity is not None assert config.local_llm_server_identity.model_id == run.model_id assert config.local_llm_server_identity.required is True From 3743e246e1f74741ae24d9efa5248f8b369aa005 Mon Sep 17 00:00:00 2001 From: Daniele21 Date: Fri, 4 Sep 2026 16:26:35 +0200 Subject: [PATCH 31/35] test: cover candidate configuration identity --- tests/test_campaign_configuration_identity.py | 82 +++++++++++++++++++ 1 file changed, 82 insertions(+) create mode 100644 tests/test_campaign_configuration_identity.py diff --git a/tests/test_campaign_configuration_identity.py b/tests/test_campaign_configuration_identity.py new file mode 100644 index 00000000..e9b24758 --- /dev/null +++ b/tests/test_campaign_configuration_identity.py @@ -0,0 +1,82 @@ +from datetime import UTC, datetime + +import pytest +from pydantic import ValidationError + +from performance_lab.domain import Campaign, CampaignEntry, DecisionPolicyRef + + +def _campaign(entries: tuple[CampaignEntry, ...]) -> Campaign: + now = datetime.now(UTC) + return Campaign( + campaign_id="campaign-configurations", + plan_digest="a" * 64, + use_case_id="general-capability", + use_case_version="1", + target_id="target-a", + suite_id="general-diagnostic-starter", + suite_version="1", + decision_policy=DecisionPolicyRef( + policy_id="strict-quality-dominance", + policy_version="1.0.0", + ), + created_at=now, + updated_at=now, + entries=entries, + ) + + +def test_campaign_entry_defaults_to_the_fixed_configuration_identity() -> None: + entry = CampaignEntry( + entry_id="entry-a", + candidate_id="candidate-a", + model_id="model-a", + config_digest="b" * 64, + ) + + assert entry.configuration_id == "fixed-1" + + +def test_campaign_allows_one_candidate_with_multiple_configuration_identities() -> None: + campaign = _campaign( + ( + CampaignEntry( + entry_id="entry-a", + candidate_id="candidate-a", + configuration_id="config-1", + model_id="model-a", + config_digest="b" * 64, + ), + CampaignEntry( + entry_id="entry-b", + candidate_id="candidate-a", + configuration_id="config-2", + model_id="model-a", + config_digest="c" * 64, + ), + ) + ) + + assert [entry.configuration_id for entry in campaign.entries] == ["config-1", "config-2"] + + +def test_campaign_rejects_duplicate_candidate_configuration_pair() -> None: + with pytest.raises(ValidationError, match="candidate/configuration pairs must be unique"): + _campaign( + ( + CampaignEntry( + entry_id="entry-a", + candidate_id="candidate-a", + configuration_id="config-1", + model_id="model-a", + config_digest="b" * 64, + ), + CampaignEntry( + entry_id="entry-b", + candidate_id="candidate-a", + configuration_id="config-1", + model_id="model-a", + config_digest="c" * 64, + ), + ) + ) From 652679977ab27f2d3a515f21a29ec79209cffaf8 Mon Sep 17 00:00:00 2001 From: Daniele21 Date: Fri, 4 Sep 2026 16:28:52 +0200 Subject: [PATCH 32/35] style: format campaign configuration identity --- src/performance_lab/domain/campaign.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/src/performance_lab/domain/campaign.py b/src/performance_lab/domain/campaign.py index 5670ec18..179f4dfd 100644 --- a/src/performance_lab/domain/campaign.py +++ b/src/performance_lab/domain/campaign.py @@ -112,9 +112,7 @@ def require_unique_entries(cls, value: tuple[CampaignEntry, ...]) -> tuple[Campa if not value: raise ValueError("campaign requires at least one entry") entry_ids = [entry.entry_id for entry in value] - candidate_configurations = [ - (entry.candidate_id, entry.configuration_id) for entry in value - ] + candidate_configurations = [(entry.candidate_id, entry.configuration_id) for entry in value] if len(entry_ids) != len(set(entry_ids)): raise ValueError("campaign entry ids must be unique") if len(candidate_configurations) != len(set(candidate_configurations)): From afca2ba1dc7858049827d1c2671e850602f23cc4 Mon Sep 17 00:00:00 2001 From: Daniele21 Date: Fri, 4 Sep 2026 16:29:57 +0200 Subject: [PATCH 33/35] style: format frozen campaign configuration --- src/performance_lab/application/planning_queries.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/src/performance_lab/application/planning_queries.py b/src/performance_lab/application/planning_queries.py index c838d64c..ddb226dd 100644 --- a/src/performance_lab/application/planning_queries.py +++ b/src/performance_lab/application/planning_queries.py @@ -273,9 +273,7 @@ def preview_campaign_plan( else snapshot.sample_count ) - frozen_configurations = ( - _frozen_generation_configuration("fixed-1", suite.generation), - ) + frozen_configurations = (_frozen_generation_configuration("fixed-1", suite.generation),) configuration = ConfigurationSearchPlanReadModel( strategy=CampaignSearchStrategy.FIXED, title="Fixed benchmark configuration", From 4d3057425c5527693f291c2c2bc9dad0349bb9d4 Mon Sep 17 00:00:00 2001 From: Daniele21 Date: Fri, 4 Sep 2026 16:30:33 +0200 Subject: [PATCH 34/35] style: format frozen generation assertion --- tests/test_campaign_planning.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/tests/test_campaign_planning.py b/tests/test_campaign_planning.py index 2ece069e..ec687a7a 100644 --- a/tests/test_campaign_planning.py +++ b/tests/test_campaign_planning.py @@ -176,7 +176,10 @@ def test_workload_use_case_maps_to_its_own_versioned_executable_plan(tmp_path: P launch = queries.prepare_campaign_launch(request, expected_plan_digest=preview.plan_digest) assert launch.runs[0].configuration_id == "fixed-1" - assert launch.runs[0].config.generation == preview.configuration_search.configurations[0].generation + assert ( + launch.runs[0].config.generation + == preview.configuration_search.configurations[0].generation + ) assert launch.runs[0].config.suite_id == "workload-structured-document-extraction" assert launch.runs[0].config.suite_version == preview.benchmark_plan.suite.suite_version assert launch.runs[0].config.evidence_mode == EvidenceMode.AGGREGATE_SAFE From fe62d1c893609ba4d4d0ae2b70f67d98d88127cb Mon Sep 17 00:00:00 2001 From: Daniele21 Date: Fri, 4 Sep 2026 18:19:19 +0200 Subject: [PATCH 35/35] fix: propagate frozen generation through execution policy --- src/performance_lab/application/execution_policy_queries.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/performance_lab/application/execution_policy_queries.py b/src/performance_lab/application/execution_policy_queries.py index 55489dea..d5cd1c93 100644 --- a/src/performance_lab/application/execution_policy_queries.py +++ b/src/performance_lab/application/execution_policy_queries.py @@ -2,7 +2,7 @@ from __future__ import annotations -from performance_lab.domain import EndpointProfile, EvidenceMode +from performance_lab.domain import EndpointProfile, EvidenceMode, GenerationConfig from performance_lab.run_config import StarterRunConfig from .evidence_models import ( @@ -87,6 +87,7 @@ def _campaign_run_config( target: TargetSummaryReadModel, endpoint: EndpointProfile, model_id: str, + generation: GenerationConfig, suite_id: str, suite_version: str, ) -> StarterRunConfig: @@ -94,6 +95,7 @@ def _campaign_run_config( target=target, endpoint=endpoint, model_id=model_id, + generation=generation, suite_id=suite_id, suite_version=suite_version, )