diff --git a/frontend/src/api/index.ts b/frontend/src/api/index.ts index b50b3f9a..d9fc7b8c 100644 --- a/frontend/src/api/index.ts +++ b/frontend/src/api/index.ts @@ -80,6 +80,7 @@ export type { EvidenceContentState, ExplanationState, FrozenExecutionPreviewReadModel, + GenerationParameterDomainReadModel, IdentityDifference, IdentitySummary, MetricDelta, diff --git a/frontend/src/api/planning-types.ts b/frontend/src/api/planning-types.ts index 46a22ba0..0768ac62 100644 --- a/frontend/src/api/planning-types.ts +++ b/frontend/src/api/planning-types.ts @@ -1,5 +1,6 @@ import type { DatasetSummaryReadModel, + GenerationParameterDomainReadModel, SuiteSummaryReadModel, TargetSummaryReadModel, UIModelIdentity, @@ -28,6 +29,7 @@ export interface CandidateModelReadModel extends UIModelIdentity { runtime_name: string | null; runtime_version: string | null; runtime_config_digest: string | null; + generation_parameter_domains: GenerationParameterDomainReadModel[]; source: "configured" | "discovered"; } diff --git a/frontend/src/api/types.ts b/frontend/src/api/types.ts index a575e658..f7ded817 100644 --- a/frontend/src/api/types.ts +++ b/frontend/src/api/types.ts @@ -104,6 +104,18 @@ export interface CapabilitySupportReadModel extends UIModelIdentity { detail: string | null; } +export interface GenerationParameterDomainReadModel extends UIModelIdentity { + name: string; + kind: "float" | "integer" | "boolean"; + scope: "request_generation"; + source: "local_llm_server"; + provenance: "registry_declared"; + minimum: number | null; + maximum: number | null; + step: number | null; + values: boolean[]; +} + export interface RuntimeParameterReadModel extends UIModelIdentity { name: string; scope: "runtime_load"; @@ -115,6 +127,7 @@ export interface RuntimeParameterReadModel extends UIModelIdentity { export interface DiscoveredModelReadModel extends UIModelIdentity { model_id: string; runtime_parameters: RuntimeParameterReadModel[]; + generation_parameter_domains: GenerationParameterDomainReadModel[]; } export interface EndpointProbeReadModel extends UIModelIdentity { diff --git a/frontend/src/pages/find-best-setup/FindBestSetupPage.test.tsx b/frontend/src/pages/find-best-setup/FindBestSetupPage.test.tsx index d59b9492..1a2bb9be 100644 --- a/frontend/src/pages/find-best-setup/FindBestSetupPage.test.tsx +++ b/frontend/src/pages/find-best-setup/FindBestSetupPage.test.tsx @@ -58,6 +58,7 @@ const context: CampaignPlanningContextReadModel = { runtime_name: null, runtime_version: null, runtime_config_digest: null, + generation_parameter_domains: [], source: "configured", }, ], diff --git a/frontend/src/pages/test-model/TestModelPage.test.tsx b/frontend/src/pages/test-model/TestModelPage.test.tsx index 04f5f3b7..e451fd93 100644 --- a/frontend/src/pages/test-model/TestModelPage.test.tsx +++ b/frontend/src/pages/test-model/TestModelPage.test.tsx @@ -120,6 +120,7 @@ const probe: EndpointProbeReadModel = { provenance: "local_llm_server", }, ], + generation_parameter_domains: [], }, ], capabilities: [ diff --git a/src/performance_lab/application/__init__.py b/src/performance_lab/application/__init__.py index 22d0fc78..79f7883c 100644 --- a/src/performance_lab/application/__init__.py +++ b/src/performance_lab/application/__init__.py @@ -61,6 +61,7 @@ EvaluatorDefinitionReadModel, EvidenceAvailability, FrozenExecutionPreviewReadModel, + GenerationParameterDomainReadModel, IdentitySummary, MetricDimension, MetricReadModel, @@ -123,6 +124,7 @@ "EvidenceContentState", "ExplanationState", "FrozenExecutionPreviewReadModel", + "GenerationParameterDomainReadModel", "IdentitySummary", "MetricDimension", "MetricReadModel", diff --git a/src/performance_lab/application/endpoint_discovery.py b/src/performance_lab/application/endpoint_discovery.py index a8c986f1..6275deda 100644 --- a/src/performance_lab/application/endpoint_discovery.py +++ b/src/performance_lab/application/endpoint_discovery.py @@ -6,7 +6,7 @@ from typing import Literal import httpx -from pydantic import HttpUrl +from pydantic import HttpUrl, ValidationError from performance_lab.adapters import OpenAICompatibleAdapter from performance_lab.domain import EndpointProfile @@ -17,6 +17,7 @@ DiscoveredModelReadModel, EndpointConnectionInput, EndpointProbeReadModel, + GenerationParameterDomainReadModel, RuntimeParameterReadModel, ) @@ -66,18 +67,22 @@ async def probe_endpoint_profile( capabilities = _capability_evidence(passive.capabilities, healthy=passive.healthy) runtime_parameters: dict[str, tuple[RuntimeParameterReadModel, ...]] = {} + generation_domains: dict[str, tuple[GenerationParameterDomainReadModel, ...]] = {} warning: str | None = None if ( passive.healthy and local_connection is not None and local_connection.server_type == "local_llm_server" ): - runtime_parameters, warning = await _probe_local_llm_server_registry(local_connection) + runtime_parameters, generation_domains, warning = await _probe_local_llm_server_registry( + local_connection + ) models = tuple( DiscoveredModelReadModel( model_id=model_id, runtime_parameters=runtime_parameters.get(model_id, ()), + generation_parameter_domains=generation_domains.get(model_id, ()), ) for model_id in passive.models ) @@ -131,7 +136,11 @@ def _capability_evidence( async def _probe_local_llm_server_registry( connection: EndpointConnectionInput, -) -> tuple[dict[str, tuple[RuntimeParameterReadModel, ...]], str | None]: +) -> tuple[ + dict[str, tuple[RuntimeParameterReadModel, ...]], + dict[str, tuple[GenerationParameterDomainReadModel, ...]], + str | None, +]: """Best-effort first-party enrichment; generic OpenAI discovery stays authoritative.""" root = str(local_server_root(connection)).rstrip("/") url = f"{root}/api/v1/models/registry" @@ -142,17 +151,20 @@ async def _probe_local_llm_server_registry( payload = response.json() except (httpx.HTTPError, ValueError, TypeError): return ( + {}, {}, "Model discovery succeeded, but Local LLM Server runtime details are unavailable.", ) if not isinstance(payload, Mapping): - return {}, "Local LLM Server returned an invalid runtime registry response." + return {}, {}, "Local LLM Server returned an invalid runtime registry response." raw_models = payload.get("models") if not isinstance(raw_models, list): - return {}, "Local LLM Server runtime registry did not include a model list." + return {}, {}, "Local LLM Server runtime registry did not include a model list." - result: dict[str, tuple[RuntimeParameterReadModel, ...]] = {} + runtime_result: dict[str, tuple[RuntimeParameterReadModel, ...]] = {} + domain_result: dict[str, tuple[GenerationParameterDomainReadModel, ...]] = {} + invalid_domain_metadata = False for raw_model in raw_models: if not isinstance(raw_model, Mapping): continue @@ -163,7 +175,7 @@ async def _probe_local_llm_server_registry( runtime_config = raw_model.get("runtime_config") config = runtime_config if isinstance(runtime_config, Mapping) else {} names = raw_capabilities if isinstance(raw_capabilities, list) else [] - parameters = tuple( + runtime_result[model_id] = tuple( RuntimeParameterReadModel( name=name, current_value=config.get(name), @@ -171,8 +183,33 @@ async def _probe_local_llm_server_registry( for name in names if isinstance(name, str) and name ) - result[model_id] = parameters - return result, None + + raw_domains = raw_model.get("generation_parameter_domains") + if raw_domains is None: + domain_result[model_id] = () + continue + if not isinstance(raw_domains, list): + invalid_domain_metadata = True + domain_result[model_id] = () + continue + domains: list[GenerationParameterDomainReadModel] = [] + for raw_domain in raw_domains: + if not isinstance(raw_domain, Mapping): + invalid_domain_metadata = True + continue + try: + domains.append(GenerationParameterDomainReadModel.model_validate(dict(raw_domain))) + except ValidationError: + invalid_domain_metadata = True + domain_result[model_id] = tuple(sorted(domains, key=lambda item: item.name)) + + warning = ( + "Local LLM Server returned invalid generation-domain metadata; " + "invalid domains were ignored." + if invalid_domain_metadata + else None + ) + return runtime_result, domain_result, warning def _model_id(raw_model: Mapping[object, object]) -> str | None: diff --git a/src/performance_lab/application/planning_models.py b/src/performance_lab/application/planning_models.py index 476e893d..6afaeac8 100644 --- a/src/performance_lab/application/planning_models.py +++ b/src/performance_lab/application/planning_models.py @@ -11,6 +11,7 @@ from .ui_models import ( DatasetSummaryReadModel, + GenerationParameterDomainReadModel, SuiteSummaryReadModel, TargetSummaryReadModel, UIModel, @@ -46,6 +47,7 @@ class CandidateModelReadModel(UIModel): runtime_name: str | None = None runtime_version: str | None = None runtime_config_digest: str | None = None + generation_parameter_domains: tuple[GenerationParameterDomainReadModel, ...] = () source: Literal["configured", "discovered"] diff --git a/src/performance_lab/application/planning_queries.py b/src/performance_lab/application/planning_queries.py index dff3b831..2205a922 100644 --- a/src/performance_lab/application/planning_queries.py +++ b/src/performance_lab/application/planning_queries.py @@ -41,6 +41,7 @@ DatasetSummaryReadModel, DiscoveredModelReadModel, EndpointConnectionInput, + GenerationParameterDomainReadModel, SuiteSummaryReadModel, TargetSummaryReadModel, ) @@ -166,6 +167,7 @@ def campaign_planning_context(self) -> CampaignPlanningContextReadModel: target.target_id, model.model_id, source="discovered", + generation_parameter_domains=model.generation_parameter_domains, ) ) candidates = list({item.candidate_id: item for item in candidates}.values()) @@ -521,6 +523,7 @@ def _candidate( model_id: str, *, source: Literal["configured", "discovered"], + generation_parameter_domains: tuple[GenerationParameterDomainReadModel, ...] = (), ) -> CandidateModelReadModel: payload = json.dumps( {"target_id": target_id, "model_id": model_id}, @@ -532,6 +535,7 @@ def _candidate( candidate_id=candidate_id, target_id=target_id, model_id=model_id, + generation_parameter_domains=generation_parameter_domains, source=source, ) diff --git a/src/performance_lab/application/ui_models.py b/src/performance_lab/application/ui_models.py index a4a7db55..033d90af 100644 --- a/src/performance_lab/application/ui_models.py +++ b/src/performance_lab/application/ui_models.py @@ -159,6 +159,55 @@ class CapabilitySupportReadModel(UIModel): detail: str | None = None +class GenerationParameterDomainReadModel(UIModel): + """Validated projection of a backend-declared request-generation domain.""" + + name: str = Field(min_length=1) + kind: Literal["float", "integer", "boolean"] + scope: Literal["request_generation"] = "request_generation" + source: Literal["local_llm_server"] = "local_llm_server" + provenance: Literal["registry_declared"] = "registry_declared" + minimum: int | float | None = None + maximum: int | float | None = None + step: int | float | None = None + values: tuple[bool, ...] = () + + @model_validator(mode="after") + def validate_domain_shape(self) -> GenerationParameterDomainReadModel: + if self.kind == "boolean": + if self.minimum is not None or self.maximum is not None or self.step is not None: + raise ValueError("boolean generation domains cannot declare numeric bounds") + if len(self.values) != 2 or set(self.values) != {False, True}: + raise ValueError("boolean generation domains must contain false and true") + return self + + if self.values: + raise ValueError("numeric generation domains cannot declare boolean values") + if self.minimum is None or self.maximum is None: + raise ValueError("numeric generation domains require minimum and maximum") + if isinstance(self.minimum, bool) or isinstance(self.maximum, bool): + raise ValueError("numeric generation domains require numeric bounds") + if self.minimum >= self.maximum: + raise ValueError("numeric generation domains require minimum < maximum") + if self.kind == "integer" and ( + not isinstance(self.minimum, int) + or not isinstance(self.maximum, int) + or isinstance(self.minimum, bool) + or isinstance(self.maximum, bool) + ): + raise ValueError("integer generation domains require integer bounds") + if self.step is not None: + if ( + isinstance(self.step, bool) + or self.step <= 0 + or self.step > self.maximum - self.minimum + ): + raise ValueError("generation domain step must be positive and within the span") + if self.kind == "integer" and not isinstance(self.step, int): + raise ValueError("integer generation domains require an integer step") + return self + + class RuntimeParameterReadModel(UIModel): name: str = Field(min_length=1) scope: Literal["runtime_load"] = "runtime_load" @@ -170,6 +219,7 @@ class RuntimeParameterReadModel(UIModel): class DiscoveredModelReadModel(UIModel): model_id: str = Field(min_length=1) runtime_parameters: tuple[RuntimeParameterReadModel, ...] = () + generation_parameter_domains: tuple[GenerationParameterDomainReadModel, ...] = () class EndpointProbeReadModel(UIModel): diff --git a/tests/test_generation_parameter_domains.py b/tests/test_generation_parameter_domains.py new file mode 100644 index 00000000..f6edde5b --- /dev/null +++ b/tests/test_generation_parameter_domains.py @@ -0,0 +1,181 @@ +from __future__ import annotations + +import asyncio + +from performance_lab.application import ( + CampaignSearchStrategy, + DiscoveredModelReadModel, + EndpointConnectionInput, + GenerationParameterDomainReadModel, + UIQueryService, +) +from performance_lab.application.endpoint_discovery import _probe_local_llm_server_registry +from performance_lab.storage import SQLiteRunStore + + +class _RegistryResponse: + def __init__(self, payload: object) -> None: + self._payload = payload + + def raise_for_status(self) -> None: + return None + + def json(self) -> object: + return self._payload + + +class _RegistryClient: + def __init__(self, payload: object) -> None: + self._payload = payload + + async def __aenter__(self) -> _RegistryClient: + return self + + async def __aexit__(self, *_args: object) -> None: + return None + + async def get(self, _url: str) -> _RegistryResponse: + return _RegistryResponse(self._payload) + + +def _patch_registry(monkeypatch, payload: object) -> None: + monkeypatch.setattr( + "performance_lab.application.endpoint_discovery.httpx.AsyncClient", + lambda **_kwargs: _RegistryClient(payload), + ) + + +def test_local_llm_registry_preserves_declared_model_scoped_domains(monkeypatch) -> None: + _patch_registry( + monkeypatch, + { + "models": [ + { + "key": "demo", + "model_id": "org/demo", + "config_capabilities": ["n_batch"], + "runtime_config": {"n_batch": 512}, + "generation_parameter_domains": [ + { + "name": "temperature", + "kind": "float", + "minimum": 0.0, + "maximum": 0.8, + "step": 0.1, + "provenance": "registry_declared", + }, + { + "name": "enable_thinking", + "kind": "boolean", + "values": [False, True], + "provenance": "registry_declared", + }, + ], + } + ] + }, + ) + + runtime_parameters, domains, warning = asyncio.run( + _probe_local_llm_server_registry( + EndpointConnectionInput( + base_url="http://127.0.0.1:1235/v1", + server_type="local_llm_server", + ) + ) + ) + + assert warning is None + assert runtime_parameters["org/demo"][0].scope == "runtime_load" + assert runtime_parameters["org/demo"][0].name == "n_batch" + assert [domain.name for domain in domains["org/demo"]] == [ + "enable_thinking", + "temperature", + ] + temperature = domains["org/demo"][1] + assert temperature.scope == "request_generation" + assert temperature.source == "local_llm_server" + assert temperature.provenance == "registry_declared" + assert temperature.minimum == 0.0 + assert temperature.maximum == 0.8 + assert temperature.step == 0.1 + + +def test_invalid_domain_metadata_is_ignored_without_losing_registry_model(monkeypatch) -> None: + _patch_registry( + monkeypatch, + { + "models": [ + { + "model_id": "org/demo", + "generation_parameter_domains": [ + { + "name": "temperature", + "kind": "float", + "minimum": 1.0, + "maximum": 0.0, + "provenance": "registry_declared", + } + ], + } + ] + }, + ) + + runtime_parameters, domains, warning = asyncio.run( + _probe_local_llm_server_registry( + EndpointConnectionInput( + base_url="http://127.0.0.1:1235/v1", + server_type="local_llm_server", + ) + ) + ) + + assert runtime_parameters["org/demo"] == () + assert domains["org/demo"] == () + assert warning is not None + assert "invalid generation-domain metadata" in warning + + +def test_planning_preserves_domains_but_does_not_enable_sweeps(tmp_path) -> None: + queries = UIQueryService(SQLiteRunStore(tmp_path / "runs.sqlite3")) + temperature = GenerationParameterDomainReadModel( + name="temperature", + kind="float", + minimum=0.0, + maximum=0.8, + step=0.1, + ) + target = queries.register_session_connection( + EndpointConnectionInput( + display_name="Declared-domain runtime", + base_url="http://127.0.0.1:1235/v1", + server_type="local_llm_server", + ), + discovered_models=( + DiscoveredModelReadModel( + model_id="org/demo", + generation_parameter_domains=(temperature,), + ), + ), + supported_generation_parameters=("temperature",), + ) + + target_context = next( + item + for item in queries.campaign_planning_context().targets + if item.target.target_id == target.target_id + ) + + assert len(target_context.candidates) == 1 + candidate = target_context.candidates[0] + assert candidate.model_id == "org/demo" + assert candidate.generation_parameter_domains == (temperature,) + assert target_context.bounded_generation_parameter_ranges == () + quick = next( + option + for option in target_context.configuration_search_options + if option.strategy == CampaignSearchStrategy.QUICK + ) + assert not quick.available + assert quick.blocked_reason is not None