Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions frontend/src/api/index.ts
Original file line number Diff line number Diff line change
Expand Up @@ -80,6 +80,7 @@ export type {
EvidenceContentState,
ExplanationState,
FrozenExecutionPreviewReadModel,
GenerationParameterDomainReadModel,
IdentityDifference,
IdentitySummary,
MetricDelta,
Expand Down
2 changes: 2 additions & 0 deletions frontend/src/api/planning-types.ts
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
import type {
DatasetSummaryReadModel,
GenerationParameterDomainReadModel,
SuiteSummaryReadModel,
TargetSummaryReadModel,
UIModelIdentity,
Expand Down Expand Up @@ -28,6 +29,7 @@ export interface CandidateModelReadModel extends UIModelIdentity {
runtime_name: string | null;
runtime_version: string | null;
runtime_config_digest: string | null;
generation_parameter_domains: GenerationParameterDomainReadModel[];
source: "configured" | "discovered";
}

Expand Down
13 changes: 13 additions & 0 deletions frontend/src/api/types.ts
Original file line number Diff line number Diff line change
Expand Up @@ -104,6 +104,18 @@ export interface CapabilitySupportReadModel extends UIModelIdentity {
detail: string | null;
}

export interface GenerationParameterDomainReadModel extends UIModelIdentity {
name: string;
kind: "float" | "integer" | "boolean";
scope: "request_generation";
source: "local_llm_server";
provenance: "registry_declared";
minimum: number | null;
maximum: number | null;
step: number | null;
values: boolean[];
}

export interface RuntimeParameterReadModel extends UIModelIdentity {
name: string;
scope: "runtime_load";
Expand All @@ -115,6 +127,7 @@ export interface RuntimeParameterReadModel extends UIModelIdentity {
export interface DiscoveredModelReadModel extends UIModelIdentity {
model_id: string;
runtime_parameters: RuntimeParameterReadModel[];
generation_parameter_domains: GenerationParameterDomainReadModel[];
}

export interface EndpointProbeReadModel extends UIModelIdentity {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -58,6 +58,7 @@ const context: CampaignPlanningContextReadModel = {
runtime_name: null,
runtime_version: null,
runtime_config_digest: null,
generation_parameter_domains: [],
source: "configured",
},
],
Expand Down
1 change: 1 addition & 0 deletions frontend/src/pages/test-model/TestModelPage.test.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -120,6 +120,7 @@ const probe: EndpointProbeReadModel = {
provenance: "local_llm_server",
},
],
generation_parameter_domains: [],
},
],
capabilities: [
Expand Down
2 changes: 2 additions & 0 deletions src/performance_lab/application/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -61,6 +61,7 @@
EvaluatorDefinitionReadModel,
EvidenceAvailability,
FrozenExecutionPreviewReadModel,
GenerationParameterDomainReadModel,
IdentitySummary,
MetricDimension,
MetricReadModel,
Expand Down Expand Up @@ -123,6 +124,7 @@
"EvidenceContentState",
"ExplanationState",
"FrozenExecutionPreviewReadModel",
"GenerationParameterDomainReadModel",
"IdentitySummary",
"MetricDimension",
"MetricReadModel",
Expand Down
55 changes: 46 additions & 9 deletions src/performance_lab/application/endpoint_discovery.py
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@
from typing import Literal

import httpx
from pydantic import HttpUrl
from pydantic import HttpUrl, ValidationError

from performance_lab.adapters import OpenAICompatibleAdapter
from performance_lab.domain import EndpointProfile
Expand All @@ -17,6 +17,7 @@
DiscoveredModelReadModel,
EndpointConnectionInput,
EndpointProbeReadModel,
GenerationParameterDomainReadModel,
RuntimeParameterReadModel,
)

Expand Down Expand Up @@ -66,18 +67,22 @@ async def probe_endpoint_profile(

capabilities = _capability_evidence(passive.capabilities, healthy=passive.healthy)
runtime_parameters: dict[str, tuple[RuntimeParameterReadModel, ...]] = {}
generation_domains: dict[str, tuple[GenerationParameterDomainReadModel, ...]] = {}
warning: str | None = None
if (
passive.healthy
and local_connection is not None
and local_connection.server_type == "local_llm_server"
):
runtime_parameters, warning = await _probe_local_llm_server_registry(local_connection)
runtime_parameters, generation_domains, warning = await _probe_local_llm_server_registry(
local_connection
)

models = tuple(
DiscoveredModelReadModel(
model_id=model_id,
runtime_parameters=runtime_parameters.get(model_id, ()),
generation_parameter_domains=generation_domains.get(model_id, ()),
)
for model_id in passive.models
)
Expand Down Expand Up @@ -131,7 +136,11 @@ def _capability_evidence(

async def _probe_local_llm_server_registry(
connection: EndpointConnectionInput,
) -> tuple[dict[str, tuple[RuntimeParameterReadModel, ...]], str | None]:
) -> tuple[
dict[str, tuple[RuntimeParameterReadModel, ...]],
dict[str, tuple[GenerationParameterDomainReadModel, ...]],
str | None,
]:
"""Best-effort first-party enrichment; generic OpenAI discovery stays authoritative."""
root = str(local_server_root(connection)).rstrip("/")
url = f"{root}/api/v1/models/registry"
Expand All @@ -142,17 +151,20 @@ async def _probe_local_llm_server_registry(
payload = response.json()
except (httpx.HTTPError, ValueError, TypeError):
return (
{},
{},
"Model discovery succeeded, but Local LLM Server runtime details are unavailable.",
)

if not isinstance(payload, Mapping):
return {}, "Local LLM Server returned an invalid runtime registry response."
return {}, {}, "Local LLM Server returned an invalid runtime registry response."
raw_models = payload.get("models")
if not isinstance(raw_models, list):
return {}, "Local LLM Server runtime registry did not include a model list."
return {}, {}, "Local LLM Server runtime registry did not include a model list."

result: dict[str, tuple[RuntimeParameterReadModel, ...]] = {}
runtime_result: dict[str, tuple[RuntimeParameterReadModel, ...]] = {}
domain_result: dict[str, tuple[GenerationParameterDomainReadModel, ...]] = {}
invalid_domain_metadata = False
for raw_model in raw_models:
if not isinstance(raw_model, Mapping):
continue
Expand All @@ -163,16 +175,41 @@ async def _probe_local_llm_server_registry(
runtime_config = raw_model.get("runtime_config")
config = runtime_config if isinstance(runtime_config, Mapping) else {}
names = raw_capabilities if isinstance(raw_capabilities, list) else []
parameters = tuple(
runtime_result[model_id] = tuple(
RuntimeParameterReadModel(
name=name,
current_value=config.get(name),
)
for name in names
if isinstance(name, str) and name
)
result[model_id] = parameters
return result, None

raw_domains = raw_model.get("generation_parameter_domains")
if raw_domains is None:
domain_result[model_id] = ()
continue
if not isinstance(raw_domains, list):
invalid_domain_metadata = True
domain_result[model_id] = ()
continue
domains: list[GenerationParameterDomainReadModel] = []
for raw_domain in raw_domains:
if not isinstance(raw_domain, Mapping):
invalid_domain_metadata = True
continue
try:
domains.append(GenerationParameterDomainReadModel.model_validate(dict(raw_domain)))
except ValidationError:
invalid_domain_metadata = True
domain_result[model_id] = tuple(sorted(domains, key=lambda item: item.name))

warning = (
"Local LLM Server returned invalid generation-domain metadata; "
"invalid domains were ignored."
if invalid_domain_metadata
else None
)
return runtime_result, domain_result, warning


def _model_id(raw_model: Mapping[object, object]) -> str | None:
Expand Down
2 changes: 2 additions & 0 deletions src/performance_lab/application/planning_models.py
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@

from .ui_models import (
DatasetSummaryReadModel,
GenerationParameterDomainReadModel,
SuiteSummaryReadModel,
TargetSummaryReadModel,
UIModel,
Expand Down Expand Up @@ -46,6 +47,7 @@ class CandidateModelReadModel(UIModel):
runtime_name: str | None = None
runtime_version: str | None = None
runtime_config_digest: str | None = None
generation_parameter_domains: tuple[GenerationParameterDomainReadModel, ...] = ()
source: Literal["configured", "discovered"]


Expand Down
4 changes: 4 additions & 0 deletions src/performance_lab/application/planning_queries.py
Original file line number Diff line number Diff line change
Expand Up @@ -41,6 +41,7 @@
DatasetSummaryReadModel,
DiscoveredModelReadModel,
EndpointConnectionInput,
GenerationParameterDomainReadModel,
SuiteSummaryReadModel,
TargetSummaryReadModel,
)
Expand Down Expand Up @@ -166,6 +167,7 @@ def campaign_planning_context(self) -> CampaignPlanningContextReadModel:
target.target_id,
model.model_id,
source="discovered",
generation_parameter_domains=model.generation_parameter_domains,
)
)
candidates = list({item.candidate_id: item for item in candidates}.values())
Expand Down Expand Up @@ -521,6 +523,7 @@ def _candidate(
model_id: str,
*,
source: Literal["configured", "discovered"],
generation_parameter_domains: tuple[GenerationParameterDomainReadModel, ...] = (),
) -> CandidateModelReadModel:
payload = json.dumps(
{"target_id": target_id, "model_id": model_id},
Expand All @@ -532,6 +535,7 @@ def _candidate(
candidate_id=candidate_id,
target_id=target_id,
model_id=model_id,
generation_parameter_domains=generation_parameter_domains,
source=source,
)

Expand Down
50 changes: 50 additions & 0 deletions src/performance_lab/application/ui_models.py
Original file line number Diff line number Diff line change
Expand Up @@ -159,6 +159,55 @@ class CapabilitySupportReadModel(UIModel):
detail: str | None = None


class GenerationParameterDomainReadModel(UIModel):
"""Validated projection of a backend-declared request-generation domain."""

name: str = Field(min_length=1)
kind: Literal["float", "integer", "boolean"]
scope: Literal["request_generation"] = "request_generation"
source: Literal["local_llm_server"] = "local_llm_server"
provenance: Literal["registry_declared"] = "registry_declared"
minimum: int | float | None = None
maximum: int | float | None = None
step: int | float | None = None
values: tuple[bool, ...] = ()

@model_validator(mode="after")
def validate_domain_shape(self) -> GenerationParameterDomainReadModel:
if self.kind == "boolean":
if self.minimum is not None or self.maximum is not None or self.step is not None:
raise ValueError("boolean generation domains cannot declare numeric bounds")
if len(self.values) != 2 or set(self.values) != {False, True}:
raise ValueError("boolean generation domains must contain false and true")
return self

if self.values:
raise ValueError("numeric generation domains cannot declare boolean values")
if self.minimum is None or self.maximum is None:
raise ValueError("numeric generation domains require minimum and maximum")
if isinstance(self.minimum, bool) or isinstance(self.maximum, bool):
raise ValueError("numeric generation domains require numeric bounds")
if self.minimum >= self.maximum:
raise ValueError("numeric generation domains require minimum < maximum")
if self.kind == "integer" and (
not isinstance(self.minimum, int)
or not isinstance(self.maximum, int)
or isinstance(self.minimum, bool)
or isinstance(self.maximum, bool)
):
raise ValueError("integer generation domains require integer bounds")
if self.step is not None:
if (
isinstance(self.step, bool)
or self.step <= 0
or self.step > self.maximum - self.minimum
):
raise ValueError("generation domain step must be positive and within the span")
if self.kind == "integer" and not isinstance(self.step, int):
raise ValueError("integer generation domains require an integer step")
return self


class RuntimeParameterReadModel(UIModel):
name: str = Field(min_length=1)
scope: Literal["runtime_load"] = "runtime_load"
Expand All @@ -170,6 +219,7 @@ class RuntimeParameterReadModel(UIModel):
class DiscoveredModelReadModel(UIModel):
model_id: str = Field(min_length=1)
runtime_parameters: tuple[RuntimeParameterReadModel, ...] = ()
generation_parameter_domains: tuple[GenerationParameterDomainReadModel, ...] = ()


class EndpointProbeReadModel(UIModel):
Expand Down
Loading
Loading