Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
35 commits
Select commit Hold shift + click to select a range
1cfd5ee
feat: carry declared domains into candidate planning
daniele21 Sep 3, 2026
eb568ac
feat: add declared generation domain read model
daniele21 Sep 3, 2026
7e2c4a8
feat: consume local llm generation domains
daniele21 Sep 3, 2026
69bb7ec
feat: preserve model scoped generation domains in planning
daniele21 Sep 3, 2026
025d808
feat: export generation domain read model
daniele21 Sep 3, 2026
c6e1f50
feat: project generation domains to typescript
daniele21 Sep 3, 2026
4304815
feat: expose candidate generation domains to frontend
daniele21 Sep 3, 2026
0c987e4
test: cover declared generation domain ingestion
daniele21 Sep 3, 2026
965e1ac
fix(value-03a): export generation domain read model
daniele21 Sep 3, 2026
34dcce8
fix(value-03a): update typed campaign fixture
daniele21 Sep 3, 2026
5d7ab94
fix(value-03a): update typed endpoint probe fixture
daniele21 Sep 3, 2026
ec09939
fix(value-03a): format generation domain validation
daniele21 Sep 3, 2026
85a61e8
fix(value-03a): format endpoint domain enrichment
daniele21 Sep 3, 2026
29ea284
fix(value-03a): format generation domain tests
daniele21 Sep 3, 2026
43ec8ed
fix(value-03a): align endpoint formatting to repo line length
daniele21 Sep 3, 2026
0ec1180
fix(value-03a): align tests to repo line length
daniele21 Sep 3, 2026
caed430
fix(value-03a): satisfy endpoint lint line length
daniele21 Sep 3, 2026
116cebd
VALUE-03B1: freeze generation in run config
daniele21 Sep 4, 2026
de4f283
VALUE-03B1: execute frozen fingerprint generation
daniele21 Sep 4, 2026
d5ac779
VALUE-03B1: bind runner to frozen generation
daniele21 Sep 4, 2026
97d9aef
VALUE-03B1: identify campaign configurations
daniele21 Sep 4, 2026
7806fd5
VALUE-03B1: carry configuration identity into campaigns
daniele21 Sep 4, 2026
1f91a35
VALUE-03B1: expose exact frozen generation configurations
daniele21 Sep 4, 2026
9e9d11a
VALUE-03B1: freeze exact campaign generation matrix
daniele21 Sep 4, 2026
8e20396
VALUE-03B1: project configuration identity
daniele21 Sep 4, 2026
dae65b7
VALUE-03B1: type frozen generation configurations
daniele21 Sep 4, 2026
eb3002c
VALUE-03B1: type campaign configuration identity
daniele21 Sep 4, 2026
8d93143
VALUE-03B1: default fixed configuration projection
daniele21 Sep 4, 2026
9c06207
test: prove frozen generation reaches inference requests
daniele21 Sep 4, 2026
963d8df
test: cover frozen campaign generation configuration
daniele21 Sep 4, 2026
3743e24
test: cover candidate configuration identity
daniele21 Sep 4, 2026
6526799
style: format campaign configuration identity
daniele21 Sep 4, 2026
afca2ba
style: format frozen campaign configuration
daniele21 Sep 4, 2026
4d30574
style: format frozen generation assertion
daniele21 Sep 4, 2026
fe62d1c
fix: propagate frozen generation through execution policy
daniele21 Sep 4, 2026
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions frontend/src/api/campaign-types.ts
Original file line number Diff line number Diff line change
Expand Up @@ -48,6 +48,7 @@ export interface CampaignResultsReadModel extends UIModelIdentity {
export interface CampaignEntryReadModel extends UIModelIdentity {
entry_id: string;
candidate_id: string;
configuration_id: string;
model_id: string;
config_digest: string;
status: CampaignEntryStatus;
Expand Down Expand Up @@ -90,6 +91,7 @@ export interface CampaignCaseSummaryReadModel extends UIModelIdentity {
export interface CampaignCaseCandidateReadModel extends UIModelIdentity {
entry_id: string;
candidate_id: string;
configuration_id: string;
model_id: string;
config_digest: string;
entry_status: CampaignEntryStatus;
Expand Down
1 change: 1 addition & 0 deletions frontend/src/api/index.ts
Original file line number Diff line number Diff line change
Expand Up @@ -80,6 +80,7 @@ export type {
EvidenceContentState,
ExplanationState,
FrozenExecutionPreviewReadModel,
GenerationParameterDomainReadModel,
IdentityDifference,
IdentitySummary,
MetricDelta,
Expand Down
9 changes: 9 additions & 0 deletions frontend/src/api/planning-types.ts
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
import type {
DatasetSummaryReadModel,
GenerationParameterDomainReadModel,
SuiteSummaryReadModel,
TargetSummaryReadModel,
UIModelIdentity,
Expand Down Expand Up @@ -28,6 +29,7 @@ export interface CandidateModelReadModel extends UIModelIdentity {
runtime_name: string | null;
runtime_version: string | null;
runtime_config_digest: string | null;
generation_parameter_domains: GenerationParameterDomainReadModel[];
source: "configured" | "discovered";
}

Expand Down Expand Up @@ -67,11 +69,18 @@ export interface CampaignPlanIssueReadModel extends UIModelIdentity {
field: string | null;
}

export interface FrozenGenerationConfigurationReadModel extends UIModelIdentity {
configuration_id: string;
generation_digest: string;
generation: Record<string, unknown>;
}

export interface ConfigurationSearchPlanReadModel extends UIModelIdentity {
strategy: CampaignSearchStrategy;
title: string;
configuration_count_per_candidate: number;
base_generation: Record<string, unknown>;
configurations: FrozenGenerationConfigurationReadModel[];
bounded_parameter_ranges: string[];
note: string;
}
Expand Down
13 changes: 13 additions & 0 deletions frontend/src/api/types.ts
Original file line number Diff line number Diff line change
Expand Up @@ -104,6 +104,18 @@ export interface CapabilitySupportReadModel extends UIModelIdentity {
detail: string | null;
}

export interface GenerationParameterDomainReadModel extends UIModelIdentity {
name: string;
kind: "float" | "integer" | "boolean";
scope: "request_generation";
source: "local_llm_server";
provenance: "registry_declared";
minimum: number | null;
maximum: number | null;
step: number | null;
values: boolean[];
}

export interface RuntimeParameterReadModel extends UIModelIdentity {
name: string;
scope: "runtime_load";
Expand All @@ -115,6 +127,7 @@ export interface RuntimeParameterReadModel extends UIModelIdentity {
export interface DiscoveredModelReadModel extends UIModelIdentity {
model_id: string;
runtime_parameters: RuntimeParameterReadModel[];
generation_parameter_domains: GenerationParameterDomainReadModel[];
}

export interface EndpointProbeReadModel extends UIModelIdentity {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -58,6 +58,7 @@ const context: CampaignPlanningContextReadModel = {
runtime_name: null,
runtime_version: null,
runtime_config_digest: null,
generation_parameter_domains: [],
source: "configured",
},
],
Expand Down
1 change: 1 addition & 0 deletions frontend/src/pages/test-model/TestModelPage.test.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -120,6 +120,7 @@ const probe: EndpointProbeReadModel = {
provenance: "local_llm_server",
},
],
generation_parameter_domains: [],
},
],
capabilities: [
Expand Down
2 changes: 2 additions & 0 deletions src/performance_lab/application/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -61,6 +61,7 @@
EvaluatorDefinitionReadModel,
EvidenceAvailability,
FrozenExecutionPreviewReadModel,
GenerationParameterDomainReadModel,
IdentitySummary,
MetricDimension,
MetricReadModel,
Expand Down Expand Up @@ -123,6 +124,7 @@
"EvidenceContentState",
"ExplanationState",
"FrozenExecutionPreviewReadModel",
"GenerationParameterDomainReadModel",
"IdentitySummary",
"MetricDimension",
"MetricReadModel",
Expand Down
3 changes: 3 additions & 0 deletions src/performance_lab/application/campaign_jobs.py
Original file line number Diff line number Diff line change
Expand Up @@ -47,6 +47,7 @@ class CampaignRunSpec:
candidate_id: str
model_id: str
config: StarterRunConfig
configuration_id: str = "fixed-1"


@dataclass(frozen=True, slots=True)
Expand Down Expand Up @@ -139,12 +140,14 @@ async def launch(self, plan: CampaignLaunchPlan) -> Campaign:
candidate_id=spec.candidate_id,
model_id=spec.model_id,
config=config,
configuration_id=spec.configuration_id,
)
)
entries.append(
CampaignEntry(
entry_id=f"entry-{index + 1}",
candidate_id=spec.candidate_id,
configuration_id=spec.configuration_id,
model_id=spec.model_id,
config_digest=starter_run_config_digest(config),
)
Expand Down
2 changes: 2 additions & 0 deletions src/performance_lab/application/campaign_models.py
Original file line number Diff line number Diff line change
Expand Up @@ -47,6 +47,7 @@ class CampaignResultsReadModel(UIModel):
class CampaignEntryReadModel(UIModel):
entry_id: str = Field(min_length=1)
candidate_id: str = Field(min_length=1)
configuration_id: str = Field(default="fixed-1", min_length=1)
model_id: str = Field(min_length=1)
config_digest: str = Field(pattern=r"^[0-9a-f]{64}$")
status: CampaignEntryStatus
Expand Down Expand Up @@ -89,6 +90,7 @@ class CampaignCaseSummaryReadModel(UIModel):
class CampaignCaseCandidateReadModel(UIModel):
entry_id: str = Field(min_length=1)
candidate_id: str = Field(min_length=1)
configuration_id: str = Field(default="fixed-1", min_length=1)
model_id: str = Field(min_length=1)
config_digest: str = Field(pattern=r"^[0-9a-f]{64}$")
entry_status: CampaignEntryStatus
Expand Down
55 changes: 46 additions & 9 deletions src/performance_lab/application/endpoint_discovery.py
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@
from typing import Literal

import httpx
from pydantic import HttpUrl
from pydantic import HttpUrl, ValidationError

from performance_lab.adapters import OpenAICompatibleAdapter
from performance_lab.domain import EndpointProfile
Expand All @@ -17,6 +17,7 @@
DiscoveredModelReadModel,
EndpointConnectionInput,
EndpointProbeReadModel,
GenerationParameterDomainReadModel,
RuntimeParameterReadModel,
)

Expand Down Expand Up @@ -66,18 +67,22 @@ async def probe_endpoint_profile(

capabilities = _capability_evidence(passive.capabilities, healthy=passive.healthy)
runtime_parameters: dict[str, tuple[RuntimeParameterReadModel, ...]] = {}
generation_domains: dict[str, tuple[GenerationParameterDomainReadModel, ...]] = {}
warning: str | None = None
if (
passive.healthy
and local_connection is not None
and local_connection.server_type == "local_llm_server"
):
runtime_parameters, warning = await _probe_local_llm_server_registry(local_connection)
runtime_parameters, generation_domains, warning = await _probe_local_llm_server_registry(
local_connection
)

models = tuple(
DiscoveredModelReadModel(
model_id=model_id,
runtime_parameters=runtime_parameters.get(model_id, ()),
generation_parameter_domains=generation_domains.get(model_id, ()),
)
for model_id in passive.models
)
Expand Down Expand Up @@ -131,7 +136,11 @@ def _capability_evidence(

async def _probe_local_llm_server_registry(
connection: EndpointConnectionInput,
) -> tuple[dict[str, tuple[RuntimeParameterReadModel, ...]], str | None]:
) -> tuple[
dict[str, tuple[RuntimeParameterReadModel, ...]],
dict[str, tuple[GenerationParameterDomainReadModel, ...]],
str | None,
]:
"""Best-effort first-party enrichment; generic OpenAI discovery stays authoritative."""
root = str(local_server_root(connection)).rstrip("/")
url = f"{root}/api/v1/models/registry"
Expand All @@ -142,17 +151,20 @@ async def _probe_local_llm_server_registry(
payload = response.json()
except (httpx.HTTPError, ValueError, TypeError):
return (
{},
{},
"Model discovery succeeded, but Local LLM Server runtime details are unavailable.",
)

if not isinstance(payload, Mapping):
return {}, "Local LLM Server returned an invalid runtime registry response."
return {}, {}, "Local LLM Server returned an invalid runtime registry response."
raw_models = payload.get("models")
if not isinstance(raw_models, list):
return {}, "Local LLM Server runtime registry did not include a model list."
return {}, {}, "Local LLM Server runtime registry did not include a model list."

result: dict[str, tuple[RuntimeParameterReadModel, ...]] = {}
runtime_result: dict[str, tuple[RuntimeParameterReadModel, ...]] = {}
domain_result: dict[str, tuple[GenerationParameterDomainReadModel, ...]] = {}
invalid_domain_metadata = False
for raw_model in raw_models:
if not isinstance(raw_model, Mapping):
continue
Expand All @@ -163,16 +175,41 @@ async def _probe_local_llm_server_registry(
runtime_config = raw_model.get("runtime_config")
config = runtime_config if isinstance(runtime_config, Mapping) else {}
names = raw_capabilities if isinstance(raw_capabilities, list) else []
parameters = tuple(
runtime_result[model_id] = tuple(
RuntimeParameterReadModel(
name=name,
current_value=config.get(name),
)
for name in names
if isinstance(name, str) and name
)
result[model_id] = parameters
return result, None

raw_domains = raw_model.get("generation_parameter_domains")
if raw_domains is None:
domain_result[model_id] = ()
continue
if not isinstance(raw_domains, list):
invalid_domain_metadata = True
domain_result[model_id] = ()
continue
domains: list[GenerationParameterDomainReadModel] = []
for raw_domain in raw_domains:
if not isinstance(raw_domain, Mapping):
invalid_domain_metadata = True
continue
try:
domains.append(GenerationParameterDomainReadModel.model_validate(dict(raw_domain)))
except ValidationError:
invalid_domain_metadata = True
domain_result[model_id] = tuple(sorted(domains, key=lambda item: item.name))

warning = (
"Local LLM Server returned invalid generation-domain metadata; "
"invalid domains were ignored."
if invalid_domain_metadata
else None
)
return runtime_result, domain_result, warning


def _model_id(raw_model: Mapping[object, object]) -> str | None:
Expand Down
4 changes: 3 additions & 1 deletion src/performance_lab/application/execution_policy_queries.py
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@

from __future__ import annotations

from performance_lab.domain import EndpointProfile, EvidenceMode
from performance_lab.domain import EndpointProfile, EvidenceMode, GenerationConfig
from performance_lab.run_config import StarterRunConfig

from .evidence_models import (
Expand Down Expand Up @@ -87,13 +87,15 @@ def _campaign_run_config(
target: TargetSummaryReadModel,
endpoint: EndpointProfile,
model_id: str,
generation: GenerationConfig,
suite_id: str,
suite_version: str,
) -> StarterRunConfig:
config = super()._campaign_run_config(
target=target,
endpoint=endpoint,
model_id=model_id,
generation=generation,
suite_id=suite_id,
suite_version=suite_version,
)
Expand Down
23 changes: 23 additions & 0 deletions src/performance_lab/application/planning_models.py
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@

from .ui_models import (
DatasetSummaryReadModel,
GenerationParameterDomainReadModel,
SuiteSummaryReadModel,
TargetSummaryReadModel,
UIModel,
Expand Down Expand Up @@ -46,6 +47,7 @@ class CandidateModelReadModel(UIModel):
runtime_name: str | None = None
runtime_version: str | None = None
runtime_config_digest: str | None = None
generation_parameter_domains: tuple[GenerationParameterDomainReadModel, ...] = ()
source: Literal["configured", "discovered"]


Expand Down Expand Up @@ -107,14 +109,35 @@ class CampaignPlanIssueReadModel(UIModel):
field: str | None = None


class FrozenGenerationConfigurationReadModel(UIModel):
"""One exact request-generation configuration reviewed before campaign launch."""

configuration_id: str = Field(min_length=1)
generation_digest: str = Field(pattern=r"^[0-9a-f]{64}$")
generation: GenerationConfig


class ConfigurationSearchPlanReadModel(UIModel):
strategy: CampaignSearchStrategy
title: str = Field(min_length=1)
configuration_count_per_candidate: int = Field(gt=0)
base_generation: GenerationConfig
configurations: tuple[FrozenGenerationConfigurationReadModel, ...]
bounded_parameter_ranges: tuple[str, ...] = ()
note: str = Field(min_length=1)

@model_validator(mode="after")
def require_exact_configuration_matrix(self) -> ConfigurationSearchPlanReadModel:
if len(self.configurations) != self.configuration_count_per_candidate:
raise ValueError("configuration count must match the frozen configuration matrix")
configuration_ids = [item.configuration_id for item in self.configurations]
if len(configuration_ids) != len(set(configuration_ids)):
raise ValueError("configuration ids must be unique")
generation_digests = [item.generation_digest for item in self.configurations]
if len(generation_digests) != len(set(generation_digests)):
raise ValueError("frozen generation configurations must be unique")
return self


class BenchmarkPlanReadModel(UIModel):
suite: SuiteSummaryReadModel
Expand Down
Loading
Loading