From 0d552d6022486fe51aad1b287f91d9f7c95f1e0b Mon Sep 17 00:00:00 2001 From: divo12 Date: Wed, 2 Sep 2026 18:05:39 +0530 Subject: [PATCH 01/25] feat: record evidence-backed harness hypotheses --- .../openflywheel/.codex-plugin/plugin.json | 8 +- .../openflywheel/program_templates/base.md | 54 +- .../skills/hypothesis-former/SKILL.md | 22 + src/ofw/__init__.py | 30 + src/ofw/evaluation/failure_workspace.py | 93 ++-- src/ofw/evolution/__init__.py | 33 ++ src/ofw/evolution/hypothesis.py | 449 +++++++++++++++ src/ofw/evolution/hypothesis_repository.py | 139 +++++ src/ofw/mcp.py | 24 +- src/ofw/preparation/__init__.py | 10 + src/ofw/preparation/contracts.py | 11 +- src/ofw/preparation/harbor.py | 26 +- src/ofw/preparation/policy.py | 319 +++++++++++ src/ofw/preparation/service.py | 28 + src/ofw/preparation/templates/base.md | 54 +- src/ofw/safe_file.py | 223 ++++++++ tests/test_experiment_policy.py | 211 +++++++ tests/test_hypothesis.py | 522 ++++++++++++++++++ tests/test_openflywheel_mcp.py | 86 +++ tests/test_program_templates.py | 12 + tests/test_safe_file.py | 98 ++++ tests/test_typing.py | 16 + tests/test_workspace_preparation.py | 22 +- 23 files changed, 2342 insertions(+), 148 deletions(-) create mode 100644 plugins/openflywheel/skills/hypothesis-former/SKILL.md create mode 100644 src/ofw/evolution/__init__.py create mode 100644 src/ofw/evolution/hypothesis.py create mode 100644 src/ofw/evolution/hypothesis_repository.py create mode 100644 src/ofw/preparation/policy.py create mode 100644 src/ofw/safe_file.py create mode 100644 tests/test_experiment_policy.py create mode 100644 tests/test_hypothesis.py create mode 100644 tests/test_safe_file.py diff --git a/plugins/openflywheel/.codex-plugin/plugin.json b/plugins/openflywheel/.codex-plugin/plugin.json index 416f04a..635d45e 100644 --- a/plugins/openflywheel/.codex-plugin/plugin.json +++ b/plugins/openflywheel/.codex-plugin/plugin.json @@ -1,7 +1,7 @@ { "name": "openflywheel", "version": "0.7.1", - "description": "Initialize ITSM-bench workspaces, mine exact failure patterns, and record evidence-bound curations.", + "description": "Prepare ITSM-bench workspaces and record evidence-backed harness hypotheses.", "author": { "name": "OpenFlyWheel" }, @@ -10,14 +10,14 @@ "skills": "./skills/", "interface": { "displayName": "OpenFlyWheel", - "shortDescription": "Mine and curate ITSM failure patterns", - "longDescription": "Initialize an ITSM-bench agent-harness optimization workspace, inspect bounded Langfuse evidence, record authoritative outcomes and compact diagnoses, mine exact recurring patterns, and persist evidence-bound cross-task curations without copying trace payloads.", + "shortDescription": "Record evidence-backed harness hypotheses", + "longDescription": "Initialize an ITSM-bench agent-harness optimization workspace, inspect bounded Langfuse evidence, record authoritative outcomes and compact diagnoses, mine exact recurring patterns, and persist one evidence-backed hypothesis before candidate editing without copying trace payloads.", "developerName": "OpenFlyWheel", "category": "Productivity", "capabilities": ["Read", "Write"], "defaultPrompt": [ "Read this repository, identify the primary agent harness the user wants to evaluate with ITSM-bench, and use $workspace-init to collect its experiment configuration and perform the initial setup.", - "When PROGRAM.md is ready, read it and continue the optimization loop until its declared goal or stop condition is met." + "When PROGRAM.md is ready, read it, record one evidence-backed hypothesis, and stop before candidate editing." ] }, "mcpServers": "./.mcp.json" diff --git a/plugins/openflywheel/program_templates/base.md b/plugins/openflywheel/program_templates/base.md index 288034d..369dcdd 100644 --- a/plugins/openflywheel/program_templates/base.md +++ b/plugins/openflywheel/program_templates/base.md @@ -4,16 +4,18 @@ This file is generated by `prepare_workspace`. Do not edit it directly. ## Mission -Improve the connected agent harness until the goal in `experiment_config.yaml` is met, -while respecting its quality, cost, latency, budget, and stopping constraints. +Record one evidence-backed hypothesis for the connected agent harness under the canonical +experiment policy, then stop before candidate editing. The baseline has already been recorded. Begin at step 2; do not rerun the unchanged baseline. ## Authority -- `experiment_config.yaml` defines the harness, editable surface, frozen controls, goal, - benchmark, verifier, budget, and stopping conditions. +- The canonical experiment policy stored in the Git common control directory defines the + prepared branch and commits, exact editable surface, frozen controls, goal, benchmark, + verifier, budget, and stopping conditions. `experiment_config.yaml` is a human-readable copy, + not an internal authority interface. - The external verifier is authoritative for task outcome. The agent's completion claim is not proof of success. - Langfuse is the source of truth for trajectories, usage, cost, and latency. @@ -24,9 +26,9 @@ baseline. ## Editable and frozen surfaces -Edit only paths explicitly allowed by `experiment_config.yaml`. Never change the benchmark, -held-out tasks, verifier, model, reasoning budget, observability identity, or this program -to improve a score. +Target only exact paths allowed by the canonical experiment policy. Do not edit them in this +program. Never target the benchmark, held-out tasks, verifier, model, reasoning budget, +observability identity, or this program. Keep one focused hypothesis per iteration. Do not mix prompt, tool, middleware, and control flow changes unless the evidence requires the combination. @@ -46,37 +48,11 @@ State the failure pattern, supporting trace and verifier evidence, proposed harn expected improvement, and possible regressions. Stop if the evidence cannot distinguish between materially different changes. -### 4. Improve the harness +Use `$hypothesis-former` with only exact supported pattern and diagnosis receipt IDs, then call +`record_hypothesis`. Retain the stable hypothesis receipt and stop before candidate editing. +Candidate editing requires a later package and must not begin in this prepared program. -Make the smallest change within the declared editable surface. Preserve frozen controls and -unrelated user changes. +## Package boundary -### 5. Gate the change - -Run only the prepared experiment command and gates declared by the workspace. Compare -task-level verifier outcomes and report quality, cost, and latency separately. Missing or -errored trials remain visible and cannot disappear from the denominator. - -### 6. Keep or revert - -Keep the change only when the configured gate admits it. Otherwise revert only the current -iteration's harness edit, retain the evidence, and try a different hypothesis. Never weaken -the gate to admit a candidate. - -Commit each admitted improvement on the prepared `ofw/` branch before the -next iteration. Keep one hypothesis per commit and include `OFW-Experiment` and `OFW-Run` -trailers. Do not commit failed candidates, generated run artifacts, credentials, or changes -outside the editable surface. Do not push or open a pull request without explicit user -authorization. - -### 7. Repeat - -Return to step 2 with the newly recorded run. Stop when the configured goal is met, the -budget or iteration limit is exhausted, the no-improvement condition is reached, or required -authoritative evidence is unavailable. - -## Final report - -Report the accepted harness revision, verifier-backed quality, cost, latency, remaining -unverified trials, iteration count, and exact stopping reason. Do not claim improvement from -an agent-authored summary alone. +Report the hypothesis receipt and its exact evidence and target paths. Do not edit, commit, +run, gate, publish, or install a candidate; those capabilities are not part of this program. diff --git a/plugins/openflywheel/skills/hypothesis-former/SKILL.md b/plugins/openflywheel/skills/hypothesis-former/SKILL.md new file mode 100644 index 0000000..64b96f7 --- /dev/null +++ b/plugins/openflywheel/skills/hypothesis-former/SKILL.md @@ -0,0 +1,22 @@ +--- +name: hypothesis-former +description: Record one OpenFlywheel harness hypothesis from exact supported failure-pattern and diagnosis receipts in a prepared experiment. Use after exact pattern mining and before any candidate edit; do not use with inconclusive evidence or to modify harness files. +--- + +# Hypothesis Former + +Use only selected pattern IDs and their exact diagnosis artifact IDs from the same prepared +experiment. Keep the global diagnosis set at fifty IDs or fewer. Read the compact diagnoses, +then propose one bounded statement, rationale, expected effect, regression-risk list, component +taxonomy, and one or more exact target paths. + +Call `record_hypothesis` once with the prepared worktree, experiment ID, current initialization +commit, exact pattern-to-diagnosis assignments, and proposed target. The tool recomputes the +patterns and rejects missing, extra, misassigned, or inconclusive evidence. It also proves each +target is exactly present in the authoritative editable allowlist; `component_kind` describes the +change and never grants path access. + +Retain the returned hypothesis ID and artifact path. Stop before editing any harness file. On an +MCP timeout, retry the identical request once because publication is idempotent; if the retry +also times out, stop with unknown operation status. Do not query Langfuse, rewrite diagnoses, +broaden target paths, infer support, or begin candidate work while following this skill. diff --git a/src/ofw/__init__.py b/src/ofw/__init__.py index 3c780bb..077b3bb 100644 --- a/src/ofw/__init__.py +++ b/src/ofw/__init__.py @@ -44,6 +44,19 @@ VerifierResult, VerifierVerdict, ) +from ofw.evolution import ( + FailurePatternReference, + FailurePatternReferenceInput, + HarnessChangeTarget, + HarnessChangeTargetInput, + HarnessHypothesis, + HypothesisErrorCode, + HypothesisFailure, + HypothesisId, + HypothesisObservation, + HypothesisStatus, + RecordHypothesisInput, +) from ofw.observability.langfuse import ( CollectionError, CollectionErrorCode, @@ -52,6 +65,9 @@ ) from ofw.observability.langfuse.domain import TraceId from ofw.preparation import ( + ExperimentPolicyErrorCode, + ExperimentPolicyFailure, + ExperimentPolicySnapshot, PreparationErrorCode, PreparationPhase, PreparationStatus, @@ -65,6 +81,9 @@ "ComponentKind", "DeferredFailure", "EvidenceReference", + "ExperimentPolicyErrorCode", + "ExperimentPolicyFailure", + "ExperimentPolicySnapshot", "FailureCuration", "FailureCurationErrorCode", "FailureCurationFailure", @@ -80,8 +99,18 @@ "FailurePatternMiningStatus", "FailurePatternOrdering", "FailurePatternSummary", + "FailurePatternReference", + "FailurePatternReferenceInput", "FailureSource", "FailureType", + "HarnessChangeTarget", + "HarnessChangeTargetInput", + "HarnessHypothesis", + "HypothesisErrorCode", + "HypothesisFailure", + "HypothesisId", + "HypothesisObservation", + "HypothesisStatus", "Langfuse", "LangfuseOtelSpanAttributes", "LangfuseOutcomeStore", @@ -98,6 +127,7 @@ "PreparationPhase", "PreparationStatus", "PrepareWorkspaceInput", + "RecordHypothesisInput", "Sha256Digest", "TaskId", "TraceId", diff --git a/src/ofw/evaluation/failure_workspace.py b/src/ofw/evaluation/failure_workspace.py index 6669a45..92e4c62 100644 --- a/src/ofw/evaluation/failure_workspace.py +++ b/src/ofw/evaluation/failure_workspace.py @@ -5,7 +5,6 @@ import hashlib import os import re -import stat from collections.abc import Iterator from contextlib import contextmanager from dataclasses import dataclass @@ -13,7 +12,7 @@ from enum import StrEnum from pathlib import Path from typing import Annotated, Literal, Never, Protocol -from uuid import NAMESPACE_URL, uuid4, uuid5 +from uuid import NAMESPACE_URL, uuid5 from pydantic import BaseModel, ConfigDict, Field, ValidationError, field_validator, model_validator @@ -46,6 +45,12 @@ VerifierVerdict, ) from ofw.observability.langfuse.domain import ObservationId, ScoreId, TraceId +from ofw.safe_file import ( + SafeFileErrorCode, + SafeFileFailure, + publish_idempotent, + read_bounded, +) _IDENTIFIER_PATTERN = r"[A-Za-z0-9][A-Za-z0-9._:@/-]*" _ARTIFACT_ID_PATTERN = r"^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$" @@ -58,7 +63,6 @@ _WORKSPACE_MARKERS = ("PROGRAM.md", "experiment_config.yaml") _DIRECTORY_FLAGS = os.O_RDONLY | os.O_DIRECTORY | os.O_NOFOLLOW _CREATE_FILE_FLAGS = os.O_WRONLY | os.O_CREAT | os.O_EXCL | os.O_NOFOLLOW -_READ_FILE_FLAGS = os.O_RDONLY | os.O_NOFOLLOW | os.O_NONBLOCK Identifier = Annotated[ str, @@ -632,28 +636,25 @@ def _publish_or_validate( artifact_id: str, ) -> None: try: - _publish_new_file(directory, name, expected) - except FileExistsError: - _validate_existing(directory, name, expected, artifact_id) - - -def _publish_new_file(directory: int, name: str, content: bytes) -> None: - temporary_name = f".ofw-{uuid4().hex}.tmp" - published = False - try: - _write_new_file(directory, temporary_name, content) - os.link( - temporary_name, + publish_idempotent( + directory, name, - src_dir_fd=directory, - dst_dir_fd=directory, - follow_symlinks=False, + expected, + maximum_bytes=_ARTIFACT_LIMIT_BYTES, + subject=artifact_id, ) - published = True - finally: - _unlink_if_present(directory, temporary_name) - if published: - os.fsync(directory) + except SafeFileFailure as error: + if error.code is SafeFileErrorCode.CONFLICT: + raise FailureWorkspaceFailure( + FailureWorkspaceErrorCode.ARTIFACT_CONFLICT, + artifact_id, + ) from None + if error.code is SafeFileErrorCode.TOO_LARGE: + raise FailureWorkspaceFailure( + FailureWorkspaceErrorCode.ARTIFACT_TOO_LARGE, + artifact_id, + ) from None + raise OSError("unsafe failure artifact") from None def _write_new_file(directory: int, name: str, content: bytes) -> None: @@ -664,27 +665,6 @@ def _write_new_file(directory: int, name: str, content: bytes) -> None: os.fsync(stream.fileno()) -def _unlink_if_present(directory: int, name: str) -> None: - try: - os.unlink(name, dir_fd=directory) - except FileNotFoundError: - return - - -def _validate_existing( - directory: int, - name: str, - expected: bytes, - artifact_id: str, -) -> None: - actual = _read_existing(directory, name, artifact_id) - if actual != expected: - raise FailureWorkspaceFailure( - FailureWorkspaceErrorCode.ARTIFACT_CONFLICT, - artifact_id, - ) - - def _read_artifact(directory: int, artifact_id: str) -> FailureDiagnosisRecord: _require_artifact_id(artifact_id) try: @@ -735,21 +715,20 @@ def _pattern_read_error(error: FailureWorkspaceFailure) -> FailurePatternMiningE def _read_existing(directory: int, name: str, artifact_id: str) -> bytes: - descriptor = os.open(name, _READ_FILE_FLAGS, dir_fd=directory) try: - if not stat.S_ISREG(os.fstat(descriptor).st_mode): - raise OSError("failure artifact is not a regular file") - except OSError: - os.close(descriptor) - raise - with os.fdopen(descriptor, "rb") as stream: - content = stream.read(_ARTIFACT_LIMIT_BYTES + 1) - if len(content) > _ARTIFACT_LIMIT_BYTES: - raise FailureWorkspaceFailure( - FailureWorkspaceErrorCode.ARTIFACT_TOO_LARGE, - artifact_id, + return read_bounded( + directory, + name, + maximum_bytes=_ARTIFACT_LIMIT_BYTES, + subject=artifact_id, ) - return content + except SafeFileFailure as error: + if error.code is SafeFileErrorCode.TOO_LARGE: + raise FailureWorkspaceFailure( + FailureWorkspaceErrorCode.ARTIFACT_TOO_LARGE, + artifact_id, + ) from None + raise OSError("unsafe failure artifact") from None @contextmanager diff --git a/src/ofw/evolution/__init__.py b/src/ofw/evolution/__init__.py new file mode 100644 index 0000000..a05c3c2 --- /dev/null +++ b/src/ofw/evolution/__init__.py @@ -0,0 +1,33 @@ +"""Prepared-experiment evolution contracts.""" + +from ofw.evolution.hypothesis import ( + FailurePatternReference, + FailurePatternReferenceInput, + HarnessChangeTarget, + HarnessChangeTargetInput, + HarnessHypothesis, + HypothesisErrorCode, + HypothesisFailure, + HypothesisId, + HypothesisObservation, + HypothesisService, + HypothesisStatus, + RecordHypothesisInput, +) +from ofw.evolution.hypothesis_repository import FileHypothesisRepository + +__all__ = [ + "FailurePatternReference", + "FailurePatternReferenceInput", + "FileHypothesisRepository", + "HarnessChangeTarget", + "HarnessChangeTargetInput", + "HarnessHypothesis", + "HypothesisErrorCode", + "HypothesisFailure", + "HypothesisId", + "HypothesisObservation", + "HypothesisService", + "HypothesisStatus", + "RecordHypothesisInput", +] diff --git a/src/ofw/evolution/hypothesis.py b/src/ofw/evolution/hypothesis.py new file mode 100644 index 0000000..e95f457 --- /dev/null +++ b/src/ofw/evolution/hypothesis.py @@ -0,0 +1,449 @@ +"""Immutable evidence-backed hypothesis contract and recording service.""" + +from __future__ import annotations + +import hashlib +import re +from dataclasses import dataclass +from enum import StrEnum +from pathlib import Path +from typing import Annotated, Literal + +from pydantic import BaseModel, ConfigDict, Field, field_validator, model_validator + +from ofw.contracts import ComponentKind +from ofw.evaluation.failure_patterns import ( + FailurePatternMiningError, + FailurePatternMiningObservation, + FailurePatternMiningService, + MineFailurePatternsInput, +) +from ofw.preparation.policy import ExperimentPolicyErrorCode, ExperimentPolicyFailure + +_ARTIFACT_ID_PATTERN = r"[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}" +_PATTERN_ID_PATTERN = r"sha256:[0-9a-f]{64}" +_HYPOTHESIS_ID_PATTERN = re.compile(r"sha256:[0-9a-f]{64}") +_COMMIT_PATTERN = r"[0-9a-f]{40}" +_EXPERIMENT_PATTERN = r"[a-z0-9]+(?:-[a-z0-9]+)*" +HypothesisTextValue = Annotated[str, Field(min_length=1, max_length=4000)] +ArtifactIdsValue = Annotated[ + tuple[Annotated[str, Field(pattern=_ARTIFACT_ID_PATTERN)], ...], + Field(strict=False, min_length=1, max_length=50), +] +RelativePathsValue = Annotated[ + tuple[Annotated[Path, Field(strict=False)], ...], + Field(strict=False, min_length=1, max_length=50), +] + + +class StrictModel(BaseModel): + model_config = ConfigDict(extra="forbid", frozen=True, strict=True) + + +class FailurePatternReferenceInput(StrictModel): + pattern_id: str = Field(pattern=_PATTERN_ID_PATTERN) + diagnosis_artifact_ids: ArtifactIdsValue + + @field_validator("diagnosis_artifact_ids") + @classmethod + def validate_unique_artifacts(cls, values: tuple[str, ...]) -> tuple[str, ...]: + if len(set(values)) != len(values): + raise ValueError("diagnosis_artifact_ids must be unique") + return values + + +class HarnessChangeTargetInput(StrictModel): + component_kind: Annotated[ComponentKind, Field(strict=False)] + relative_paths: RelativePathsValue + + @field_validator("relative_paths") + @classmethod + def validate_paths(cls, values: tuple[Path, ...]) -> tuple[Path, ...]: + normalized = tuple(_relative_path(value) for value in values) + if len(set(normalized)) != len(normalized): + raise ValueError("relative_paths must be unique") + return normalized + + +class RecordHypothesisInput(StrictModel): + workspace_root: Path = Field(strict=False) + experiment_id: str = Field(min_length=1, max_length=80, pattern=_EXPERIMENT_PATTERN) + source_commit: str = Field(pattern=_COMMIT_PATTERN) + patterns: tuple[FailurePatternReferenceInput, ...] = Field( + strict=False, + min_length=1, + max_length=50, + ) + statement: HypothesisTextValue + rationale: HypothesisTextValue + target: HarnessChangeTargetInput + expected_effect: HypothesisTextValue + regression_risks: tuple[HypothesisTextValue, ...] = Field( + strict=False, + max_length=10, + ) + + @field_validator("workspace_root") + @classmethod + def validate_workspace_root(cls, value: Path) -> Path: + if not value.is_absolute(): + raise ValueError("workspace_root must be absolute") + return value + + @field_validator("statement", "rationale", "expected_effect") + @classmethod + def normalize_text(cls, value: str) -> str: + return _normalized_text(value) + + @field_validator("regression_risks") + @classmethod + def validate_risks(cls, values: tuple[str, ...]) -> tuple[str, ...]: + normalized = tuple(_normalized_text(value) for value in values) + if len(set(normalized)) != len(normalized): + raise ValueError("regression_risks must be unique") + return normalized + + @model_validator(mode="after") + def validate_evidence_bound(self) -> RecordHypothesisInput: + pattern_ids = tuple(pattern.pattern_id for pattern in self.patterns) + artifact_ids = tuple( + artifact_id + for pattern in self.patterns + for artifact_id in pattern.diagnosis_artifact_ids + ) + _require_unique_patterns(pattern_ids) + _require_bounded_unique_artifacts(artifact_ids) + return self + + +@dataclass(frozen=True, slots=True) +class HypothesisId: + value: str + + def __post_init__(self) -> None: + if _HYPOTHESIS_ID_PATTERN.fullmatch(self.value) is None: + raise ValueError("invalid hypothesis id") + + def __str__(self) -> str: + return self.value + + +@dataclass(frozen=True, slots=True) +class FailurePatternReference: + pattern_id: str + diagnosis_artifact_ids: tuple[str, ...] + + +@dataclass(frozen=True, slots=True) +class HarnessChangeTarget: + component_kind: ComponentKind + relative_paths: tuple[Path, ...] + + +@dataclass(frozen=True, slots=True) +class HarnessHypothesis: + id: HypothesisId + experiment_id: str + source_commit: str + patterns: tuple[FailurePatternReference, ...] + statement: str + rationale: str + target: HarnessChangeTarget + expected_effect: str + regression_risks: tuple[str, ...] + + +class _HypothesisContent(StrictModel): + schema_version: Literal[1] = 1 + experiment_id: str = Field(min_length=1, max_length=80, pattern=_EXPERIMENT_PATTERN) + source_commit: str = Field(pattern=_COMMIT_PATTERN) + patterns: tuple[FailurePatternReferenceInput, ...] = Field(min_length=1, max_length=50) + statement: HypothesisTextValue + rationale: HypothesisTextValue + target: HarnessChangeTargetInput + expected_effect: HypothesisTextValue + regression_risks: tuple[HypothesisTextValue, ...] = Field(max_length=10) + + +class HypothesisArtifact(_HypothesisContent): + hypothesis_id: str = Field(pattern=_PATTERN_ID_PATTERN) + + @classmethod + def from_hypothesis(cls, hypothesis: HarnessHypothesis) -> HypothesisArtifact: + return cls( + hypothesis_id=hypothesis.id.value, + experiment_id=hypothesis.experiment_id, + source_commit=hypothesis.source_commit, + patterns=tuple( + FailurePatternReferenceInput( + pattern_id=pattern.pattern_id, + diagnosis_artifact_ids=pattern.diagnosis_artifact_ids, + ) + for pattern in hypothesis.patterns + ), + statement=hypothesis.statement, + rationale=hypothesis.rationale, + target=HarnessChangeTargetInput( + component_kind=hypothesis.target.component_kind, + relative_paths=hypothesis.target.relative_paths, + ), + expected_effect=hypothesis.expected_effect, + regression_risks=hypothesis.regression_risks, + ) + + +class HypothesisStatus(StrEnum): + SUCCESS = "success" + + +class HypothesisObservation(StrictModel): + status: HypothesisStatus + summary: str = Field(min_length=1, max_length=256) + next_actions: tuple[str, ...] = Field(max_length=2) + artifacts: tuple[str, ...] = Field(min_length=2, max_length=2) + hypothesis_id: str = Field(pattern=_PATTERN_ID_PATTERN) + experiment_id: str = Field(min_length=1, max_length=80) + source_commit: str = Field(pattern=_COMMIT_PATTERN) + relative_path: Path + pattern_count: int = Field(strict=True, ge=1, le=50) + diagnosis_count: int = Field(strict=True, ge=1, le=50) + target_paths: RelativePathsValue + + +class HypothesisErrorCode(StrEnum): + POLICY_SNAPSHOT_REQUIRED = "policy_snapshot_required" + POLICY_INVALID = "policy_invalid" + STALE_POLICY = "stale_policy" + STALE_COMMIT = "stale_commit" + DIRTY_WORKSPACE = "dirty_workspace" + PATTERN_EVIDENCE_MISMATCH = "pattern_evidence_mismatch" + INCONCLUSIVE_EVIDENCE = "inconclusive_evidence" + TARGET_NOT_EDITABLE = "target_not_editable" + INVALID_TARGET = "invalid_target" + HYPOTHESIS_CONFLICT = "hypothesis_conflict" + HYPOTHESIS_TOO_LARGE = "hypothesis_too_large" + WRITE_FAILED = "write_failed" + + +class HypothesisFailure(Exception): + """Typed sanitized hypothesis rejection.""" + + __slots__ = ("code", "subject") + + def __init__(self, code: HypothesisErrorCode, subject: str) -> None: + self.code = code + self.subject = subject + super().__init__(f"{code.value}: {subject}") + + +@dataclass(frozen=True, slots=True) +class HypothesisService: + pattern_miner: FailurePatternMiningService + repository: FileHypothesisRepository + + def record(self, request: RecordHypothesisInput) -> HypothesisObservation: + policy = _load_policy(self.repository, request) + _validate_policy(request, policy) + paths = tuple(sorted(request.target.relative_paths, key=Path.as_posix)) + _validate_editable_paths(paths, policy.editable_paths) + self.repository.validate_workspace( + request.workspace_root, + policy, + request.source_commit, + paths, + ) + patterns = _canonical_patterns(request.patterns) + mined = _mine_patterns(self.pattern_miner, request.workspace_root, patterns) + _validate_mined_patterns(patterns, mined) + hypothesis = _hypothesis(request, patterns, paths) + relative_path = self.repository.store(request.workspace_root, hypothesis) + return _observation(hypothesis, relative_path) + + +def _load_policy( + repository: FileHypothesisRepository, + request: RecordHypothesisInput, +) -> ExperimentPolicySnapshot: + try: + return repository.load_policy(request.workspace_root, request.experiment_id) + except ExperimentPolicyFailure as error: + code = ( + HypothesisErrorCode.POLICY_SNAPSHOT_REQUIRED + if error.code is ExperimentPolicyErrorCode.POLICY_SNAPSHOT_REQUIRED + else HypothesisErrorCode.POLICY_INVALID + ) + raise HypothesisFailure(code, request.experiment_id) from None + + +def _validate_policy( + request: RecordHypothesisInput, + policy: ExperimentPolicySnapshot, +) -> None: + if request.source_commit != policy.initialization_commit: + raise HypothesisFailure(HypothesisErrorCode.STALE_COMMIT, request.experiment_id) + + +def _validate_editable_paths(paths: tuple[Path, ...], editable: tuple[Path, ...]) -> None: + for path in paths: + if path not in editable: + raise HypothesisFailure(HypothesisErrorCode.TARGET_NOT_EDITABLE, path.as_posix()) + + +def _canonical_patterns( + patterns: tuple[FailurePatternReferenceInput, ...], +) -> tuple[FailurePatternReference, ...]: + return tuple( + FailurePatternReference( + pattern.pattern_id, + tuple(sorted(pattern.diagnosis_artifact_ids)), + ) + for pattern in sorted(patterns, key=_pattern_reference_id) + ) + + +def _pattern_reference_id(pattern: FailurePatternReferenceInput) -> str: + return pattern.pattern_id + + +def _mine_patterns( + service: FailurePatternMiningService, + root: Path, + patterns: tuple[FailurePatternReference, ...], +) -> FailurePatternMiningObservation: + artifact_ids = tuple( + sorted( + artifact_id + for pattern in patterns + for artifact_id in pattern.diagnosis_artifact_ids + ) + ) + try: + request = MineFailurePatternsInput(workspace_root=root, artifact_ids=artifact_ids) + return service.mine(request) + except FailurePatternMiningError as error: + raise HypothesisFailure( + HypothesisErrorCode.PATTERN_EVIDENCE_MISMATCH, + error.subject, + ) from None + + +def _validate_mined_patterns( + declared: tuple[FailurePatternReference, ...], + mined: FailurePatternMiningObservation, +) -> None: + if mined.inconclusive_artifact_ids: + raise HypothesisFailure( + HypothesisErrorCode.INCONCLUSIVE_EVIDENCE, + mined.inconclusive_artifact_ids[0], + ) + if len(declared) != len(mined.patterns): + raise HypothesisFailure(HypothesisErrorCode.PATTERN_EVIDENCE_MISMATCH, "patterns") + for reference in declared: + _validate_pattern_reference(reference, mined) + + +def _validate_pattern_reference( + reference: FailurePatternReference, + mined: FailurePatternMiningObservation, +) -> None: + matching = tuple(item for item in mined.patterns if item.pattern_id == reference.pattern_id) + if len(matching) != 1 or matching[0].artifact_ids != reference.diagnosis_artifact_ids: + raise HypothesisFailure( + HypothesisErrorCode.PATTERN_EVIDENCE_MISMATCH, + reference.pattern_id, + ) + + +def _hypothesis( + request: RecordHypothesisInput, + patterns: tuple[FailurePatternReference, ...], + paths: tuple[Path, ...], +) -> HarnessHypothesis: + target = HarnessChangeTarget(request.target.component_kind, paths) + risks = tuple(sorted(request.regression_risks)) + content = _content(request, patterns, target, risks) + hypothesis_id = HypothesisId( + f"sha256:{hashlib.sha256(content.model_dump_json().encode('utf-8')).hexdigest()}" + ) + return HarnessHypothesis( + hypothesis_id, + request.experiment_id, + request.source_commit, + patterns, + request.statement, + request.rationale, + target, + request.expected_effect, + risks, + ) + + +def _content( + request: RecordHypothesisInput, + patterns: tuple[FailurePatternReference, ...], + target: HarnessChangeTarget, + risks: tuple[str, ...], +) -> _HypothesisContent: + return _HypothesisContent( + experiment_id=request.experiment_id, + source_commit=request.source_commit, + patterns=tuple( + FailurePatternReferenceInput( + pattern_id=pattern.pattern_id, + diagnosis_artifact_ids=pattern.diagnosis_artifact_ids, + ) + for pattern in patterns + ), + statement=request.statement, + rationale=request.rationale, + target=HarnessChangeTargetInput( + component_kind=target.component_kind, + relative_paths=target.relative_paths, + ), + expected_effect=request.expected_effect, + regression_risks=risks, + ) + + +def _observation(hypothesis: HarnessHypothesis, relative_path: Path) -> HypothesisObservation: + diagnosis_count = sum(len(pattern.diagnosis_artifact_ids) for pattern in hypothesis.patterns) + return HypothesisObservation( + status=HypothesisStatus.SUCCESS, + summary="Recorded one evidence-backed hypothesis for the prepared experiment.", + next_actions=("Stop before candidate editing and retain this hypothesis receipt.",), + artifacts=(str(relative_path), hypothesis.id.value), + hypothesis_id=hypothesis.id.value, + experiment_id=hypothesis.experiment_id, + source_commit=hypothesis.source_commit, + relative_path=relative_path, + pattern_count=len(hypothesis.patterns), + diagnosis_count=diagnosis_count, + target_paths=hypothesis.target.relative_paths, + ) + + +def _relative_path(path: Path) -> Path: + if path.is_absolute() or path == Path(".") or ".." in path.parts: + raise ValueError("relative_paths must be contained") + return path + + +def _require_unique_patterns(pattern_ids: tuple[str, ...]) -> None: + if len(set(pattern_ids)) != len(pattern_ids): + raise ValueError("pattern_ids must be unique") + + +def _require_bounded_unique_artifacts(artifact_ids: tuple[str, ...]) -> None: + if len(artifact_ids) > 50 or len(set(artifact_ids)) != len(artifact_ids): + raise ValueError("diagnosis artifacts must be globally unique and bounded") + + +def _normalized_text(value: str) -> str: + normalized = value.strip() + if not normalized: + raise ValueError("hypothesis text must not be blank") + return normalized + + +from ofw.evolution.hypothesis_repository import FileHypothesisRepository # noqa: E402 +from ofw.preparation.policy import ExperimentPolicySnapshot # noqa: E402 diff --git a/src/ofw/evolution/hypothesis_repository.py b/src/ofw/evolution/hypothesis_repository.py new file mode 100644 index 0000000..1b96f4d --- /dev/null +++ b/src/ofw/evolution/hypothesis_repository.py @@ -0,0 +1,139 @@ +"""Git and safe-file repository for prepared-experiment hypotheses.""" + +from __future__ import annotations + +import stat +import subprocess # nosec B404 +from pathlib import Path + +from ofw.evolution.hypothesis import ( + HarnessHypothesis, + HypothesisArtifact, + HypothesisErrorCode, + HypothesisFailure, +) +from ofw.preparation.policy import ( + ExperimentPolicySnapshot, + FileExperimentPolicyRepository, +) +from ofw.safe_file import ( + SafeFileErrorCode, + SafeFileFailure, + open_directory_chain, + publish_idempotent, +) + +_HYPOTHESIS_LIMIT_BYTES = 64 * 1024 +_WORKSPACE_MARKERS = ("PROGRAM.md", "experiment_config.yaml") + + +class FileHypothesisRepository: + """Validate one prepared Git worktree and store immutable hypothesis JSON.""" + + def load_policy(self, root: Path, experiment_id: str) -> ExperimentPolicySnapshot: + return FileExperimentPolicyRepository().load(root, experiment_id) + + def validate_workspace( + self, + root: Path, + policy: ExperimentPolicySnapshot, + source_commit: str, + paths: tuple[Path, ...], + ) -> None: + prepared_root = _prepared_root(root) + if _git(prepared_root, "rev-parse", "HEAD") != source_commit: + raise HypothesisFailure(HypothesisErrorCode.STALE_COMMIT, policy.experiment_id) + if _git(prepared_root, "branch", "--show-current") != policy.branch_name: + raise HypothesisFailure(HypothesisErrorCode.STALE_POLICY, policy.experiment_id) + if _git(prepared_root, "status", "--porcelain=v1", "--untracked-files=no"): + raise HypothesisFailure(HypothesisErrorCode.DIRTY_WORKSPACE, policy.experiment_id) + for path in paths: + _require_regular_target(prepared_root, path) + + def store(self, root: Path, hypothesis: HarnessHypothesis) -> Path: + prepared_root = _prepared_root(root) + artifact = HypothesisArtifact.from_hypothesis(hypothesis) + content = (artifact.model_dump_json(indent=2) + "\n").encode("utf-8") + try: + _ensure_workspace_ignore(prepared_root) + with open_directory_chain( + prepared_root, + (".workspace", "hypotheses"), + create=True, + ) as directory: + publish_idempotent( + directory, + f"{hypothesis.id.value}.json", + content, + maximum_bytes=_HYPOTHESIS_LIMIT_BYTES, + subject=hypothesis.id.value, + ) + except SafeFileFailure as error: + raise _storage_failure(error, hypothesis.id.value) from None + except OSError: + raise HypothesisFailure( + HypothesisErrorCode.WRITE_FAILED, + hypothesis.id.value, + ) from None + return Path(".workspace/hypotheses") / f"{hypothesis.id.value}.json" + + +def _prepared_root(root: Path) -> Path: + try: + resolved = root.resolve(strict=True) + except (OSError, RuntimeError): + raise HypothesisFailure(HypothesisErrorCode.STALE_POLICY, "workspace_root") from None + if not all(_regular_marker(resolved / name) for name in _WORKSPACE_MARKERS): + raise HypothesisFailure(HypothesisErrorCode.STALE_POLICY, "workspace_root") + return resolved + + +def _regular_marker(path: Path) -> bool: + try: + return stat.S_ISREG(path.lstat().st_mode) and not path.is_symlink() + except OSError: + return False + + +def _require_regular_target(root: Path, relative: Path) -> None: + path = root / relative + try: + metadata = path.lstat() + path.resolve(strict=True).relative_to(root) + except (OSError, ValueError): + raise HypothesisFailure(HypothesisErrorCode.INVALID_TARGET, relative.as_posix()) from None + if not stat.S_ISREG(metadata.st_mode) or path.is_symlink(): + raise HypothesisFailure(HypothesisErrorCode.INVALID_TARGET, relative.as_posix()) + + +def _ensure_workspace_ignore(root: Path) -> None: + with open_directory_chain(root, (".workspace",), create=True) as workspace: + publish_idempotent( + workspace, + ".gitignore", + b"*\n", + maximum_bytes=16, + subject=".workspace", + ) + + +def _git(root: Path, *arguments: str) -> str: + result = subprocess.run( + ("git", "-C", str(root), *arguments), + check=False, + capture_output=True, + text=True, + ) + if result.returncode != 0: + raise HypothesisFailure(HypothesisErrorCode.STALE_POLICY, arguments[0]) + return result.stdout.strip() + + +def _storage_failure(error: SafeFileFailure, hypothesis_id: str) -> HypothesisFailure: + if error.code is SafeFileErrorCode.CONFLICT: + code = HypothesisErrorCode.HYPOTHESIS_CONFLICT + elif error.code is SafeFileErrorCode.TOO_LARGE: + code = HypothesisErrorCode.HYPOTHESIS_TOO_LARGE + else: + code = HypothesisErrorCode.WRITE_FAILED + return HypothesisFailure(code, hypothesis_id) diff --git a/src/ofw/mcp.py b/src/ofw/mcp.py index cc50329..e684793 100644 --- a/src/ofw/mcp.py +++ b/src/ofw/mcp.py @@ -44,6 +44,12 @@ VerifierResult, VerifierVerdict, ) +from ofw.evolution import ( + FileHypothesisRepository, + HypothesisObservation, + HypothesisService, + RecordHypothesisInput, +) from ofw.observability.langfuse.contracts import LangfuseProject from ofw.observability.langfuse.domain import TraceId from ofw.observability.langfuse.trace_query import ( @@ -86,8 +92,8 @@ instructions=( "Prepare isolated ITSM harness workspaces, read bounded Langfuse trace evidence, and " "record authoritative outcomes, compact failure diagnoses, exact patterns, and " - "evidence-bound curations. Never infer outcomes, mutate traces, or copy trace payloads " - "into local storage." + "evidence-backed hypotheses. Never infer outcomes, mutate traces, copy trace payloads " + "into local storage, or edit candidates while recording a hypothesis." ), log_level="DEBUG", ) @@ -156,6 +162,14 @@ def _curation_service() -> FailureCurationService: return FailureCurationService(FileFailureCurationWorkspace()) +def _hypothesis_service() -> HypothesisService: + workspace = FileFailureWorkspace() + return HypothesisService( + pattern_miner=FailurePatternMiningService(workspace), + repository=FileHypothesisRepository(), + ) + + def _program_template(name: str) -> str: content = files("ofw.preparation.templates").joinpath(name).read_bytes() if len(content) > _PROGRAM_TEMPLATE_LIMIT_BYTES: @@ -297,6 +311,12 @@ def record_failure_curation( return _curation_service().record(request) +@server.tool(annotations=record_write, structured_output=True) +def record_hypothesis(request: RecordHypothesisInput) -> HypothesisObservation: + """Record one exact evidence-backed hypothesis, then stop before candidate editing.""" + return _hypothesis_service().record(request) + + def main() -> None: """Run the OpenFlywheel MCP server over stdio.""" server.run(transport="stdio") diff --git a/src/ofw/preparation/__init__.py b/src/ofw/preparation/__init__.py index 832cb6f..f2fd7b8 100644 --- a/src/ofw/preparation/__init__.py +++ b/src/ofw/preparation/__init__.py @@ -14,6 +14,12 @@ WorkspaceGateway, WorkspacePreparationObservation, ) +from ofw.preparation.policy import ( + ExperimentPolicyErrorCode, + ExperimentPolicyFailure, + ExperimentPolicySnapshot, + FileExperimentPolicyRepository, +) from ofw.preparation.service import WorkspacePreparationService __all__ = [ @@ -21,6 +27,10 @@ "BaselineRun", "BaselineRunner", "BaselineSummary", + "ExperimentPolicyErrorCode", + "ExperimentPolicyFailure", + "ExperimentPolicySnapshot", + "FileExperimentPolicyRepository", "PreparationErrorCode", "PreparationFailure", "PreparationPhase", diff --git a/src/ofw/preparation/contracts.py b/src/ofw/preparation/contracts.py index 17eaddf..cf046cf 100644 --- a/src/ofw/preparation/contracts.py +++ b/src/ofw/preparation/contracts.py @@ -133,6 +133,8 @@ class PreparationErrorCode(StrEnum): INVALID_BASELINE_RESULT = "invalid_baseline_result" PREPARATION_BUSY = "preparation_busy" GIT_FAILED = "git_failed" + POLICY_CONFLICT = "policy_conflict" + POLICY_WRITE_FAILED = "policy_write_failed" class WorkspacePreparationObservation(StrictModel): @@ -164,7 +166,14 @@ class WorkspacePreparationObservation(StrictModel): @dataclass(frozen=True, slots=True) class BaselineConfiguration: model: str - task_count: int + task_ids: tuple[str, ...] + benchmark_config_digest: str + verifier: str + environment: str + + @property + def task_count(self) -> int: + return len(self.task_ids) @dataclass(frozen=True, slots=True) diff --git a/src/ofw/preparation/harbor.py b/src/ofw/preparation/harbor.py index 416f425..776a660 100644 --- a/src/ofw/preparation/harbor.py +++ b/src/ofw/preparation/harbor.py @@ -2,6 +2,7 @@ from __future__ import annotations +import hashlib import os import subprocess # nosec B404 from dataclasses import dataclass @@ -35,7 +36,7 @@ class _HarborAgentWire(_WireModel): class _HarborTaskWire(_WireModel): - path: str + path: str = Field(min_length=1, max_length=256) class _HarborConfigWire(_WireModel): @@ -77,7 +78,8 @@ class HarborBaselineRunner: def validate(self, request: PrepareWorkspaceInput) -> BaselineConfiguration: _executable(request.harbor_executable) config_path = _contained(request.benchmark_root, request.harbor_config) - config = _parse_config(config_path) + config_content = _bounded_text(config_path, _MAX_CONFIG_BYTES) + config = _parse_config(config_path, config_content) if len(config.tasks) != request.expected_task_count: raise PreparationFailure( PreparationErrorCode.TASK_COUNT_MISMATCH, @@ -89,9 +91,23 @@ def validate(self, request: PrepareWorkspaceInput) -> BaselineConfiguration: PreparationErrorCode.INVALID_HARBOR_CONFIG, "agent", ) + task_ids = tuple(task.path for task in config.tasks) + if len(set(task_ids)) != len(task_ids): + raise PreparationFailure( + PreparationErrorCode.INVALID_HARBOR_CONFIG, + "tasks", + ) _validate_source_adapter(request.benchmark_root) _credentials() - return BaselineConfiguration(model=agent.model_name, task_count=len(config.tasks)) + return BaselineConfiguration( + model=agent.model_name, + task_ids=task_ids, + benchmark_config_digest=( + f"sha256:{hashlib.sha256(config_content.encode('utf-8')).hexdigest()}" + ), + verifier="itsm-bench", + environment="itsm-bench", + ) def start(self, run: BaselineRun) -> int: if run.job_path.exists(): @@ -191,9 +207,9 @@ def _contained(root: Path, relative: Path) -> Path: return resolved -def _parse_config(path: Path) -> _HarborConfigWire: +def _parse_config(path: Path, content: str) -> _HarborConfigWire: try: - return _HarborConfigWire.model_validate_json(_bounded_text(path, _MAX_CONFIG_BYTES)) + return _HarborConfigWire.model_validate_json(content) except (OSError, ValidationError, ValueError) as error: raise PreparationFailure( PreparationErrorCode.INVALID_HARBOR_CONFIG, diff --git a/src/ofw/preparation/policy.py b/src/ofw/preparation/policy.py new file mode 100644 index 0000000..ef9b6ac --- /dev/null +++ b/src/ofw/preparation/policy.py @@ -0,0 +1,319 @@ +"""Canonical prepared-experiment policy stored outside the worktree.""" + +from __future__ import annotations + +import hashlib +import re +import subprocess # nosec B404 +from contextlib import AbstractContextManager +from enum import StrEnum +from pathlib import Path +from typing import Annotated, Literal + +from pydantic import Field, ValidationError, field_validator, model_validator + +from ofw.preparation.contracts import ( + BaselineConfiguration, + EditablePaths, + ExperimentIdentifier, + GoalText, + IterationCount, + NormalizedScore, + PositiveMetric, + PreparedGitWorkspace, + PrepareWorkspaceInput, + StrictModel, +) +from ofw.safe_file import ( + SafeFileErrorCode, + SafeFileFailure, + open_directory_chain, + publish_idempotent, + read_bounded, +) + +_POLICY_LIMIT_BYTES = 64 * 1024 +_COMMIT_PATTERN = r"[0-9a-f]{40}" +_DIGEST_PATTERN = r"sha256:[0-9a-f]{64}" +_TASK_ID_PATTERN = r"[^\x00]+" +TaskIds = Annotated[ + tuple[Annotated[str, Field(min_length=1, max_length=256, pattern=_TASK_ID_PATTERN)], ...], + Field(min_length=1, max_length=500), +] + + +class _ExperimentPolicyContent(StrictModel): + schema_version: Literal[1] = 1 + experiment_id: ExperimentIdentifier + branch_name: str = Field(min_length=1, max_length=256) + base_commit: str = Field(pattern=_COMMIT_PATTERN) + initialization_commit: str = Field(pattern=_COMMIT_PATTERN) + editable_paths: EditablePaths + goal: GoalText + quality_target: NormalizedScore + max_iterations: IterationCount + no_improvement_limit: IterationCount + max_cost_per_task_usd: PositiveMetric | None = None + max_latency_seconds: PositiveMetric | None = None + max_baseline_seconds: int = Field(strict=True, ge=60, le=172800) + benchmark: Literal["itsm-bench"] = "itsm-bench" + benchmark_config_digest: str = Field(pattern=_DIGEST_PATTERN) + task_ids: TaskIds + model: str = Field(min_length=1, max_length=256) + verifier: str = Field(min_length=1, max_length=128) + environment: str = Field(min_length=1, max_length=128) + concurrency: Literal[1] = 1 + max_retries: Literal[0] = 0 + + @field_validator("task_ids") + @classmethod + def validate_task_ids(cls, values: tuple[str, ...]) -> tuple[str, ...]: + if len(set(values)) != len(values): + raise ValueError("task_ids must be unique") + return values + + @field_validator("editable_paths") + @classmethod + def validate_editable_paths(cls, values: tuple[Path, ...]) -> tuple[Path, ...]: + for path in values: + _require_contained_editable_path(path) + _require_unique_editable_paths(values) + return values + + +class ExperimentPolicySnapshot(_ExperimentPolicyContent): + """Immutable authority captured from validated preparation inputs and results.""" + + controls_digest: str = Field(pattern=_DIGEST_PATTERN) + + @model_validator(mode="after") + def validate_controls_digest(self) -> ExperimentPolicySnapshot: + if self.controls_digest != self.recomputed_controls_digest(): + raise ValueError("controls_digest does not match canonical policy") + return self + + def recomputed_controls_digest(self) -> str: + content = _content_from_snapshot(self) + return _digest(content.model_dump_json()) + + +class ExperimentPolicyErrorCode(StrEnum): + POLICY_SNAPSHOT_REQUIRED = "policy_snapshot_required" + POLICY_INVALID = "policy_invalid" + POLICY_CONFLICT = "policy_conflict" + POLICY_TOO_LARGE = "policy_too_large" + POLICY_WRITE_FAILED = "policy_write_failed" + + +class ExperimentPolicyFailure(Exception): + """Typed sanitized policy persistence failure.""" + + __slots__ = ("code", "subject") + + def __init__(self, code: ExperimentPolicyErrorCode, subject: str) -> None: + self.code = code + self.subject = subject + super().__init__(f"{code.value}: {subject}") + + +class FileExperimentPolicyRepository: + """Publish and reload one canonical policy in the Git common control directory.""" + + def publish(self, control_directory: Path, policy: ExperimentPolicySnapshot) -> Path: + content = (policy.model_dump_json(indent=2) + "\n").encode("utf-8") + try: + with _open_control_directory(control_directory) as directory: + publish_idempotent( + directory, + "policy.json", + content, + maximum_bytes=_POLICY_LIMIT_BYTES, + subject=policy.experiment_id, + ) + except SafeFileFailure as error: + raise _publication_failure(error, policy.experiment_id) from None + except OSError: + raise ExperimentPolicyFailure( + ExperimentPolicyErrorCode.POLICY_WRITE_FAILED, + policy.experiment_id, + ) from None + return control_directory / "policy.json" + + def load(self, workspace_root: Path, experiment_id: str) -> ExperimentPolicySnapshot: + _require_experiment_id(experiment_id) + control_directory = _control_directory(workspace_root, experiment_id) + content = _read_policy_bytes(control_directory, experiment_id) + try: + policy = ExperimentPolicySnapshot.model_validate_json(content) + except (ValidationError, ValueError, UnicodeError): + raise ExperimentPolicyFailure( + ExperimentPolicyErrorCode.POLICY_INVALID, + experiment_id, + ) from None + if policy.experiment_id != experiment_id: + raise ExperimentPolicyFailure( + ExperimentPolicyErrorCode.POLICY_INVALID, + experiment_id, + ) + return policy + + +def build_experiment_policy( + request: PrepareWorkspaceInput, + prepared: PreparedGitWorkspace, + baseline: BaselineConfiguration, +) -> ExperimentPolicySnapshot: + """Derive the authoritative snapshot solely from validated preparation values.""" + content = _ExperimentPolicyContent( + experiment_id=request.experiment_id, + branch_name=prepared.branch_name, + base_commit=prepared.base_commit, + initialization_commit=prepared.initialization_commit, + editable_paths=request.editable_paths, + goal=request.goal, + quality_target=request.quality_target, + max_iterations=request.max_iterations, + no_improvement_limit=request.no_improvement_limit, + max_cost_per_task_usd=request.max_cost_per_task_usd, + max_latency_seconds=request.max_latency_seconds, + max_baseline_seconds=request.max_baseline_seconds, + benchmark_config_digest=baseline.benchmark_config_digest, + task_ids=baseline.task_ids, + model=baseline.model, + verifier=baseline.verifier, + environment=baseline.environment, + ) + return _snapshot_from_content(content) + + +def _content_from_snapshot(policy: ExperimentPolicySnapshot) -> _ExperimentPolicyContent: + return _ExperimentPolicyContent( + experiment_id=policy.experiment_id, + branch_name=policy.branch_name, + base_commit=policy.base_commit, + initialization_commit=policy.initialization_commit, + editable_paths=policy.editable_paths, + goal=policy.goal, + quality_target=policy.quality_target, + max_iterations=policy.max_iterations, + no_improvement_limit=policy.no_improvement_limit, + max_cost_per_task_usd=policy.max_cost_per_task_usd, + max_latency_seconds=policy.max_latency_seconds, + max_baseline_seconds=policy.max_baseline_seconds, + benchmark_config_digest=policy.benchmark_config_digest, + task_ids=policy.task_ids, + model=policy.model, + verifier=policy.verifier, + environment=policy.environment, + ) + + +def _snapshot_from_content(content: _ExperimentPolicyContent) -> ExperimentPolicySnapshot: + return ExperimentPolicySnapshot( + experiment_id=content.experiment_id, + branch_name=content.branch_name, + base_commit=content.base_commit, + initialization_commit=content.initialization_commit, + editable_paths=content.editable_paths, + goal=content.goal, + quality_target=content.quality_target, + max_iterations=content.max_iterations, + no_improvement_limit=content.no_improvement_limit, + max_cost_per_task_usd=content.max_cost_per_task_usd, + max_latency_seconds=content.max_latency_seconds, + max_baseline_seconds=content.max_baseline_seconds, + benchmark_config_digest=content.benchmark_config_digest, + task_ids=content.task_ids, + model=content.model, + verifier=content.verifier, + environment=content.environment, + controls_digest=_digest(content.model_dump_json()), + ) + + +def _control_directory(workspace_root: Path, experiment_id: str) -> Path: + result = subprocess.run( + ("git", "-C", str(workspace_root), "rev-parse", "--git-common-dir"), + check=False, + capture_output=True, + text=True, + ) + if result.returncode != 0: + raise ExperimentPolicyFailure(ExperimentPolicyErrorCode.POLICY_INVALID, experiment_id) + common = Path(result.stdout.strip()) + if not common.is_absolute(): + common = workspace_root / common + return common / "ofw" / "preparations" / experiment_id + + +def _open_control_directory(control_directory: Path) -> AbstractContextManager[int]: + common = control_directory.parents[2] + return open_directory_chain( + common, + ("ofw", "preparations", control_directory.name), + create=False, + ) + + +def _read_policy_bytes(control_directory: Path, experiment_id: str) -> bytes: + try: + with _open_control_directory(control_directory) as directory: + return read_bounded( + directory, + "policy.json", + maximum_bytes=_POLICY_LIMIT_BYTES, + subject=experiment_id, + ) + except FileNotFoundError: + raise ExperimentPolicyFailure( + ExperimentPolicyErrorCode.POLICY_SNAPSHOT_REQUIRED, + experiment_id, + ) from None + except SafeFileFailure as error: + raise _read_failure(error, experiment_id) from None + except OSError: + raise ExperimentPolicyFailure( + ExperimentPolicyErrorCode.POLICY_INVALID, + experiment_id, + ) from None + + +def _require_experiment_id(experiment_id: str) -> None: + if re.fullmatch(r"[a-z0-9]+(?:-[a-z0-9]+)*", experiment_id) is None: + raise ExperimentPolicyFailure(ExperimentPolicyErrorCode.POLICY_INVALID, "experiment_id") + + +def _publication_failure( + error: SafeFileFailure, + experiment_id: str, +) -> ExperimentPolicyFailure: + if error.code is SafeFileErrorCode.CONFLICT: + code = ExperimentPolicyErrorCode.POLICY_CONFLICT + elif error.code is SafeFileErrorCode.TOO_LARGE: + code = ExperimentPolicyErrorCode.POLICY_TOO_LARGE + else: + code = ExperimentPolicyErrorCode.POLICY_WRITE_FAILED + return ExperimentPolicyFailure(code, experiment_id) + + +def _read_failure(error: SafeFileFailure, experiment_id: str) -> ExperimentPolicyFailure: + code = ( + ExperimentPolicyErrorCode.POLICY_TOO_LARGE + if error.code is SafeFileErrorCode.TOO_LARGE + else ExperimentPolicyErrorCode.POLICY_INVALID + ) + return ExperimentPolicyFailure(code, experiment_id) + + +def _digest(value: str) -> str: + return f"sha256:{hashlib.sha256(value.encode('utf-8')).hexdigest()}" + + +def _require_contained_editable_path(path: Path) -> None: + if path.is_absolute() or path == Path(".") or ".." in path.parts: + raise ValueError("editable_paths must be contained relative paths") + + +def _require_unique_editable_paths(paths: tuple[Path, ...]) -> None: + if len(set(paths)) != len(paths): + raise ValueError("editable_paths must be unique") diff --git a/src/ofw/preparation/service.py b/src/ofw/preparation/service.py index 3705004..8863ab3 100644 --- a/src/ofw/preparation/service.py +++ b/src/ofw/preparation/service.py @@ -14,6 +14,7 @@ from pydantic import Field from ofw.preparation.contracts import ( + BaselineConfiguration, BaselineRun, BaselineRunner, BaselineSummary, @@ -27,6 +28,12 @@ WorkspaceGateway, WorkspacePreparationObservation, ) +from ofw.preparation.policy import ( + ExperimentPolicyErrorCode, + ExperimentPolicyFailure, + FileExperimentPolicyRepository, + build_experiment_policy, +) _COMMIT_PATTERN = r"[0-9a-f]{40}" @@ -112,6 +119,7 @@ def _start( self._program, configuration, ) + _publish_policy(state_directory, request, prepared, configuration) job_path = request.benchmark_root / "jobs" / request.experiment_id log_path = state_directory / "baseline.log" run = _baseline_run(request, prepared, job_path, log_path) @@ -187,6 +195,26 @@ def _directory(path: Path, field: str) -> Path: return resolved +def _publish_policy( + state_directory: Path, + request: PrepareWorkspaceInput, + prepared: PreparedGitWorkspace, + configuration: BaselineConfiguration, +) -> None: + try: + FileExperimentPolicyRepository().publish( + state_directory, + build_experiment_policy(request, prepared, configuration), + ) + except ExperimentPolicyFailure as error: + code = ( + PreparationErrorCode.POLICY_CONFLICT + if error.code is ExperimentPolicyErrorCode.POLICY_CONFLICT + else PreparationErrorCode.POLICY_WRITE_FAILED + ) + raise PreparationFailure(code, request.experiment_id) from None + + def _compose_program(base: str, itsm: str) -> str: if not base.strip() or not itsm.strip(): raise ValueError("program templates must not be empty") diff --git a/src/ofw/preparation/templates/base.md b/src/ofw/preparation/templates/base.md index 288034d..369dcdd 100644 --- a/src/ofw/preparation/templates/base.md +++ b/src/ofw/preparation/templates/base.md @@ -4,16 +4,18 @@ This file is generated by `prepare_workspace`. Do not edit it directly. ## Mission -Improve the connected agent harness until the goal in `experiment_config.yaml` is met, -while respecting its quality, cost, latency, budget, and stopping constraints. +Record one evidence-backed hypothesis for the connected agent harness under the canonical +experiment policy, then stop before candidate editing. The baseline has already been recorded. Begin at step 2; do not rerun the unchanged baseline. ## Authority -- `experiment_config.yaml` defines the harness, editable surface, frozen controls, goal, - benchmark, verifier, budget, and stopping conditions. +- The canonical experiment policy stored in the Git common control directory defines the + prepared branch and commits, exact editable surface, frozen controls, goal, benchmark, + verifier, budget, and stopping conditions. `experiment_config.yaml` is a human-readable copy, + not an internal authority interface. - The external verifier is authoritative for task outcome. The agent's completion claim is not proof of success. - Langfuse is the source of truth for trajectories, usage, cost, and latency. @@ -24,9 +26,9 @@ baseline. ## Editable and frozen surfaces -Edit only paths explicitly allowed by `experiment_config.yaml`. Never change the benchmark, -held-out tasks, verifier, model, reasoning budget, observability identity, or this program -to improve a score. +Target only exact paths allowed by the canonical experiment policy. Do not edit them in this +program. Never target the benchmark, held-out tasks, verifier, model, reasoning budget, +observability identity, or this program. Keep one focused hypothesis per iteration. Do not mix prompt, tool, middleware, and control flow changes unless the evidence requires the combination. @@ -46,37 +48,11 @@ State the failure pattern, supporting trace and verifier evidence, proposed harn expected improvement, and possible regressions. Stop if the evidence cannot distinguish between materially different changes. -### 4. Improve the harness +Use `$hypothesis-former` with only exact supported pattern and diagnosis receipt IDs, then call +`record_hypothesis`. Retain the stable hypothesis receipt and stop before candidate editing. +Candidate editing requires a later package and must not begin in this prepared program. -Make the smallest change within the declared editable surface. Preserve frozen controls and -unrelated user changes. +## Package boundary -### 5. Gate the change - -Run only the prepared experiment command and gates declared by the workspace. Compare -task-level verifier outcomes and report quality, cost, and latency separately. Missing or -errored trials remain visible and cannot disappear from the denominator. - -### 6. Keep or revert - -Keep the change only when the configured gate admits it. Otherwise revert only the current -iteration's harness edit, retain the evidence, and try a different hypothesis. Never weaken -the gate to admit a candidate. - -Commit each admitted improvement on the prepared `ofw/` branch before the -next iteration. Keep one hypothesis per commit and include `OFW-Experiment` and `OFW-Run` -trailers. Do not commit failed candidates, generated run artifacts, credentials, or changes -outside the editable surface. Do not push or open a pull request without explicit user -authorization. - -### 7. Repeat - -Return to step 2 with the newly recorded run. Stop when the configured goal is met, the -budget or iteration limit is exhausted, the no-improvement condition is reached, or required -authoritative evidence is unavailable. - -## Final report - -Report the accepted harness revision, verifier-backed quality, cost, latency, remaining -unverified trials, iteration count, and exact stopping reason. Do not claim improvement from -an agent-authored summary alone. +Report the hypothesis receipt and its exact evidence and target paths. Do not edit, commit, +run, gate, publish, or install a candidate; those capabilities are not part of this program. diff --git a/src/ofw/safe_file.py b/src/ofw/safe_file.py new file mode 100644 index 0000000..fef367b --- /dev/null +++ b/src/ofw/safe_file.py @@ -0,0 +1,223 @@ +"""Small descriptor-anchored primitives for bounded immutable local files.""" + +from __future__ import annotations + +import os +import stat +from collections.abc import Iterator +from contextlib import contextmanager +from enum import StrEnum +from pathlib import Path +from uuid import uuid4 + +_DIRECTORY_FLAGS = os.O_RDONLY | os.O_DIRECTORY | os.O_NOFOLLOW +_CREATE_FILE_FLAGS = os.O_WRONLY | os.O_CREAT | os.O_EXCL | os.O_NOFOLLOW +_READ_FILE_FLAGS = os.O_RDONLY | os.O_NOFOLLOW | os.O_NONBLOCK + + +class SafeFileErrorCode(StrEnum): + CONFLICT = "conflict" + DIRECTORY_CHANGED = "directory_changed" + INVALID_FILE = "invalid_file" + TOO_LARGE = "too_large" + + +class SafeFileFailure(Exception): + """Sanitized failure from a local immutable-file boundary.""" + + __slots__ = ("code", "subject") + + def __init__(self, code: SafeFileErrorCode, subject: str) -> None: + self.code = code + self.subject = subject + super().__init__(f"{code.value}: {subject}") + + +@contextmanager +def open_directory(path: Path) -> Iterator[int]: + """Open one directory without following links and recheck its identity on exit.""" + descriptor = os.open(path, _DIRECTORY_FLAGS) + expected = _descriptor_identity(descriptor) + try: + _require_path_identity(path, expected) + yield descriptor + _require_path_identity(path, expected) + finally: + os.close(descriptor) + + +@contextmanager +def open_directory_chain( + root: Path, + parts: tuple[str, ...], + *, + create: bool, +) -> Iterator[int]: + """Open a contained child chain and recheck every link before and after use.""" + root_descriptor = os.open(root, _DIRECTORY_FLAGS) + descriptors = [root_descriptor] + expected_root = _descriptor_identity(root_descriptor) + try: + for part in parts: + descriptors.append(_open_child(descriptors[-1], part, create=create)) + _require_chain(root, parts, descriptors, expected_root) + yield descriptors[-1] + _require_chain(root, parts, descriptors, expected_root) + finally: + for descriptor in reversed(descriptors): + os.close(descriptor) + + +def publish_idempotent( + directory: int, + name: str, + content: bytes, + *, + maximum_bytes: int, + subject: str, +) -> None: + """Publish new bytes once, accepting only an identical existing regular file.""" + _require_leaf(name) + _require_size(content, maximum_bytes, subject) + try: + _publish_new_file(directory, name, content) + except FileExistsError: + if read_bounded(directory, name, maximum_bytes=maximum_bytes, subject=subject) != content: + raise SafeFileFailure(SafeFileErrorCode.CONFLICT, subject) from None + + +def read_bounded( + directory: int, + name: str, + *, + maximum_bytes: int, + subject: str, +) -> bytes: + """Read one no-follow regular file through its containing directory descriptor.""" + _require_leaf(name) + try: + descriptor = os.open(name, _READ_FILE_FLAGS, dir_fd=directory) + except OSError as error: + if isinstance(error, FileNotFoundError): + raise + raise SafeFileFailure(SafeFileErrorCode.INVALID_FILE, subject) from None + try: + _require_regular_file(descriptor, subject) + content = _read_descriptor(descriptor, maximum_bytes) + finally: + os.close(descriptor) + _require_size(content, maximum_bytes, subject) + return content + + +def _open_child(parent: int, name: str, *, create: bool) -> int: + _require_leaf(name) + if create: + _mkdir_if_missing(parent, name) + return os.open(name, _DIRECTORY_FLAGS, dir_fd=parent) + + +def _mkdir_if_missing(parent: int, name: str) -> None: + try: + os.mkdir(name, 0o700, dir_fd=parent) + except FileExistsError: + return + os.fsync(parent) + + +def _require_chain( + root: Path, + parts: tuple[str, ...], + descriptors: list[int], + expected_root: tuple[int, int], +) -> None: + _require_path_identity(root, expected_root) + for index, name in enumerate(parts): + _require_child_identity(descriptors[index], name, descriptors[index + 1]) + + +def _require_path_identity(path: Path, expected: tuple[int, int]) -> None: + try: + current = _stat_identity(os.stat(path, follow_symlinks=False)) + except OSError: + raise SafeFileFailure(SafeFileErrorCode.DIRECTORY_CHANGED, path.name) from None + if current != expected: + raise SafeFileFailure(SafeFileErrorCode.DIRECTORY_CHANGED, path.name) + + +def _require_child_identity(parent: int, name: str, descriptor: int) -> None: + try: + current = _stat_identity(os.stat(name, dir_fd=parent, follow_symlinks=False)) + except OSError: + raise SafeFileFailure(SafeFileErrorCode.DIRECTORY_CHANGED, name) from None + if current != _descriptor_identity(descriptor): + raise SafeFileFailure(SafeFileErrorCode.DIRECTORY_CHANGED, name) + + +def _publish_new_file(directory: int, name: str, content: bytes) -> None: + temporary_name = f".ofw-{uuid4().hex}.tmp" + published = False + try: + _write_new_file(directory, temporary_name, content) + os.link( + temporary_name, + name, + src_dir_fd=directory, + dst_dir_fd=directory, + follow_symlinks=False, + ) + published = True + finally: + _unlink_if_present(directory, temporary_name) + if published: + os.fsync(directory) + + +def _write_new_file(directory: int, name: str, content: bytes) -> None: + descriptor = os.open(name, _CREATE_FILE_FLAGS, 0o600, dir_fd=directory) + with os.fdopen(descriptor, "wb") as stream: + stream.write(content) + stream.flush() + os.fsync(stream.fileno()) + + +def _unlink_if_present(directory: int, name: str) -> None: + try: + os.unlink(name, dir_fd=directory) + except FileNotFoundError: + return + + +def _read_descriptor(descriptor: int, maximum_bytes: int) -> bytes: + chunks: list[bytes] = [] + remaining = maximum_bytes + 1 + while remaining: + chunk = os.read(descriptor, remaining) + if not chunk: + break + chunks.append(chunk) + remaining -= len(chunk) + return b"".join(chunks) + + +def _require_regular_file(descriptor: int, subject: str) -> None: + if not stat.S_ISREG(os.fstat(descriptor).st_mode): + raise SafeFileFailure(SafeFileErrorCode.INVALID_FILE, subject) + + +def _require_size(content: bytes, maximum_bytes: int, subject: str) -> None: + if len(content) > maximum_bytes: + raise SafeFileFailure(SafeFileErrorCode.TOO_LARGE, subject) + + +def _require_leaf(name: str) -> None: + if not name or name in (".", "..") or "/" in name or "\x00" in name: + raise SafeFileFailure(SafeFileErrorCode.INVALID_FILE, "file_name") + + +def _descriptor_identity(descriptor: int) -> tuple[int, int]: + return _stat_identity(os.fstat(descriptor)) + + +def _stat_identity(value: os.stat_result) -> tuple[int, int]: + return value.st_dev, value.st_ino diff --git a/tests/test_experiment_policy.py b/tests/test_experiment_policy.py new file mode 100644 index 0000000..2e76153 --- /dev/null +++ b/tests/test_experiment_policy.py @@ -0,0 +1,211 @@ +"""Canonical experiment policy publication and reload boundaries.""" + +from __future__ import annotations + +import hashlib +import os +import subprocess +from pathlib import Path + +import pytest +from pydantic import ValidationError + +from ofw.preparation.contracts import ( + BaselineConfiguration, + PreparedGitWorkspace, + PrepareWorkspaceInput, +) +from ofw.preparation.policy import ( + ExperimentPolicyErrorCode, + ExperimentPolicyFailure, + ExperimentPolicySnapshot, + FileExperimentPolicyRepository, + build_experiment_policy, +) + + +def _git(root: Path, *arguments: str) -> str: + return subprocess.run( + ("git", "-C", str(root), *arguments), + check=True, + capture_output=True, + text=True, + ).stdout.strip() + + +def _repository(tmp_path: Path) -> tuple[Path, str]: + root = tmp_path / "harness" + root.mkdir() + (root / "prompt.md").write_text("Original prompt.\n", encoding="utf-8") + _git(root, "init", "-q") + _git(root, "config", "user.name", "OpenFlywheel Test") + _git(root, "config", "user.email", "ofw@example.test") + _git(root, "add", "prompt.md") + _git(root, "commit", "-qm", "initial") + return root, _git(root, "rev-parse", "HEAD") + + +def _request( + tmp_path: Path, + root: Path, + *, + goal: str = "Improve verifier-backed quality.", +) -> PrepareWorkspaceInput: + benchmark = tmp_path / "benchmark" + benchmark.mkdir() + executable = tmp_path / "harbor" + executable.touch(mode=0o700) + return PrepareWorkspaceInput( + experiment_id="experiment-one", + harness_root=root, + base_ref="HEAD", + worktree_parent=tmp_path, + benchmark_root=benchmark, + harbor_executable=executable, + harbor_config=Path("config.json"), + expected_task_count=2, + editable_paths=(Path("prompt.md"),), + goal=goal, + quality_target=0.9, + max_iterations=4, + no_improvement_limit=2, + max_cost_per_task_usd=1.5, + max_latency_seconds=120.0, + max_baseline_seconds=600, + ) + + +def _snapshot( + tmp_path: Path, + *, + goal: str = "Improve verifier-backed quality.", +) -> tuple[Path, ExperimentPolicySnapshot]: + tmp_path.mkdir(parents=True, exist_ok=True) + root, commit = _repository(tmp_path) + request = _request(tmp_path, root, goal=goal) + prepared = PreparedGitWorkspace( + branch_name="ofw/experiment-one", + worktree_path=root, + base_commit=commit, + initialization_commit=commit, + program_path=root / "PROGRAM.md", + ) + baseline = BaselineConfiguration( + model="openai/gpt-5.4-mini", + task_ids=("tasks/a", "tasks/b"), + benchmark_config_digest=f"sha256:{hashlib.sha256(b'config').hexdigest()}", + verifier="itsm-bench", + environment="itsm-bench", + ) + return root, build_experiment_policy(request, prepared, baseline) + + +def _control_directory(root: Path) -> Path: + path = root / ".git/ofw/preparations/experiment-one" + path.mkdir(parents=True) + return path + + +def test_policy_is_derived_from_validated_preparation_inputs(tmp_path: Path) -> None: + _, policy = _snapshot(tmp_path) + + assert policy.experiment_id == "experiment-one" + assert policy.editable_paths == (Path("prompt.md"),) + assert policy.task_ids == ("tasks/a", "tasks/b") + assert policy.model == "openai/gpt-5.4-mini" + assert policy.verifier == "itsm-bench" + assert policy.environment == "itsm-bench" + assert policy.concurrency == 1 + assert policy.max_retries == 0 + assert policy.controls_digest.startswith("sha256:") + assert policy.controls_digest == policy.recomputed_controls_digest() + + +def test_policy_schema_rejects_extra_fields_and_tampered_controls(tmp_path: Path) -> None: + _, policy = _snapshot(tmp_path) + payload = policy.model_dump_json() + extra = payload[:-1] + ',"unexpected":true}' + + with pytest.raises(ValidationError): + ExperimentPolicySnapshot.model_validate_json(extra) + + tampered = payload.replace(policy.model, "different-model") + with pytest.raises(ValidationError): + ExperimentPolicySnapshot.model_validate_json(tampered) + + escaped = payload.replace('"editable_paths":["prompt.md"]', '"editable_paths":["../x"]') + with pytest.raises(ValidationError): + ExperimentPolicySnapshot.model_validate_json(escaped) + + +def test_policy_publish_is_atomic_idempotent_and_conflict_detecting(tmp_path: Path) -> None: + root, policy = _snapshot(tmp_path) + control = _control_directory(root) + repository = FileExperimentPolicyRepository() + + first = repository.publish(control, policy) + repeated = repository.publish(control, policy) + + assert first == repeated == control / "policy.json" + assert repository.load(root, "experiment-one") == policy + _, conflicting = _snapshot(tmp_path / "conflict", goal="Conflicting goal.") + with pytest.raises(ExperimentPolicyFailure) as raised: + repository.publish(control, conflicting) + assert raised.value.code is ExperimentPolicyErrorCode.POLICY_CONFLICT + + +@pytest.mark.parametrize("kind", ("symlink", "fifo")) +def test_policy_rejects_non_regular_existing_file(tmp_path: Path, kind: str) -> None: + root, policy = _snapshot(tmp_path) + control = _control_directory(root) + path = control / "policy.json" + if kind == "symlink": + path.symlink_to(root / "prompt.md") + else: + os.mkfifo(path) + + with pytest.raises(ExperimentPolicyFailure) as raised: + FileExperimentPolicyRepository().publish(control, policy) + + assert raised.value.code is ExperimentPolicyErrorCode.POLICY_WRITE_FAILED + + +def test_policy_reload_rejects_missing_oversized_and_invalid_files(tmp_path: Path) -> None: + root, _ = _snapshot(tmp_path) + control = _control_directory(root) + repository = FileExperimentPolicyRepository() + + with pytest.raises(ExperimentPolicyFailure) as missing: + repository.load(root, "experiment-one") + assert missing.value.code is ExperimentPolicyErrorCode.POLICY_SNAPSHOT_REQUIRED + + path = control / "policy.json" + path.write_bytes(b"x" * (64 * 1024 + 1)) + with pytest.raises(ExperimentPolicyFailure) as oversized: + repository.load(root, "experiment-one") + assert oversized.value.code is ExperimentPolicyErrorCode.POLICY_TOO_LARGE + + path.write_text("{}", encoding="utf-8") + with pytest.raises(ExperimentPolicyFailure) as invalid: + repository.load(root, "experiment-one") + assert invalid.value.code is ExperimentPolicyErrorCode.POLICY_INVALID + + +def test_policy_publish_fsyncs_file_and_directory( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + root, policy = _snapshot(tmp_path) + control = _control_directory(root) + calls: list[int] = [] + original = os.fsync + + def capture(descriptor: int) -> None: + calls.append(descriptor) + original(descriptor) + + monkeypatch.setattr(os, "fsync", capture) + + FileExperimentPolicyRepository().publish(control, policy) + + assert len(calls) >= 2 diff --git a/tests/test_hypothesis.py b/tests/test_hypothesis.py new file mode 100644 index 0000000..37cab97 --- /dev/null +++ b/tests/test_hypothesis.py @@ -0,0 +1,522 @@ +"""Evidence-backed hypothesis contracts, authority, and persistence.""" + +from __future__ import annotations + +import os +import subprocess +from concurrent.futures import ThreadPoolExecutor +from datetime import UTC, datetime +from pathlib import Path + +import pytest +from pydantic import ValidationError + +from ofw.contracts import ComponentKind +from ofw.evaluation.failure import FailureEvidenceStatus, FailureType +from ofw.evaluation.failure_patterns import FailurePatternMiningService, failure_pattern_id +from ofw.evaluation.failure_workspace import ( + FailedOutcomeInput, + FailureWorkspaceService, + FileFailureWorkspace, + RecordFailureInput, +) +from ofw.evolution.hypothesis import ( + FailurePatternReferenceInput, + HarnessChangeTargetInput, + HypothesisErrorCode, + HypothesisFailure, + HypothesisService, + HypothesisStatus, + RecordHypothesisInput, +) +from ofw.evolution.hypothesis_repository import FileHypothesisRepository +from ofw.preparation.contracts import ( + BaselineConfiguration, + PreparedGitWorkspace, + PrepareWorkspaceInput, +) +from ofw.preparation.policy import FileExperimentPolicyRepository, build_experiment_policy + +_ROOT_CAUSE = "The agent finalizes before checking the updated incident state." + + +def _git(root: Path, *arguments: str, check: bool = True) -> str: + return subprocess.run( + ("git", "-C", str(root), *arguments), + check=check, + capture_output=True, + text=True, + ).stdout.strip() + + +def _workspace(tmp_path: Path) -> tuple[Path, str]: + root = tmp_path / "harness" + root.mkdir() + (root / "prompt.md").write_text("Check state before finalizing.\n", encoding="utf-8") + (root / "tools.py").write_text("def check_state():\n return True\n", encoding="utf-8") + (root / "PROGRAM.md").write_text("# Program\n", encoding="utf-8") + (root / "experiment_config.yaml").write_text("benchmark: itsm-bench\n", encoding="utf-8") + _git(root, "init", "-q") + _git(root, "config", "user.name", "OpenFlywheel Test") + _git(root, "config", "user.email", "ofw@example.test") + _git(root, "add", ".") + _git(root, "commit", "-qm", "prepared experiment") + _git(root, "branch", "-m", "ofw/experiment-one") + commit = _git(root, "rev-parse", "HEAD") + _publish_policy(tmp_path, root, commit) + return root, commit + + +def _publish_policy(tmp_path: Path, root: Path, commit: str) -> None: + benchmark = tmp_path / "benchmark" + benchmark.mkdir(exist_ok=True) + executable = tmp_path / "harbor" + executable.touch(mode=0o700, exist_ok=True) + request = PrepareWorkspaceInput( + experiment_id="experiment-one", + harness_root=root, + base_ref="HEAD", + worktree_parent=tmp_path, + benchmark_root=benchmark, + harbor_executable=executable, + harbor_config=Path("config.json"), + expected_task_count=2, + editable_paths=(Path("prompt.md"),), + goal="Improve verifier-backed quality.", + quality_target=1.0, + max_iterations=3, + no_improvement_limit=2, + max_baseline_seconds=600, + ) + prepared = PreparedGitWorkspace( + branch_name="ofw/experiment-one", + worktree_path=root, + base_commit=commit, + initialization_commit=commit, + program_path=root / "PROGRAM.md", + ) + baseline = BaselineConfiguration( + model="openai/gpt-5.4-mini", + task_ids=("task-1", "task-2"), + benchmark_config_digest="sha256:" + "1" * 64, + verifier="itsm-bench", + environment="itsm-bench", + ) + policy = build_experiment_policy(request, prepared, baseline) + control = root / ".git/ofw/preparations/experiment-one" + control.mkdir(parents=True) + FileExperimentPolicyRepository().publish(control, policy) + + +def _diagnosis( + root: Path, + suffix: str, + *, + root_cause: str = _ROOT_CAUSE, + supported: bool = True, +) -> str: + evidence_status = ( + FailureEvidenceStatus.SUPPORTED if supported else FailureEvidenceStatus.INCONCLUSIVE + ) + observation = FailureWorkspaceService(FileFailureWorkspace()).record( + RecordFailureInput( + workspace_root=root, + outcome=FailedOutcomeInput( + trace_id=f"trace-{suffix}", + task_id=f"task-{suffix}", + verifier_id="itsm-bench", + evaluated_at=datetime(2026, 9, 1, 12, int(suffix), tzinfo=UTC), + score=0.0, + evidence=(f"score://{suffix}",), + outcome_score_id=f"score-{suffix}", + ), + evidence_status=evidence_status, + issue_type=FailureType.CONTROL_FLOW_FAILURE if supported else None, + expected_outcome="The incident is closed.", + actual_outcome="The incident remains open.", + critical_observation_id=f"observation-{suffix}" if supported else None, + evidence_observation_ids=(f"observation-{suffix}",) if supported else (), + root_cause=root_cause if supported else None, + counterfactual_action="Read state before finalizing." if supported else None, + inconclusive_reason=None if supported else "The decisive span is unavailable.", + ) + ) + return observation.artifact_id + + +def _service() -> HypothesisService: + workspace = FileFailureWorkspace() + return HypothesisService( + pattern_miner=FailurePatternMiningService(workspace), + repository=FileHypothesisRepository(), + ) + + +def _request(root: Path, commit: str, artifacts: tuple[str, ...]) -> RecordHypothesisInput: + return RecordHypothesisInput( + workspace_root=root, + experiment_id="experiment-one", + source_commit=commit, + patterns=( + FailurePatternReferenceInput( + pattern_id=failure_pattern_id( + FailureType.CONTROL_FLOW_FAILURE, + _ROOT_CAUSE, + ), + diagnosis_artifact_ids=artifacts, + ), + ), + statement="Require a state check before the agent finalizes.", + rationale="Both supported failures finalize immediately after mutation.", + target=HarnessChangeTargetInput( + component_kind=ComponentKind.PROMPT, + relative_paths=(Path("prompt.md"),), + ), + expected_effect="The agent verifies task completion before returning success.", + regression_risks=("The extra check may increase latency.",), + ) + + +def _with_patterns( + request: RecordHypothesisInput, + patterns: tuple[FailurePatternReferenceInput, ...], +) -> RecordHypothesisInput: + return RecordHypothesisInput( + workspace_root=request.workspace_root, + experiment_id=request.experiment_id, + source_commit=request.source_commit, + patterns=patterns, + statement=request.statement, + rationale=request.rationale, + target=request.target, + expected_effect=request.expected_effect, + regression_risks=request.regression_risks, + ) + + +def _with_target( + request: RecordHypothesisInput, + target: HarnessChangeTargetInput, +) -> RecordHypothesisInput: + return RecordHypothesisInput( + workspace_root=request.workspace_root, + experiment_id=request.experiment_id, + source_commit=request.source_commit, + patterns=request.patterns, + statement=request.statement, + rationale=request.rationale, + target=target, + expected_effect=request.expected_effect, + regression_risks=request.regression_risks, + ) + + +def _evidence_case( + case: str, + first: str, + second: str, + other: str, + inconclusive: str, +) -> tuple[str, ...]: + if case == "missing": + return first, "00000000-0000-0000-0000-000000000000" + if case == "extra": + return first, second, other + if case == "misassigned": + return first, other + return first, inconclusive + + +def test_record_hypothesis_recomputes_evidence_and_is_deterministic(tmp_path: Path) -> None: + root, commit = _workspace(tmp_path) + artifacts = (_diagnosis(root, "1"), _diagnosis(root, "2")) + request = _request(root, commit, artifacts) + + first = _service().record(request) + repeated = _service().record(request) + + assert first == repeated + assert first.status is HypothesisStatus.SUCCESS + assert first.source_commit == commit + assert first.pattern_count == 1 + assert first.diagnosis_count == 2 + assert first.target_paths == (Path("prompt.md"),) + assert first.relative_path == Path(f".workspace/hypotheses/{first.hypothesis_id}.json") + assert "stop" in first.next_actions[0].lower() + assert _git(root, "status", "--short") == "" + + +def test_hypothesis_identity_canonicalizes_declared_order(tmp_path: Path) -> None: + root, commit = _workspace(tmp_path) + artifacts = (_diagnosis(root, "1"), _diagnosis(root, "2")) + request = _request(root, commit, artifacts) + reversed_request = _with_patterns( + request, + ( + FailurePatternReferenceInput( + pattern_id=request.patterns[0].pattern_id, + diagnosis_artifact_ids=tuple(reversed(artifacts)), + ), + ), + ) + + assert _service().record(reversed_request) == _service().record(request) + + +@pytest.mark.parametrize("case", ("missing", "extra", "misassigned", "inconclusive")) +def test_hypothesis_fails_closed_on_inexact_evidence(tmp_path: Path, case: str) -> None: + root, commit = _workspace(tmp_path) + first = _diagnosis(root, "1") + second = _diagnosis(root, "2") + other = _diagnosis(root, "3", root_cause="The tool result is ignored after mutation.") + inconclusive = _diagnosis(root, "4", supported=False) + artifacts = _evidence_case(case, first, second, other, inconclusive) + request = _request(root, commit, artifacts) + if case == "misassigned": + request = _with_patterns( + request, + ( + FailurePatternReferenceInput( + pattern_id=failure_pattern_id( + FailureType.CONTROL_FLOW_FAILURE, + _ROOT_CAUSE, + ), + diagnosis_artifact_ids=(other,), + ), + FailurePatternReferenceInput( + pattern_id=failure_pattern_id( + FailureType.CONTROL_FLOW_FAILURE, + "The tool result is ignored after mutation.", + ), + diagnosis_artifact_ids=(first,), + ), + ), + ) + + with pytest.raises(HypothesisFailure) as raised: + _service().record(request) + + expected = ( + HypothesisErrorCode.INCONCLUSIVE_EVIDENCE + if case == "inconclusive" + else HypothesisErrorCode.PATTERN_EVIDENCE_MISMATCH + ) + assert raised.value.code is expected + + +@pytest.mark.parametrize("drift", ("commit", "dirty", "branch", "target-symlink")) +def test_hypothesis_rejects_stale_or_drifted_workspace(tmp_path: Path, drift: str) -> None: + root, commit = _workspace(tmp_path) + artifacts = (_diagnosis(root, "1"), _diagnosis(root, "2")) + request = _request(root, commit, artifacts) + if drift == "commit": + (root / "tools.py").write_text("# changed\n", encoding="utf-8") + _git(root, "add", "tools.py") + _git(root, "commit", "-qm", "move head") + elif drift == "dirty": + (root / "prompt.md").write_text("dirty\n", encoding="utf-8") + elif drift == "branch": + _git(root, "branch", "-m", "wrong-branch") + else: + (root / "prompt.md").unlink() + (root / "prompt.md").symlink_to(root / "tools.py") + + with pytest.raises(HypothesisFailure) as raised: + _service().record(request) + + assert raised.value.code in { + HypothesisErrorCode.STALE_COMMIT, + HypothesisErrorCode.DIRTY_WORKSPACE, + HypothesisErrorCode.STALE_POLICY, + HypothesisErrorCode.INVALID_TARGET, + } + + +@pytest.mark.parametrize("path", (Path("tools.py"), Path("PROGRAM.md"), Path("prompt.md/child"))) +def test_hypothesis_uses_exact_editable_allowlist_and_freezes_everything_else( + tmp_path: Path, + path: Path, +) -> None: + root, commit = _workspace(tmp_path) + artifacts = (_diagnosis(root, "1"), _diagnosis(root, "2")) + request = _request(root, commit, artifacts) + request = _with_target( + request, + HarnessChangeTargetInput( + component_kind=ComponentKind.TOOL, + relative_paths=(path,), + ), + ) + + with pytest.raises(HypothesisFailure) as raised: + _service().record(request) + + assert raised.value.code is HypothesisErrorCode.TARGET_NOT_EDITABLE + + +def test_hypothesis_input_rejects_duplicates_escapes_empty_and_extra_fields(tmp_path: Path) -> None: + root, commit = _workspace(tmp_path) + artifact = _diagnosis(root, "1") + payload = _request(root, commit, (artifact,)).model_dump_json() + extra = payload[:-1] + ',"unexpected":true}' + with pytest.raises(ValidationError): + RecordHypothesisInput.model_validate_json(extra) + + escaped = payload.replace('"relative_paths":["prompt.md"]', '"relative_paths":["../prompt.md"]') + with pytest.raises(ValidationError): + RecordHypothesisInput.model_validate_json(escaped) + + empty = payload.replace('"relative_paths":["prompt.md"]', '"relative_paths":[]') + with pytest.raises(ValidationError): + RecordHypothesisInput.model_validate_json(empty) + + duplicate = payload.replace( + f'"diagnosis_artifact_ids":["{artifact}"]', + f'"diagnosis_artifact_ids":["{artifact}","{artifact}"]', + ) + with pytest.raises(ValidationError): + RecordHypothesisInput.model_validate_json(duplicate) + + request = _request(root, commit, (artifact,)) + with pytest.raises(ValidationError): + _with_patterns(request, (request.patterns[0], request.patterns[0])) + with pytest.raises(ValidationError): + HarnessChangeTargetInput( + component_kind=ComponentKind.PROMPT, + relative_paths=(Path("prompt.md"), Path("prompt.md")), + ) + with pytest.raises(ValidationError): + RecordHypothesisInput( + workspace_root=request.workspace_root, + experiment_id=request.experiment_id, + source_commit=request.source_commit, + patterns=request.patterns, + statement=request.statement, + rationale=request.rationale, + target=request.target, + expected_effect=request.expected_effect, + regression_risks=("same risk", "same risk"), + ) + + +def test_hypothesis_input_accepts_its_exact_global_maximum() -> None: + artifacts = tuple(f"00000000-0000-0000-0000-{index:012x}" for index in range(50)) + request = RecordHypothesisInput( + workspace_root=Path("/prepared"), + experiment_id="maximum", + source_commit="1" * 40, + patterns=( + FailurePatternReferenceInput( + pattern_id="sha256:" + "1" * 64, + diagnosis_artifact_ids=artifacts, + ), + ), + statement="s" * 4000, + rationale="r" * 4000, + target=HarnessChangeTargetInput( + component_kind=ComponentKind.SKILL, + relative_paths=tuple(Path(f"skills/{index}.md") for index in range(50)), + ), + expected_effect="e" * 4000, + regression_risks=tuple(f"risk-{index}" for index in range(10)), + ) + + assert len(request.patterns[0].diagnosis_artifact_ids) == 50 + assert len(request.target.relative_paths) == 50 + assert len(request.regression_risks) == 10 + + +def test_hypothesis_missing_policy_is_typed(tmp_path: Path) -> None: + root, commit = _workspace(tmp_path) + (root / ".git/ofw/preparations/experiment-one/policy.json").unlink() + artifacts = (_diagnosis(root, "1"), _diagnosis(root, "2")) + + with pytest.raises(HypothesisFailure) as raised: + _service().record(_request(root, commit, artifacts)) + + assert raised.value.code is HypothesisErrorCode.POLICY_SNAPSHOT_REQUIRED + + +def test_hypothesis_concurrent_identical_writes_are_idempotent(tmp_path: Path) -> None: + root, commit = _workspace(tmp_path) + request = _request(root, commit, (_diagnosis(root, "1"), _diagnosis(root, "2"))) + + with ThreadPoolExecutor(max_workers=2) as executor: + results = tuple(executor.map(lambda _: _service().record(request), range(2))) + + assert results[0] == results[1] + + +def test_hypothesis_conflicting_existing_artifact_is_rejected(tmp_path: Path) -> None: + root, commit = _workspace(tmp_path) + request = _request(root, commit, (_diagnosis(root, "1"), _diagnosis(root, "2"))) + recorded = _service().record(request) + path = root / recorded.relative_path + path.write_text("{}", encoding="utf-8") + + with pytest.raises(HypothesisFailure) as raised: + _service().record(request) + + assert raised.value.code is HypothesisErrorCode.HYPOTHESIS_CONFLICT + + +@pytest.mark.parametrize("kind", ("symlink", "fifo")) +def test_hypothesis_rejects_non_regular_artifact_target(tmp_path: Path, kind: str) -> None: + root, commit = _workspace(tmp_path) + request = _request(root, commit, (_diagnosis(root, "1"), _diagnosis(root, "2"))) + recorded = _service().record(request) + path = root / recorded.relative_path + path.unlink() + if kind == "symlink": + path.symlink_to(root / "prompt.md") + else: + os.mkfifo(path) + + with pytest.raises(HypothesisFailure) as raised: + _service().record(request) + + assert raised.value.code is HypothesisErrorCode.WRITE_FAILED + + +@pytest.mark.parametrize( + "invalid", + ("missing-root", "missing-marker", "missing-target", "directory"), +) +def test_hypothesis_repository_rejects_invalid_workspace_objects( + tmp_path: Path, + invalid: str, +) -> None: + root, commit = _workspace(tmp_path) + repository = FileHypothesisRepository() + policy = repository.load_policy(root, "experiment-one") + target = Path("prompt.md") + if invalid == "missing-root": + root = tmp_path / "missing" + elif invalid == "missing-marker": + (root / "PROGRAM.md").unlink() + elif invalid == "missing-target": + target = Path("missing.md") + else: + target = Path("directory") + (root / target).mkdir() + + with pytest.raises(HypothesisFailure) as raised: + repository.validate_workspace(root, policy, commit, (target,)) + + assert raised.value.code in { + HypothesisErrorCode.STALE_POLICY, + HypothesisErrorCode.INVALID_TARGET, + } + + +def test_hypothesis_repository_sanitizes_git_failure(tmp_path: Path) -> None: + root, commit = _workspace(tmp_path) + repository = FileHypothesisRepository() + policy = repository.load_policy(root, "experiment-one") + (root / ".git").rename(root / ".git-moved") + + with pytest.raises(HypothesisFailure) as raised: + repository.validate_workspace(root, policy, commit, (Path("prompt.md"),)) + + assert raised.value.code is HypothesisErrorCode.STALE_POLICY + assert "fatal" not in str(raised.value).lower() diff --git a/tests/test_openflywheel_mcp.py b/tests/test_openflywheel_mcp.py index ca37c49..e6adfd2 100644 --- a/tests/test_openflywheel_mcp.py +++ b/tests/test_openflywheel_mcp.py @@ -51,6 +51,14 @@ VerifierId, VerifierVerdict, ) +from ofw.evolution import ( + FailurePatternReferenceInput, + HarnessChangeTargetInput, + HypothesisObservation, + HypothesisService, + HypothesisStatus, + RecordHypothesisInput, +) from ofw.observability.langfuse.domain import ScoreId, TraceId from ofw.observability.langfuse.trace_query import ( GetSpanContextInput, @@ -71,6 +79,9 @@ _FAILURE_ARTIFACT_ID = "00000000-0000-0000-0000-000000000001" _SECOND_FAILURE_ARTIFACT_ID = "00000000-0000-0000-0000-000000000002" _CURATION_ID = "00000000-0000-0000-0000-000000000003" +_PATTERN_ID = "sha256:" + "1" * 64 +_HYPOTHESIS_ID = "sha256:" + "2" * 64 +_COMMIT = "1" * 40 _JSON_OBJECT_ADAPTER: TypeAdapter[dict[str, JsonValue]] = TypeAdapter(dict[str, JsonValue]) @@ -84,6 +95,8 @@ def _failure_service(self) -> FailureWorkspaceService: ... def _curation_service(self) -> FailureCurationService: ... + def _hypothesis_service(self) -> HypothesisService: ... + def _program_template(self, name: str) -> str: ... def prepare_workspace( @@ -140,6 +153,8 @@ def record_failure_curation( request: RecordFailureCurationInput, ) -> FailureCurationObservation: ... + def record_hypothesis(self, request: RecordHypothesisInput) -> HypothesisObservation: ... + class _FakeOutcomeStore: def __init__(self) -> None: @@ -192,6 +207,16 @@ def record(self, request: RecordFailureCurationInput) -> FailureCurationObservat return self.observation +class _FakeHypothesisService: + def __init__(self, observation: HypothesisObservation) -> None: + self.observation = observation + self.requests: list[RecordHypothesisInput] = [] + + def record(self, request: RecordHypothesisInput) -> HypothesisObservation: + self.requests.append(request) + return self.observation + + def _module() -> OpenFlywheelMcpModule: return cast(OpenFlywheelMcpModule, importlib.import_module("ofw.mcp")) @@ -310,6 +335,45 @@ def _curation_observation() -> FailureCurationObservation: ) +def _hypothesis_request(root: Path) -> RecordHypothesisInput: + return RecordHypothesisInput( + workspace_root=root, + experiment_id="experiment-one", + source_commit=_COMMIT, + patterns=( + FailurePatternReferenceInput( + pattern_id=_PATTERN_ID, + diagnosis_artifact_ids=(_FAILURE_ARTIFACT_ID,), + ), + ), + statement="Require a final state check.", + rationale="The supported pattern shows premature finalization.", + target=HarnessChangeTargetInput( + component_kind=ComponentKind.PROMPT, + relative_paths=(Path("prompt.md"),), + ), + expected_effect="The agent verifies success before finalizing.", + regression_risks=("The extra check may add latency.",), + ) + + +def _hypothesis_observation() -> HypothesisObservation: + relative_path = Path(f".workspace/hypotheses/{_HYPOTHESIS_ID}.json") + return HypothesisObservation( + status=HypothesisStatus.SUCCESS, + summary="Recorded one evidence-backed hypothesis for the prepared experiment.", + next_actions=("Stop before candidate editing and retain this hypothesis receipt.",), + artifacts=(str(relative_path), _HYPOTHESIS_ID), + hypothesis_id=_HYPOTHESIS_ID, + experiment_id="experiment-one", + source_commit=_COMMIT, + relative_path=relative_path, + pattern_count=1, + diagnosis_count=1, + target_paths=(Path("prompt.md"),), + ) + + def test_mcp_exposes_scoped_read_and_recording_tools() -> None: tools = asyncio.run(_server().list_tools()) @@ -323,6 +387,7 @@ def test_mcp_exposes_scoped_read_and_recording_tools() -> None: "record_failure", "mine_failure_patterns", "record_failure_curation", + "record_hypothesis", ] assert tuple(map(_annotation_flags, tools)) == ( (False, False, True), @@ -334,6 +399,7 @@ def test_mcp_exposes_scoped_read_and_recording_tools() -> None: (False, False, True), (True, False, True), (False, False, True), + (False, False, True), ) @@ -633,3 +699,23 @@ def test_record_failure_curation_accepts_mcp_json_mapping(tmp_path: Path) -> Non request = _curation_request(tmp_path) assert RecordFailureCurationInput.model_validate(_json_request(request)) == request + + +def test_record_hypothesis_passes_one_strict_object_to_the_service( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + module = _module() + expected = _hypothesis_observation() + service = _FakeHypothesisService(expected) + request = _hypothesis_request(tmp_path) + monkeypatch.setattr(module, "_hypothesis_service", lambda: service) + + assert module.record_hypothesis(request) == expected + assert service.requests == [request] + + +def test_record_hypothesis_accepts_mcp_json_mapping(tmp_path: Path) -> None: + request = _hypothesis_request(tmp_path) + + assert RecordHypothesisInput.model_validate(_json_request(request)) == request diff --git a/tests/test_program_templates.py b/tests/test_program_templates.py index 0cd9cb3..d07d9d4 100644 --- a/tests/test_program_templates.py +++ b/tests/test_program_templates.py @@ -60,6 +60,18 @@ def test_failure_pattern_miner_skill_is_packaged() -> None: assert "mine_failure_patterns" in skill.read_text(encoding="utf-8") +def test_hypothesis_former_skill_and_program_stop_before_candidate_editing() -> None: + root = Path(__file__).parents[1] + skill = root / "plugins/openflywheel/skills/hypothesis-former/SKILL.md" + program = files("ofw.preparation.templates").joinpath("base.md").read_text(encoding="utf-8") + + assert skill.is_file() + assert "record_hypothesis" in skill.read_text(encoding="utf-8") + assert "$hypothesis-former" in program + assert "stable hypothesis receipt" in program + assert "stop before candidate editing" in program + + def test_base_program_stops_after_repeated_managed_mcp_timeout() -> None: content = files("ofw.preparation.templates").joinpath("base.md").read_text(encoding="utf-8") content = " ".join(content.split()) diff --git a/tests/test_safe_file.py b/tests/test_safe_file.py new file mode 100644 index 0000000..0d5e09f --- /dev/null +++ b/tests/test_safe_file.py @@ -0,0 +1,98 @@ +"""Shared descriptor-anchored immutable-file safety checks.""" + +from __future__ import annotations + +import os +from pathlib import Path + +import pytest + +from ofw.safe_file import ( + SafeFileErrorCode, + SafeFileFailure, + open_directory, + publish_idempotent, + read_bounded, +) + + +def test_regular_file_reader_rejects_device() -> None: + with open_directory(Path("/dev")) as directory, pytest.raises(SafeFileFailure) as raised: + read_bounded(directory, "null", maximum_bytes=16, subject="device") + + assert raised.value.code is SafeFileErrorCode.INVALID_FILE + + +def test_directory_swap_is_detected_without_redirecting_publication(tmp_path: Path) -> None: + directory_path = tmp_path / "control" + moved_path = tmp_path / "moved" + replacement_path = tmp_path / "replacement" + directory_path.mkdir() + replacement_path.mkdir() + + with pytest.raises(SafeFileFailure) as raised, open_directory(directory_path) as directory: + directory_path.rename(moved_path) + replacement_path.rename(directory_path) + publish_idempotent( + directory, + "policy.json", + b"{}\n", + maximum_bytes=16, + subject="policy", + ) + + assert raised.value.code is SafeFileErrorCode.DIRECTORY_CHANGED + assert not (directory_path / "policy.json").exists() + assert (moved_path / "policy.json").read_bytes() == b"{}\n" + + +def test_failed_atomic_link_leaves_no_publication_or_temporary_file( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + directory_path = tmp_path / "control" + directory_path.mkdir() + + def crash( + source: str, + destination: str, + *, + src_dir_fd: int, + dst_dir_fd: int, + follow_symlinks: bool, + ) -> None: + del source, destination, src_dir_fd, dst_dir_fd, follow_symlinks + raise OSError("simulated crash boundary") + + monkeypatch.setattr(os, "link", crash) + + with ( + open_directory(directory_path) as directory, + pytest.raises(OSError, match="simulated crash boundary"), + ): + publish_idempotent( + directory, + "policy.json", + b"{}\n", + maximum_bytes=16, + subject="policy", + ) + + assert tuple(directory_path.iterdir()) == () + + +def test_new_content_is_rejected_before_any_file_write_when_oversized(tmp_path: Path) -> None: + directory_path = tmp_path / "control" + directory_path.mkdir() + + with open_directory(directory_path) as directory, pytest.raises(SafeFileFailure) as raised: + publish_idempotent( + directory, + "policy.json", + b"too large", + maximum_bytes=4, + subject="policy", + ) + + assert raised.value.code is SafeFileErrorCode.TOO_LARGE + assert tuple(directory_path.iterdir()) == () diff --git a/tests/test_typing.py b/tests/test_typing.py index adc8fc3..ac0da6e 100644 --- a/tests/test_typing.py +++ b/tests/test_typing.py @@ -83,3 +83,19 @@ def test_namespace_exports_workspace_preparation_contract() -> None: assert "PreparationStatus" in package.__all__ assert "PrepareWorkspaceInput" in package.__all__ assert "WorkspacePreparationObservation" in package.__all__ + + +def test_namespace_exports_policy_and_hypothesis_contracts_without_legacy_ownership() -> None: + expected = { + "ExperimentPolicySnapshot", + "FailurePatternReference", + "HarnessChangeTarget", + "HarnessHypothesis", + "HypothesisErrorCode", + "HypothesisId", + "HypothesisObservation", + "RecordHypothesisInput", + } + + assert expected <= set(package.__all__) + assert {"AssetAccess", "HarnessRevision", "HarnessRevisionId"}.isdisjoint(package.__all__) diff --git a/tests/test_workspace_preparation.py b/tests/test_workspace_preparation.py index 6116e98..fbcc622 100644 --- a/tests/test_workspace_preparation.py +++ b/tests/test_workspace_preparation.py @@ -22,6 +22,7 @@ WorkspacePreparationService, ) from ofw.preparation.harbor import HarborBaselineRunner +from ofw.preparation.policy import FileExperimentPolicyRepository from ofw.preparation.worktree import GitWorktreeGateway @@ -36,7 +37,13 @@ class _EnvironmentCapture(BaseModel): class _FailingRunner: def validate(self, request: PrepareWorkspaceInput) -> BaselineConfiguration: - return BaselineConfiguration(model="openai/gpt-5.4-mini", task_count=1) + return BaselineConfiguration( + model="openai/gpt-5.4-mini", + task_ids=("task-1",), + benchmark_config_digest="sha256:" + "1" * 64, + verifier="itsm-bench", + environment="itsm-bench", + ) def start(self, run: BaselineRun) -> int: raise PreparationFailure(PreparationErrorCode.LAUNCH_FAILED, "harbor") @@ -260,6 +267,15 @@ def test_prepare_workspace_creates_isolated_branch_commit_and_baseline( release=initialization_commit, session="itsm-hermes-demo", ) + policy = FileExperimentPolicyRepository().load(worktree, "itsm-hermes-demo") + assert policy.base_commit == ready.base_commit + assert policy.initialization_commit == ready.initialization_commit + assert policy.editable_paths == (Path("prompt.md"),) + assert policy.task_ids == ("tasks/task-pass", "tasks/task-fail") + assert policy.model == "openai/gpt-5.4-mini" + assert policy.concurrency == 1 + assert policy.max_retries == 0 + assert policy.controls_digest == policy.recomputed_controls_digest() assert (benchmark_root / "invocations.txt").read_text(encoding="utf-8") == "run\n" persisted_text = "\n".join( ( @@ -273,6 +289,10 @@ def test_prepare_workspace_creates_isolated_branch_commit_and_baseline( harness_root / ".git/ofw/preparations/itsm-hermes-demo/baseline.log" ).read_text(encoding="utf-8"), + ( + harness_root + / ".git/ofw/preparations/itsm-hermes-demo/policy.json" + ).read_text(encoding="utf-8"), ) ) assert "test-openai-key" not in persisted_text From 40216a9508c536a754c2821e8e61b384e2782b82 Mon Sep 17 00:00:00 2001 From: divo12 Date: Wed, 2 Sep 2026 18:46:47 +0530 Subject: [PATCH 02/25] fix: address hypothesis boundary reviews --- src/ofw/evaluation/failure_workspace.py | 64 +++++-- src/ofw/evolution/hypothesis.py | 9 +- src/ofw/evolution/hypothesis_repository.py | 2 +- src/ofw/preparation/contracts.py | 19 ++- src/ofw/preparation/harbor.py | 14 +- src/ofw/preparation/policy.py | 15 +- src/ofw/preparation/service.py | 187 ++++++++++++++++----- src/ofw/safe_file.py | 44 +++-- tests/test_experiment_policy.py | 55 +++++- tests/test_failure_workspace.py | 21 +-- tests/test_hypothesis.py | 26 ++- tests/test_safe_file.py | 22 +++ tests/test_workspace_preparation.py | 147 +++++++++++++++- 13 files changed, 501 insertions(+), 124 deletions(-) diff --git a/src/ofw/evaluation/failure_workspace.py b/src/ofw/evaluation/failure_workspace.py index 92e4c62..a768ae5 100644 --- a/src/ofw/evaluation/failure_workspace.py +++ b/src/ofw/evaluation/failure_workspace.py @@ -50,6 +50,7 @@ SafeFileFailure, publish_idempotent, read_bounded, + write_new_file, ) _IDENTIFIER_PATTERN = r"[A-Za-z0-9][A-Za-z0-9._:@/-]*" @@ -62,7 +63,6 @@ _IGNORE_CONTENT = "*\n" _WORKSPACE_MARKERS = ("PROGRAM.md", "experiment_config.yaml") _DIRECTORY_FLAGS = os.O_RDONLY | os.O_DIRECTORY | os.O_NOFOLLOW -_CREATE_FILE_FLAGS = os.O_WRONLY | os.O_CREAT | os.O_EXCL | os.O_NOFOLLOW Identifier = Annotated[ str, @@ -177,8 +177,9 @@ class FailureRecordObservation(StrictModel): class FailureArtifact(StrictModel): - schema_version: Literal[1] = 1 + schema_version: Literal[2] = 2 artifact_id: str = Field(pattern=_ARTIFACT_ID_PATTERN) + content_digest: str = Field(pattern=r"sha256:[0-9a-f]{64}") trace_id: Identifier task_id: Identifier verifier_id: Identifier @@ -199,9 +200,11 @@ class FailureArtifact(StrictModel): @model_validator(mode="after") def validate_domain_contract(self) -> FailureArtifact: try: - self.to_diagnosis() + diagnosis = self.to_diagnosis() except (FailureDiagnosisError, OutcomeEvaluationError) as error: raise ValueError("invalid failure artifact") from error + if self.content_digest != _diagnosis_content_digest(diagnosis): + raise ValueError("failure artifact content digest mismatch") return self @classmethod @@ -213,6 +216,7 @@ def from_diagnosis( outcome = diagnosis.outcome return cls( artifact_id=artifact_id, + content_digest=_diagnosis_content_digest(diagnosis), trace_id=outcome.trace_id.value, task_id=outcome.task_id.value, verifier_id=outcome.verifier_id.value, @@ -512,6 +516,50 @@ def _artifact_id(diagnosis: FailureDiagnosis) -> str: return str(uuid5(NAMESPACE_URL, identity)) +def _diagnosis_content_digest(diagnosis: FailureDiagnosis) -> str: + outcome = diagnosis.outcome + score = _required_score(outcome) + values = ( + "ofw.failure-artifact/1", + outcome.trace_id.value, + outcome.task_id.value, + outcome.verifier_id.value, + outcome.evaluated_at.isoformat(), + score.hex(), + str(len(outcome.evidence)), + *(reference.value for reference in outcome.evidence), + diagnosis.outcome_score_id.value, + diagnosis.evidence_status.value, + _optional_enum(diagnosis.issue_type), + diagnosis.expected_outcome, + diagnosis.actual_outcome, + _optional_identifier(diagnosis.critical_observation_id), + str(len(diagnosis.evidence_observation_ids)), + *(identifier.value for identifier in diagnosis.evidence_observation_ids), + _optional_text(diagnosis.root_cause), + _optional_text(diagnosis.counterfactual_action), + _optional_text(diagnosis.inconclusive_reason), + ) + digest = hashlib.sha256() + for value in values: + encoded = value.encode("utf-8") + digest.update(len(encoded).to_bytes(8, "big")) + digest.update(encoded) + return f"sha256:{digest.hexdigest()}" + + +def _optional_enum(value: FailureType | None) -> str: + return "0" if value is None else f"1{value.value}" + + +def _optional_identifier(value: ObservationId | None) -> str: + return "0" if value is None else f"1{value.value}" + + +def _optional_text(value: str | None) -> str: + return "0" if value is None else f"1{value}" + + def _prepared_root(root: Path) -> Path: resolved = _resolve_root(root) if not _is_prepared_root(resolved): @@ -624,7 +672,7 @@ def _existing_workspace_directories( def _write_ignore_file(directory: int) -> None: try: - _write_new_file(directory, ".gitignore", _IGNORE_CONTENT.encode("utf-8")) + write_new_file(directory, ".gitignore", _IGNORE_CONTENT.encode("utf-8")) except FileExistsError: return @@ -657,14 +705,6 @@ def _publish_or_validate( raise OSError("unsafe failure artifact") from None -def _write_new_file(directory: int, name: str, content: bytes) -> None: - descriptor = os.open(name, _CREATE_FILE_FLAGS, 0o600, dir_fd=directory) - with os.fdopen(descriptor, "wb") as stream: - stream.write(content) - stream.flush() - os.fsync(stream.fileno()) - - def _read_artifact(directory: int, artifact_id: str) -> FailureDiagnosisRecord: _require_artifact_id(artifact_id) try: diff --git a/src/ofw/evolution/hypothesis.py b/src/ofw/evolution/hypothesis.py index e95f457..5b66e7d 100644 --- a/src/ofw/evolution/hypothesis.py +++ b/src/ofw/evolution/hypothesis.py @@ -18,6 +18,7 @@ FailurePatternMiningService, MineFailurePatternsInput, ) +from ofw.preparation.contracts import contained_relative_path from ofw.preparation.policy import ExperimentPolicyErrorCode, ExperimentPolicyFailure _ARTIFACT_ID_PATTERN = r"[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}" @@ -59,7 +60,7 @@ class HarnessChangeTargetInput(StrictModel): @field_validator("relative_paths") @classmethod def validate_paths(cls, values: tuple[Path, ...]) -> tuple[Path, ...]: - normalized = tuple(_relative_path(value) for value in values) + normalized = tuple(contained_relative_path(value, "relative_paths") for value in values) if len(set(normalized)) != len(normalized): raise ValueError("relative_paths must be unique") return normalized @@ -422,12 +423,6 @@ def _observation(hypothesis: HarnessHypothesis, relative_path: Path) -> Hypothes ) -def _relative_path(path: Path) -> Path: - if path.is_absolute() or path == Path(".") or ".." in path.parts: - raise ValueError("relative_paths must be contained") - return path - - def _require_unique_patterns(pattern_ids: tuple[str, ...]) -> None: if len(set(pattern_ids)) != len(pattern_ids): raise ValueError("pattern_ids must be unique") diff --git a/src/ofw/evolution/hypothesis_repository.py b/src/ofw/evolution/hypothesis_repository.py index 1b96f4d..1e852a2 100644 --- a/src/ofw/evolution/hypothesis_repository.py +++ b/src/ofw/evolution/hypothesis_repository.py @@ -45,7 +45,7 @@ def validate_workspace( raise HypothesisFailure(HypothesisErrorCode.STALE_COMMIT, policy.experiment_id) if _git(prepared_root, "branch", "--show-current") != policy.branch_name: raise HypothesisFailure(HypothesisErrorCode.STALE_POLICY, policy.experiment_id) - if _git(prepared_root, "status", "--porcelain=v1", "--untracked-files=no"): + if _git(prepared_root, "status", "--porcelain=v1"): raise HypothesisFailure(HypothesisErrorCode.DIRTY_WORKSPACE, policy.experiment_id) for path in paths: _require_regular_target(prepared_root, path) diff --git a/src/ofw/preparation/contracts.py b/src/ofw/preparation/contracts.py index cf046cf..acb5851 100644 --- a/src/ofw/preparation/contracts.py +++ b/src/ofw/preparation/contracts.py @@ -93,12 +93,12 @@ def validate_absolute_paths(cls, value: Path) -> Path: @field_validator("harbor_config") @classmethod def validate_harbor_config(cls, value: Path) -> Path: - return _relative_path(value, "harbor_config") + return contained_relative_path(value, "harbor_config") @field_validator("editable_paths") @classmethod def validate_editable_paths(cls, values: tuple[Path, ...]) -> tuple[Path, ...]: - normalized = tuple(_relative_path(value, "editable_paths") for value in values) + normalized = tuple(contained_relative_path(value, "editable_paths") for value in values) if len(set(normalized)) != len(normalized): raise ValueError("editable_paths must be unique") return normalized @@ -234,7 +234,18 @@ def __init__(self, code: PreparationErrorCode, subject: str) -> None: super().__init__(f"{code.value}: {subject}") -def _relative_path(value: Path, field: str) -> Path: +def contained_relative_path(value: Path, field: str) -> Path: + _require_relative_shape(value, field) + _require_bounded_path_text(value, field) + return value + + +def _require_relative_shape(value: Path, field: str) -> None: if value.is_absolute() or ".." in value.parts or value == Path("."): raise ValueError(f"{field} must be a contained relative path") - return value + + +def _require_bounded_path_text(value: Path, field: str) -> None: + text = value.as_posix() + if "\x00" in text or len(text.encode("utf-8")) > 1024: + raise ValueError(f"{field} must be a bounded text path") diff --git a/src/ofw/preparation/harbor.py b/src/ofw/preparation/harbor.py index 776a660..d1d10e3 100644 --- a/src/ofw/preparation/harbor.py +++ b/src/ofw/preparation/harbor.py @@ -31,8 +31,8 @@ class _WireModel(BaseModel): class _HarborAgentWire(_WireModel): - name: str - model_name: str + name: str = Field(min_length=1, max_length=256) + model_name: str = Field(min_length=1, max_length=256) class _HarborTaskWire(_WireModel): @@ -78,8 +78,7 @@ class HarborBaselineRunner: def validate(self, request: PrepareWorkspaceInput) -> BaselineConfiguration: _executable(request.harbor_executable) config_path = _contained(request.benchmark_root, request.harbor_config) - config_content = _bounded_text(config_path, _MAX_CONFIG_BYTES) - config = _parse_config(config_path, config_content) + config, config_content = _parse_config(config_path) if len(config.tasks) != request.expected_task_count: raise PreparationFailure( PreparationErrorCode.TASK_COUNT_MISMATCH, @@ -207,10 +206,11 @@ def _contained(root: Path, relative: Path) -> Path: return resolved -def _parse_config(path: Path, content: str) -> _HarborConfigWire: +def _parse_config(path: Path) -> tuple[_HarborConfigWire, str]: try: - return _HarborConfigWire.model_validate_json(content) - except (OSError, ValidationError, ValueError) as error: + content = _bounded_text(path, _MAX_CONFIG_BYTES) + return _HarborConfigWire.model_validate_json(content), content + except (OSError, UnicodeError, ValidationError, ValueError) as error: raise PreparationFailure( PreparationErrorCode.INVALID_HARBOR_CONFIG, path.name, diff --git a/src/ofw/preparation/policy.py b/src/ofw/preparation/policy.py index ef9b6ac..5a3cdcb 100644 --- a/src/ofw/preparation/policy.py +++ b/src/ofw/preparation/policy.py @@ -23,6 +23,7 @@ PreparedGitWorkspace, PrepareWorkspaceInput, StrictModel, + contained_relative_path, ) from ofw.safe_file import ( SafeFileErrorCode, @@ -32,7 +33,7 @@ read_bounded, ) -_POLICY_LIMIT_BYTES = 64 * 1024 +_POLICY_LIMIT_BYTES = 256 * 1024 _COMMIT_PATTERN = r"[0-9a-f]{40}" _DIGEST_PATTERN = r"sha256:[0-9a-f]{64}" _TASK_ID_PATTERN = r"[^\x00]+" @@ -75,10 +76,9 @@ def validate_task_ids(cls, values: tuple[str, ...]) -> tuple[str, ...]: @field_validator("editable_paths") @classmethod def validate_editable_paths(cls, values: tuple[Path, ...]) -> tuple[Path, ...]: - for path in values: - _require_contained_editable_path(path) - _require_unique_editable_paths(values) - return values + normalized = tuple(contained_relative_path(path, "editable_paths") for path in values) + _require_unique_editable_paths(normalized) + return normalized class ExperimentPolicySnapshot(_ExperimentPolicyContent): @@ -309,11 +309,6 @@ def _digest(value: str) -> str: return f"sha256:{hashlib.sha256(value.encode('utf-8')).hexdigest()}" -def _require_contained_editable_path(path: Path) -> None: - if path.is_absolute() or path == Path(".") or ".." in path.parts: - raise ValueError("editable_paths must be contained relative paths") - - def _require_unique_editable_paths(paths: tuple[Path, ...]) -> None: if len(set(paths)) != len(paths): raise ValueError("editable_paths must be unique") diff --git a/src/ofw/preparation/service.py b/src/ofw/preparation/service.py index 8863ab3..1c52706 100644 --- a/src/ofw/preparation/service.py +++ b/src/ofw/preparation/service.py @@ -11,7 +11,7 @@ from datetime import UTC, datetime, timedelta from pathlib import Path -from pydantic import Field +from pydantic import Field, ValidationError from ofw.preparation.contracts import ( BaselineConfiguration, @@ -49,7 +49,12 @@ class _PreparationStateWire(StrictModel): program_path: Path job_path: Path log_path: Path - model: str + model: str = Field(min_length=1, max_length=256) + task_ids: tuple[str, ...] = Field(min_length=1, max_length=500) + benchmark_config_digest: str = Field(pattern=r"sha256:[0-9a-f]{64}") + verifier: str = Field(min_length=1, max_length=128) + environment: str = Field(min_length=1, max_length=128) + policy_published: bool = False process_id: int | None = Field(default=None, strict=True, ge=1) started_at: datetime deadline_at: datetime @@ -119,27 +124,26 @@ def _start( self._program, configuration, ) - _publish_policy(state_directory, request, prepared, configuration) - job_path = request.benchmark_root / "jobs" / request.experiment_id - log_path = state_directory / "baseline.log" - run = _baseline_run(request, prepared, job_path, log_path) - started_at = datetime.now(UTC) - state = _PreparationStateWire( - request_digest=digest, - phase=PreparationPhase.RUNNING, - branch_name=prepared.branch_name, - worktree_path=prepared.worktree_path, - base_commit=prepared.base_commit, - initialization_commit=prepared.initialization_commit, - program_path=prepared.program_path, - job_path=job_path, - log_path=log_path, - model=configuration.model, - process_id=None, - started_at=started_at, - deadline_at=started_at + timedelta(seconds=request.max_baseline_seconds), - ) + state = _initial_state(request, state_directory, digest, prepared, configuration) _write_state(state_directory, state) + return self._resume_start(request, state_directory, state) + + def _resume_start( + self, + request: PrepareWorkspaceInput, + state_directory: Path, + state: _PreparationStateWire, + ) -> WorkspacePreparationObservation: + if not state.policy_published: + _publish_policy( + state_directory, + request, + _prepared_from_state(state), + _configuration_from_state(state), + ) + state = _state_with_policy(state) + _write_state(state_directory, state) + run = _run_from_state(request, state) try: process_id = self._runner.start(run) except PreparationFailure as error: @@ -165,9 +169,9 @@ def _poll( PreparationErrorCode.REQUEST_CONFLICT, request.experiment_id, ) - terminal = _terminal_observation(request, state) - if terminal is not None: - return terminal + continuation = self._continue_existing(request, state_directory, state) + if continuation is not None: + return continuation run = _run_from_state(request, state) summary = self._runner.summarize(run) if summary is not None: @@ -184,6 +188,19 @@ def _poll( _write_state(state_directory, failed) return _persisted_failure_observation(request, failed) + def _continue_existing( + self, + request: PrepareWorkspaceInput, + state_directory: Path, + state: _PreparationStateWire, + ) -> WorkspacePreparationObservation | None: + terminal = _terminal_observation(request, state) + if terminal is not None: + return terminal + if state.process_id is None: + return self._resume_start(request, state_directory, state) + return None + def _directory(path: Path, field: str) -> Path: try: @@ -213,6 +230,39 @@ def _publish_policy( else PreparationErrorCode.POLICY_WRITE_FAILED ) raise PreparationFailure(code, request.experiment_id) from None + except (ValidationError, ValueError): + raise PreparationFailure( + PreparationErrorCode.POLICY_WRITE_FAILED, + request.experiment_id, + ) from None + + +def _initial_state( + request: PrepareWorkspaceInput, + state_directory: Path, + digest: str, + prepared: PreparedGitWorkspace, + configuration: BaselineConfiguration, +) -> _PreparationStateWire: + started_at = datetime.now(UTC) + return _PreparationStateWire( + request_digest=digest, + phase=PreparationPhase.RUNNING, + branch_name=prepared.branch_name, + worktree_path=prepared.worktree_path, + base_commit=prepared.base_commit, + initialization_commit=prepared.initialization_commit, + program_path=prepared.program_path, + job_path=request.benchmark_root / "jobs" / request.experiment_id, + log_path=state_directory / "baseline.log", + model=configuration.model, + task_ids=configuration.task_ids, + benchmark_config_digest=configuration.benchmark_config_digest, + verifier=configuration.verifier, + environment=configuration.environment, + started_at=started_at, + deadline_at=started_at + timedelta(seconds=request.max_baseline_seconds), + ) def _compose_program(base: str, itsm: str) -> str: @@ -267,24 +317,6 @@ def _write_state(state_directory: Path, state: _PreparationStateWire) -> None: os.replace(temporary_path, path) -def _baseline_run( - request: PrepareWorkspaceInput, - prepared: PreparedGitWorkspace, - job_path: Path, - log_path: Path, -) -> BaselineRun: - return BaselineRun( - experiment_id=request.experiment_id, - benchmark_root=request.benchmark_root, - harbor_executable=request.harbor_executable, - harbor_config=request.benchmark_root / request.harbor_config, - job_path=job_path, - log_path=log_path, - worktree_path=prepared.worktree_path, - initialization_commit=prepared.initialization_commit, - ) - - def _run_from_state( request: PrepareWorkspaceInput, state: _PreparationStateWire, @@ -301,6 +333,26 @@ def _run_from_state( ) +def _prepared_from_state(state: _PreparationStateWire) -> PreparedGitWorkspace: + return PreparedGitWorkspace( + branch_name=state.branch_name, + worktree_path=state.worktree_path, + base_commit=state.base_commit, + initialization_commit=state.initialization_commit, + program_path=state.program_path, + ) + + +def _configuration_from_state(state: _PreparationStateWire) -> BaselineConfiguration: + return BaselineConfiguration( + model=state.model, + task_ids=state.task_ids, + benchmark_config_digest=state.benchmark_config_digest, + verifier=state.verifier, + environment=state.environment, + ) + + def _ready_state( state: _PreparationStateWire, summary: BaselineSummary, @@ -337,6 +389,11 @@ def _terminal_state( job_path=state.job_path, log_path=state.log_path, model=state.model, + task_ids=state.task_ids, + benchmark_config_digest=state.benchmark_config_digest, + verifier=state.verifier, + environment=state.environment, + policy_published=state.policy_published, process_id=state.process_id, started_at=state.started_at, deadline_at=state.deadline_at, @@ -376,12 +433,40 @@ def _state_with_process( job_path=state.job_path, log_path=state.log_path, model=state.model, + task_ids=state.task_ids, + benchmark_config_digest=state.benchmark_config_digest, + verifier=state.verifier, + environment=state.environment, + policy_published=state.policy_published, process_id=process_id, started_at=state.started_at, deadline_at=state.deadline_at, ) +def _state_with_policy(state: _PreparationStateWire) -> _PreparationStateWire: + return _PreparationStateWire( + request_digest=state.request_digest, + phase=state.phase, + branch_name=state.branch_name, + worktree_path=state.worktree_path, + base_commit=state.base_commit, + initialization_commit=state.initialization_commit, + program_path=state.program_path, + job_path=state.job_path, + log_path=state.log_path, + model=state.model, + task_ids=state.task_ids, + benchmark_config_digest=state.benchmark_config_digest, + verifier=state.verifier, + environment=state.environment, + policy_published=True, + process_id=state.process_id, + started_at=state.started_at, + deadline_at=state.deadline_at, + ) + + def _running_observation( request: PrepareWorkspaceInput, state: _PreparationStateWire, @@ -486,6 +571,15 @@ def _failure_observation( def _root_cause(code: PreparationErrorCode) -> str: + if code is PreparationErrorCode.POLICY_CONFLICT: + return "The experiment already has different immutable policy bytes." + specific = _specific_root_cause(code) + if specific is not None: + return specific + return "A validated workspace, Git, Harbor, or result boundary rejected the request." + + +def _specific_root_cause(code: PreparationErrorCode) -> str | None: if code is PreparationErrorCode.REQUEST_CONFLICT: return "The preparation ID is already bound to different canonical inputs." if code is PreparationErrorCode.TASK_COUNT_MISMATCH: @@ -494,7 +588,7 @@ def _root_cause(code: PreparationErrorCode) -> str: return "One or more required credential environment variables are absent." if code is PreparationErrorCode.BASELINE_TIMEOUT: return "The Harbor job did not publish a terminal result before its deadline." - return "A validated workspace, Git, Harbor, or result boundary rejected the request." + return None def _retry(code: PreparationErrorCode) -> str: @@ -502,6 +596,8 @@ def _retry(code: PreparationErrorCode) -> str: return "Poll the identical request after 30 seconds." if code is PreparationErrorCode.BASELINE_TIMEOUT: return "Inspect the bounded baseline log, then use a new experiment ID if rerunning." + if code is PreparationErrorCode.POLICY_CONFLICT: + return "Do not retry this experiment ID with different policy content." return "Correct the reported configuration boundary, then retry without forcing Git state." @@ -510,6 +606,7 @@ def _stop_when(code: PreparationErrorCode) -> str: PreparationErrorCode.BRANCH_EXISTS, PreparationErrorCode.WORKTREE_EXISTS, PreparationErrorCode.REQUEST_CONFLICT, + PreparationErrorCode.POLICY_CONFLICT, ): - return "Stop until the existing experiment ownership is resolved explicitly." + return "Stop and use a new experiment ID unless the existing ownership is accepted." return "Stop if correction requires deleting, resetting, or overwriting user-owned data." diff --git a/src/ofw/safe_file.py b/src/ofw/safe_file.py index fef367b..60be33a 100644 --- a/src/ofw/safe_file.py +++ b/src/ofw/safe_file.py @@ -36,14 +36,8 @@ def __init__(self, code: SafeFileErrorCode, subject: str) -> None: @contextmanager def open_directory(path: Path) -> Iterator[int]: """Open one directory without following links and recheck its identity on exit.""" - descriptor = os.open(path, _DIRECTORY_FLAGS) - expected = _descriptor_identity(descriptor) - try: - _require_path_identity(path, expected) + with open_directory_chain(path, (), create=False) as descriptor: yield descriptor - _require_path_identity(path, expected) - finally: - os.close(descriptor) @contextmanager @@ -54,15 +48,18 @@ def open_directory_chain( create: bool, ) -> Iterator[int]: """Open a contained child chain and recheck every link before and after use.""" - root_descriptor = os.open(root, _DIRECTORY_FLAGS) - descriptors = [root_descriptor] - expected_root = _descriptor_identity(root_descriptor) + anchor, existing_parts = _absolute_directory_parts(root) + all_parts = existing_parts + parts + anchor_descriptor = os.open(anchor, _DIRECTORY_FLAGS) + descriptors = [anchor_descriptor] + expected_anchor = _descriptor_identity(anchor_descriptor) try: - for part in parts: - descriptors.append(_open_child(descriptors[-1], part, create=create)) - _require_chain(root, parts, descriptors, expected_root) + for index, part in enumerate(all_parts): + child_create = create and index >= len(existing_parts) + descriptors.append(_open_child(descriptors[-1], part, create=child_create)) + _require_chain(anchor, all_parts, descriptors, expected_anchor) yield descriptors[-1] - _require_chain(root, parts, descriptors, expected_root) + _require_chain(anchor, all_parts, descriptors, expected_anchor) finally: for descriptor in reversed(descriptors): os.close(descriptor) @@ -114,7 +111,12 @@ def _open_child(parent: int, name: str, *, create: bool) -> int: _require_leaf(name) if create: _mkdir_if_missing(parent, name) - return os.open(name, _DIRECTORY_FLAGS, dir_fd=parent) + try: + return os.open(name, _DIRECTORY_FLAGS, dir_fd=parent) + except FileNotFoundError: + raise + except OSError: + raise SafeFileFailure(SafeFileErrorCode.INVALID_FILE, name) from None def _mkdir_if_missing(parent: int, name: str) -> None: @@ -158,7 +160,7 @@ def _publish_new_file(directory: int, name: str, content: bytes) -> None: temporary_name = f".ofw-{uuid4().hex}.tmp" published = False try: - _write_new_file(directory, temporary_name, content) + write_new_file(directory, temporary_name, content) os.link( temporary_name, name, @@ -173,7 +175,9 @@ def _publish_new_file(directory: int, name: str, content: bytes) -> None: os.fsync(directory) -def _write_new_file(directory: int, name: str, content: bytes) -> None: +def write_new_file(directory: int, name: str, content: bytes) -> None: + """Create and fsync one exclusive no-follow file under a directory descriptor.""" + _require_leaf(name) descriptor = os.open(name, _CREATE_FILE_FLAGS, 0o600, dir_fd=directory) with os.fdopen(descriptor, "wb") as stream: stream.write(content) @@ -221,3 +225,9 @@ def _descriptor_identity(descriptor: int) -> tuple[int, int]: def _stat_identity(value: os.stat_result) -> tuple[int, int]: return value.st_dev, value.st_ino + + +def _absolute_directory_parts(path: Path) -> tuple[Path, tuple[str, ...]]: + if not path.is_absolute(): + raise SafeFileFailure(SafeFileErrorCode.INVALID_FILE, "directory") + return Path(path.anchor), path.parts[1:] diff --git a/tests/test_experiment_policy.py b/tests/test_experiment_policy.py index 2e76153..8a99594 100644 --- a/tests/test_experiment_policy.py +++ b/tests/test_experiment_policy.py @@ -121,6 +121,46 @@ def test_policy_is_derived_from_validated_preparation_inputs(tmp_path: Path) -> assert policy.controls_digest == policy.recomputed_controls_digest() +def test_maximum_valid_policy_fits_its_publication_bound(tmp_path: Path) -> None: + request = PrepareWorkspaceInput( + experiment_id="maximum", + harness_root=tmp_path / "harness", + base_ref="HEAD", + worktree_parent=tmp_path, + benchmark_root=tmp_path / "benchmark", + harbor_executable=tmp_path / "harbor", + harbor_config=Path("config.json"), + expected_task_count=500, + editable_paths=tuple(Path(f"{index:02d}/" + "p" * 1021) for index in range(50)), + goal="g" * 2000, + quality_target=1.0, + max_iterations=100, + no_improvement_limit=100, + max_baseline_seconds=172800, + ) + prepared = PreparedGitWorkspace( + branch_name="ofw/maximum", + worktree_path=tmp_path / "worktree", + base_commit="1" * 40, + initialization_commit="2" * 40, + program_path=tmp_path / "worktree/PROGRAM.md", + ) + baseline = BaselineConfiguration( + model="m" * 256, + task_ids=tuple(f"{index:03d}-" + "t" * 252 for index in range(500)), + benchmark_config_digest="sha256:" + "3" * 64, + verifier="v" * 128, + environment="e" * 128, + ) + policy = build_experiment_policy(request, prepared, baseline) + control = tmp_path / "common/ofw/preparations/maximum" + control.mkdir(parents=True) + + published = FileExperimentPolicyRepository().publish(control, policy) + + assert published.stat().st_size <= 256 * 1024 + + def test_policy_schema_rejects_extra_fields_and_tampered_controls(tmp_path: Path) -> None: _, policy = _snapshot(tmp_path) payload = policy.model_dump_json() @@ -154,6 +194,19 @@ def test_policy_publish_is_atomic_idempotent_and_conflict_detecting(tmp_path: Pa assert raised.value.code is ExperimentPolicyErrorCode.POLICY_CONFLICT +def test_policy_load_rejects_a_mismatched_experiment(tmp_path: Path) -> None: + root, policy = _snapshot(tmp_path) + control = root / ".git/ofw/preparations/experiment-two" + control.mkdir(parents=True) + repository = FileExperimentPolicyRepository() + repository.publish(control, policy) + + with pytest.raises(ExperimentPolicyFailure) as raised: + repository.load(root, "experiment-two") + + assert raised.value.code is ExperimentPolicyErrorCode.POLICY_INVALID + + @pytest.mark.parametrize("kind", ("symlink", "fifo")) def test_policy_rejects_non_regular_existing_file(tmp_path: Path, kind: str) -> None: root, policy = _snapshot(tmp_path) @@ -180,7 +233,7 @@ def test_policy_reload_rejects_missing_oversized_and_invalid_files(tmp_path: Pat assert missing.value.code is ExperimentPolicyErrorCode.POLICY_SNAPSHOT_REQUIRED path = control / "policy.json" - path.write_bytes(b"x" * (64 * 1024 + 1)) + path.write_bytes(b"x" * (256 * 1024 + 1)) with pytest.raises(ExperimentPolicyFailure) as oversized: repository.load(root, "experiment-one") assert oversized.value.code is ExperimentPolicyErrorCode.POLICY_TOO_LARGE diff --git a/tests/test_failure_workspace.py b/tests/test_failure_workspace.py index cc2d700..da7543b 100644 --- a/tests/test_failure_workspace.py +++ b/tests/test_failure_workspace.py @@ -186,24 +186,9 @@ def _oversized_curation() -> FailureCuration: def _expected_artifact(artifact_id: str) -> FailureArtifact: - return FailureArtifact( - artifact_id=artifact_id, - trace_id="trace-1", - task_id="task-1", - verifier_id="itsm-bench@v1", - evaluated_at=_EVALUATED_AT, - normalized_score=0.0, - outcome_score_id="outcome-score-1", - outcome_evidence=("harbor://trial-1/verifier/result",), - evidence_status=FailureEvidenceStatus.SUPPORTED, - issue_type=FailureType.CONTROL_FLOW_FAILURE, - expected_outcome="Incident INC-123 is closed.", - actual_outcome="Incident INC-123 remains open.", - critical_observation_id="observation-7", - evidence_observation_ids=("observation-7", "observation-9"), - root_cause="The agent finalized before reading state.", - counterfactual_action="Read the incident state before finalizing.", - inconclusive_reason=None, + return FailureArtifact.from_diagnosis( + artifact_id, + _request(Path("/")).to_diagnosis(), ) diff --git a/tests/test_hypothesis.py b/tests/test_hypothesis.py index 37cab97..cbe2878 100644 --- a/tests/test_hypothesis.py +++ b/tests/test_hypothesis.py @@ -304,7 +304,10 @@ def test_hypothesis_fails_closed_on_inexact_evidence(tmp_path: Path, case: str) assert raised.value.code is expected -@pytest.mark.parametrize("drift", ("commit", "dirty", "branch", "target-symlink")) +@pytest.mark.parametrize( + "drift", + ("commit", "dirty", "untracked", "branch", "target-symlink"), +) def test_hypothesis_rejects_stale_or_drifted_workspace(tmp_path: Path, drift: str) -> None: root, commit = _workspace(tmp_path) artifacts = (_diagnosis(root, "1"), _diagnosis(root, "2")) @@ -315,6 +318,8 @@ def test_hypothesis_rejects_stale_or_drifted_workspace(tmp_path: Path, drift: st _git(root, "commit", "-qm", "move head") elif drift == "dirty": (root / "prompt.md").write_text("dirty\n", encoding="utf-8") + elif drift == "untracked": + (root / "untracked.txt").write_text("not prepared\n", encoding="utf-8") elif drift == "branch": _git(root, "branch", "-m", "wrong-branch") else: @@ -332,6 +337,25 @@ def test_hypothesis_rejects_stale_or_drifted_workspace(tmp_path: Path, drift: st } +def test_hypothesis_rejects_semantically_tampered_diagnosis(tmp_path: Path) -> None: + root, commit = _workspace(tmp_path) + artifacts = (_diagnosis(root, "1"), _diagnosis(root, "2")) + path = root / ".workspace/failures" / f"{artifacts[0]}.json" + content = path.read_text(encoding="utf-8") + path.write_text( + content.replace( + "The incident remains open.", + "The incident was altered after recording.", + ), + encoding="utf-8", + ) + + with pytest.raises(HypothesisFailure) as raised: + _service().record(_request(root, commit, artifacts)) + + assert raised.value.code is HypothesisErrorCode.PATTERN_EVIDENCE_MISMATCH + + @pytest.mark.parametrize("path", (Path("tools.py"), Path("PROGRAM.md"), Path("prompt.md/child"))) def test_hypothesis_uses_exact_editable_allowlist_and_freezes_everything_else( tmp_path: Path, diff --git a/tests/test_safe_file.py b/tests/test_safe_file.py index 0d5e09f..2cdba9d 100644 --- a/tests/test_safe_file.py +++ b/tests/test_safe_file.py @@ -23,6 +23,28 @@ def test_regular_file_reader_rejects_device() -> None: assert raised.value.code is SafeFileErrorCode.INVALID_FILE +def test_regular_file_reader_rejects_oversized_content(tmp_path: Path) -> None: + directory_path = tmp_path / "control" + directory_path.mkdir() + (directory_path / "policy.json").write_bytes(b"oversized") + + with open_directory(directory_path) as directory, pytest.raises(SafeFileFailure) as raised: + read_bounded(directory, "policy.json", maximum_bytes=4, subject="policy") + + assert raised.value.code is SafeFileErrorCode.TOO_LARGE + + +def test_directory_open_rejects_a_symlinked_ancestor(tmp_path: Path) -> None: + real_parent = tmp_path / "real" + directory = real_parent / "control" + directory.mkdir(parents=True) + linked_parent = tmp_path / "linked" + linked_parent.symlink_to(real_parent, target_is_directory=True) + + with pytest.raises((OSError, SafeFileFailure)), open_directory(linked_parent / "control"): + pytest.fail("symlinked ancestor was followed") + + def test_directory_swap_is_detected_without_redirecting_publication(tmp_path: Path) -> None: directory_path = tmp_path / "control" moved_path = tmp_path / "moved" diff --git a/tests/test_workspace_preparation.py b/tests/test_workspace_preparation.py index fbcc622..e270a2c 100644 --- a/tests/test_workspace_preparation.py +++ b/tests/test_workspace_preparation.py @@ -22,7 +22,12 @@ WorkspacePreparationService, ) from ofw.preparation.harbor import HarborBaselineRunner -from ofw.preparation.policy import FileExperimentPolicyRepository +from ofw.preparation.policy import ( + ExperimentPolicyErrorCode, + ExperimentPolicyFailure, + ExperimentPolicySnapshot, + FileExperimentPolicyRepository, +) from ofw.preparation.worktree import GitWorktreeGateway @@ -358,6 +363,125 @@ def test_prepare_workspace_rejects_task_count_before_creating_branch(tmp_path: P assert result.error_code is PreparationErrorCode.TASK_COUNT_MISMATCH assert _git(harness_root, "branch", "--list", "ofw/itsm-hermes-demo") == "" + +@pytest.mark.parametrize( + "content", + (b"x" * (2 * 1024 * 1024 + 1), b"\xff"), + ids=("oversized", "invalid-utf8"), +) +def test_prepare_workspace_sanitizes_invalid_harbor_config_reads( + tmp_path: Path, + content: bytes, +) -> None: + harness_root = _harness_repository(tmp_path) + harbor = _fake_harbor(tmp_path) + benchmark_root, config = _benchmark_repository(tmp_path, harbor) + config.write_bytes(content) + worktree_parent = tmp_path / "worktrees" + worktree_parent.mkdir() + request = _request(harness_root, worktree_parent, benchmark_root, harbor, config) + + result = _service().prepare(request) + + assert result.error_code is PreparationErrorCode.INVALID_HARBOR_CONFIG + assert _git(harness_root, "branch", "--list", "ofw/itsm-hermes-demo") == "" + + +def test_prepare_workspace_rejects_oversized_model_before_creating_branch(tmp_path: Path) -> None: + harness_root = _harness_repository(tmp_path) + harbor = _fake_harbor(tmp_path) + benchmark_root, config = _benchmark_repository(tmp_path, harbor) + config.write_text( + config.read_text(encoding="utf-8").replace( + "openai/gpt-5.4-mini", + "m" * 257, + ), + encoding="utf-8", + ) + worktree_parent = tmp_path / "worktrees" + worktree_parent.mkdir() + request = _request(harness_root, worktree_parent, benchmark_root, harbor, config) + + result = _service().prepare(request) + + assert result.error_code is PreparationErrorCode.INVALID_HARBOR_CONFIG + assert _git(harness_root, "branch", "--list", "ofw/itsm-hermes-demo") == "" + + +def test_policy_publication_failure_resumes_without_recreating_git_workspace( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + harness_root = _harness_repository(tmp_path) + harbor = _fake_harbor(tmp_path) + benchmark_root, config = _benchmark_repository(tmp_path, harbor) + worktree_parent = tmp_path / "worktrees" + worktree_parent.mkdir() + request = _request(harness_root, worktree_parent, benchmark_root, harbor, config) + _credentials(monkeypatch) + original = FileExperimentPolicyRepository.publish + attempts = 0 + + def fail_once( + repository: FileExperimentPolicyRepository, + control_directory: Path, + policy: ExperimentPolicySnapshot, + ) -> Path: + nonlocal attempts + attempts += 1 + if attempts == 1: + raise ExperimentPolicyFailure( + ExperimentPolicyErrorCode.POLICY_WRITE_FAILED, + request.experiment_id, + ) + return original(repository, control_directory, policy) + + monkeypatch.setattr(FileExperimentPolicyRepository, "publish", fail_once) + service = _service() + + first = service.prepare(request) + ready = _wait_until_ready(service, request) + + assert first.error_code is PreparationErrorCode.POLICY_WRITE_FAILED + assert ready.phase is PreparationPhase.READY + assert attempts == 2 + assert (benchmark_root / "invocations.txt").read_text(encoding="utf-8") == "run\n" + + +def test_policy_conflict_remains_typed_and_non_retryable_after_git_preparation( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + harness_root = _harness_repository(tmp_path) + harbor = _fake_harbor(tmp_path) + benchmark_root, config = _benchmark_repository(tmp_path, harbor) + worktree_parent = tmp_path / "worktrees" + worktree_parent.mkdir() + request = _request(harness_root, worktree_parent, benchmark_root, harbor, config) + _credentials(monkeypatch) + + def conflict( + repository: FileExperimentPolicyRepository, + control_directory: Path, + policy: ExperimentPolicySnapshot, + ) -> Path: + del repository, control_directory, policy + raise ExperimentPolicyFailure( + ExperimentPolicyErrorCode.POLICY_CONFLICT, + request.experiment_id, + ) + + monkeypatch.setattr(FileExperimentPolicyRepository, "publish", conflict) + service = _service() + + first = service.prepare(request) + repeated = service.prepare(request) + + assert first.error_code is repeated.error_code is PreparationErrorCode.POLICY_CONFLICT + assert first.retry is not None and "do not retry" in first.retry.lower() + assert first.stop_when is not None and "new experiment" in first.stop_when.lower() + assert _git(harness_root, "branch", "--list", "ofw/itsm-hermes-demo") + def test_prepare_workspace_input_rejects_relative_roots(tmp_path: Path) -> None: with pytest.raises(ValidationError): PrepareWorkspaceInput( @@ -378,6 +502,27 @@ def test_prepare_workspace_input_rejects_relative_roots(tmp_path: Path) -> None: ) +def test_prepare_workspace_input_rejects_nul_and_oversized_editable_paths(tmp_path: Path) -> None: + for editable in (Path("bad\x00path"), Path("p" * 1025)): + with pytest.raises(ValidationError): + PrepareWorkspaceInput( + experiment_id="demo", + harness_root=tmp_path, + base_ref="HEAD", + worktree_parent=tmp_path, + benchmark_root=tmp_path, + harbor_executable=tmp_path / "harbor", + harbor_config=Path("config.json"), + expected_task_count=1, + editable_paths=(editable,), + goal="Improve.", + quality_target=1.0, + max_iterations=1, + no_improvement_limit=1, + max_baseline_seconds=60, + ) + + def test_prepare_workspace_input_accepts_json_path_strings(tmp_path: Path) -> None: config = PrepareWorkspaceInput.model_validate_json( f"""{{ From a588854a982c8e0d7076c43bbc68063912f76c2e Mon Sep 17 00:00:00 2001 From: divo12 Date: Wed, 2 Sep 2026 19:05:42 +0530 Subject: [PATCH 03/25] fix: close follow-up preparation reviews --- src/ofw/preparation/contracts.py | 9 +++++- src/ofw/preparation/service.py | 41 ++++++++++++++++++++++++--- tests/test_failure_workspace.py | 22 +++++++++++++-- tests/test_safe_file.py | 4 ++- tests/test_workspace_preparation.py | 43 +++++++++++++++++++++++++++++ 5 files changed, 110 insertions(+), 9 deletions(-) diff --git a/src/ofw/preparation/contracts.py b/src/ofw/preparation/contracts.py index acb5851..76ec9b0 100644 --- a/src/ofw/preparation/contracts.py +++ b/src/ofw/preparation/contracts.py @@ -28,13 +28,19 @@ def _numeric_float(value: object, field: str) -> float: return float(value) +def _bounded_path_input(value: object) -> object: + if isinstance(value, str) and ("\x00" in value or len(value.encode("utf-8")) > 1024): + raise ValueError("path input must be bounded text") + return value + + ExperimentIdentifier = Annotated[ str, Field(min_length=1, max_length=80, pattern=_EXPERIMENT_PATTERN), ] GitReference = Annotated[str, Field(min_length=1, max_length=256, pattern=_REF_PATTERN)] GoalText = Annotated[str, Field(min_length=1, max_length=2000)] -PathValue = Annotated[Path, Field(strict=False)] +PathValue = Annotated[Path, BeforeValidator(_bounded_path_input), Field(strict=False)] TaskCount = Annotated[int, Field(strict=True, ge=1, le=500)] IterationCount = Annotated[int, Field(strict=True, ge=1, le=100)] DurationSeconds = Annotated[int, Field(strict=True, ge=60, le=172800)] @@ -134,6 +140,7 @@ class PreparationErrorCode(StrEnum): PREPARATION_BUSY = "preparation_busy" GIT_FAILED = "git_failed" POLICY_CONFLICT = "policy_conflict" + POLICY_SNAPSHOT_REQUIRED = "policy_snapshot_required" POLICY_WRITE_FAILED = "policy_write_failed" diff --git a/src/ofw/preparation/service.py b/src/ofw/preparation/service.py index 1c52706..052d709 100644 --- a/src/ofw/preparation/service.py +++ b/src/ofw/preparation/service.py @@ -11,7 +11,7 @@ from datetime import UTC, datetime, timedelta from pathlib import Path -from pydantic import Field, ValidationError +from pydantic import BaseModel, ConfigDict, Field, ValidationError from ofw.preparation.contracts import ( BaselineConfiguration, @@ -39,7 +39,7 @@ class _PreparationStateWire(StrictModel): - schema_version: int = Field(default=1, ge=1, le=1) + schema_version: int = Field(default=2, ge=2, le=2) request_digest: str = Field(pattern=r"sha256:[0-9a-f]{64}") phase: PreparationPhase branch_name: str @@ -66,6 +66,12 @@ class _PreparationStateWire(StrictModel): error_code: PreparationErrorCode | None = None +class _StateVersionWire(BaseModel): + model_config = ConfigDict(extra="ignore", frozen=True, strict=True) + + schema_version: int = Field(strict=True) + + @dataclass(frozen=True, slots=True) class _TerminalCounts: terminal_trials: int | None @@ -296,8 +302,31 @@ def _read_state(state_directory: Path) -> _PreparationStateWire | None: if not path.exists(): return None try: - return _PreparationStateWire.model_validate_json(path.read_text(encoding="utf-8")) - except (OSError, ValueError) as error: + content = path.read_text(encoding="utf-8") + except OSError as error: + raise PreparationFailure( + PreparationErrorCode.INVALID_BASELINE_RESULT, + "state.json", + ) from error + return _parse_state(content) + + +def _parse_state(content: str) -> _PreparationStateWire: + try: + version = _StateVersionWire.model_validate_json(content) + except (ValidationError, ValueError) as error: + raise PreparationFailure( + PreparationErrorCode.INVALID_BASELINE_RESULT, + "state.json", + ) from error + if version.schema_version != 2: + raise PreparationFailure( + PreparationErrorCode.POLICY_SNAPSHOT_REQUIRED, + "state.json", + ) + try: + return _PreparationStateWire.model_validate_json(content) + except (ValidationError, ValueError) as error: raise PreparationFailure( PreparationErrorCode.INVALID_BASELINE_RESULT, "state.json", @@ -573,6 +602,8 @@ def _failure_observation( def _root_cause(code: PreparationErrorCode) -> str: if code is PreparationErrorCode.POLICY_CONFLICT: return "The experiment already has different immutable policy bytes." + if code is PreparationErrorCode.POLICY_SNAPSHOT_REQUIRED: + return "The existing preparation predates the canonical policy snapshot." specific = _specific_root_cause(code) if specific is not None: return specific @@ -598,6 +629,8 @@ def _retry(code: PreparationErrorCode) -> str: return "Inspect the bounded baseline log, then use a new experiment ID if rerunning." if code is PreparationErrorCode.POLICY_CONFLICT: return "Do not retry this experiment ID with different policy content." + if code is PreparationErrorCode.POLICY_SNAPSHOT_REQUIRED: + return "Use a new experiment ID and prepare again; do not infer policy from YAML." return "Correct the reported configuration boundary, then retry without forcing Git state." diff --git a/tests/test_failure_workspace.py b/tests/test_failure_workspace.py index da7543b..0c170ad 100644 --- a/tests/test_failure_workspace.py +++ b/tests/test_failure_workspace.py @@ -186,9 +186,25 @@ def _oversized_curation() -> FailureCuration: def _expected_artifact(artifact_id: str) -> FailureArtifact: - return FailureArtifact.from_diagnosis( - artifact_id, - _request(Path("/")).to_diagnosis(), + return FailureArtifact( + artifact_id=artifact_id, + content_digest="sha256:e712d9d50d334533c1dd8ec75011478d588392d806a1e99633e386aede9e5ba8", + trace_id="trace-1", + task_id="task-1", + verifier_id="itsm-bench@v1", + evaluated_at=_EVALUATED_AT, + normalized_score=0.0, + outcome_score_id="outcome-score-1", + outcome_evidence=("harbor://trial-1/verifier/result",), + evidence_status=FailureEvidenceStatus.SUPPORTED, + issue_type=FailureType.CONTROL_FLOW_FAILURE, + expected_outcome="Incident INC-123 is closed.", + actual_outcome="Incident INC-123 remains open.", + critical_observation_id="observation-7", + evidence_observation_ids=("observation-7", "observation-9"), + root_cause="The agent finalized before reading state.", + counterfactual_action="Read the incident state before finalizing.", + inconclusive_reason=None, ) diff --git a/tests/test_safe_file.py b/tests/test_safe_file.py index 2cdba9d..5b4a498 100644 --- a/tests/test_safe_file.py +++ b/tests/test_safe_file.py @@ -41,9 +41,11 @@ def test_directory_open_rejects_a_symlinked_ancestor(tmp_path: Path) -> None: linked_parent = tmp_path / "linked" linked_parent.symlink_to(real_parent, target_is_directory=True) - with pytest.raises((OSError, SafeFileFailure)), open_directory(linked_parent / "control"): + with pytest.raises(SafeFileFailure) as raised, open_directory(linked_parent / "control"): pytest.fail("symlinked ancestor was followed") + assert raised.value.code is SafeFileErrorCode.INVALID_FILE + def test_directory_swap_is_detected_without_redirecting_publication(tmp_path: Path) -> None: directory_path = tmp_path / "control" diff --git a/tests/test_workspace_preparation.py b/tests/test_workspace_preparation.py index e270a2c..b587d7f 100644 --- a/tests/test_workspace_preparation.py +++ b/tests/test_workspace_preparation.py @@ -523,6 +523,49 @@ def test_prepare_workspace_input_rejects_nul_and_oversized_editable_paths(tmp_pa ) +def test_prepare_workspace_input_rejects_oversized_raw_path_before_normalization( + tmp_path: Path, +) -> None: + redundant = "./" * 600 + "prompt.md" + payload = f"""{{ + "experiment_id": "demo", + "harness_root": "{tmp_path}", + "base_ref": "HEAD", + "worktree_parent": "{tmp_path}", + "benchmark_root": "{tmp_path}", + "harbor_executable": "{tmp_path / 'harbor'}", + "harbor_config": "config.json", + "expected_task_count": 1, + "editable_paths": ["{redundant}"], + "goal": "Improve.", + "quality_target": 1.0, + "max_iterations": 1, + "no_improvement_limit": 1, + "max_baseline_seconds": 60 +}}""" + + with pytest.raises(ValidationError): + PrepareWorkspaceInput.model_validate_json(payload) + + +def test_legacy_preparation_state_requires_a_fresh_preparation_id(tmp_path: Path) -> None: + harness_root = _harness_repository(tmp_path) + harbor = _fake_harbor(tmp_path) + benchmark_root, config = _benchmark_repository(tmp_path, harbor) + worktree_parent = tmp_path / "worktrees" + worktree_parent.mkdir() + request = _request(harness_root, worktree_parent, benchmark_root, harbor, config) + control = GitWorktreeGateway().control_directory(harness_root, request.experiment_id) + control.mkdir(parents=True) + (control / "state.json").write_text('{"schema_version":1}\n', encoding="utf-8") + + result = _service().prepare(request) + + assert result.error_code is PreparationErrorCode.POLICY_SNAPSHOT_REQUIRED + assert result.retry is not None and "new experiment id" in result.retry.lower() + assert _git(harness_root, "branch", "--list", "ofw/itsm-hermes-demo") == "" + + def test_prepare_workspace_input_accepts_json_path_strings(tmp_path: Path) -> None: config = PrepareWorkspaceInput.model_validate_json( f"""{{ From d6d19a7388837b545023b1615164e91ebc91c3fc Mon Sep 17 00:00:00 2001 From: divo12 Date: Wed, 2 Sep 2026 19:25:28 +0530 Subject: [PATCH 04/25] fix: type remaining preparation boundaries --- src/ofw/preparation/contracts.py | 4 +++- src/ofw/preparation/service.py | 2 +- tests/test_workspace_preparation.py | 37 +++++++++++++++++++++++++++++ 3 files changed, 41 insertions(+), 2 deletions(-) diff --git a/src/ofw/preparation/contracts.py b/src/ofw/preparation/contracts.py index 76ec9b0..4903e97 100644 --- a/src/ofw/preparation/contracts.py +++ b/src/ofw/preparation/contracts.py @@ -29,7 +29,9 @@ def _numeric_float(value: object, field: str) -> float: def _bounded_path_input(value: object) -> object: - if isinstance(value, str) and ("\x00" in value or len(value.encode("utf-8")) > 1024): + if isinstance(value, (str, Path)) and ( + "\x00" in str(value) or len(str(value).encode("utf-8")) > 1024 + ): raise ValueError("path input must be bounded text") return value diff --git a/src/ofw/preparation/service.py b/src/ofw/preparation/service.py index 052d709..f43c929 100644 --- a/src/ofw/preparation/service.py +++ b/src/ofw/preparation/service.py @@ -303,7 +303,7 @@ def _read_state(state_directory: Path) -> _PreparationStateWire | None: return None try: content = path.read_text(encoding="utf-8") - except OSError as error: + except (OSError, ValueError) as error: raise PreparationFailure( PreparationErrorCode.INVALID_BASELINE_RESULT, "state.json", diff --git a/tests/test_workspace_preparation.py b/tests/test_workspace_preparation.py index b587d7f..a0ffda8 100644 --- a/tests/test_workspace_preparation.py +++ b/tests/test_workspace_preparation.py @@ -548,6 +548,26 @@ def test_prepare_workspace_input_rejects_oversized_raw_path_before_normalization PrepareWorkspaceInput.model_validate_json(payload) +def test_prepare_workspace_input_rejects_oversized_direct_path(tmp_path: Path) -> None: + with pytest.raises(ValidationError): + PrepareWorkspaceInput( + experiment_id="demo", + harness_root=Path("/") / ("p" * 1025), + base_ref="HEAD", + worktree_parent=tmp_path, + benchmark_root=tmp_path, + harbor_executable=tmp_path / "harbor", + harbor_config=Path("config.json"), + expected_task_count=1, + editable_paths=(Path("prompt.md"),), + goal="Improve.", + quality_target=1.0, + max_iterations=1, + no_improvement_limit=1, + max_baseline_seconds=60, + ) + + def test_legacy_preparation_state_requires_a_fresh_preparation_id(tmp_path: Path) -> None: harness_root = _harness_repository(tmp_path) harbor = _fake_harbor(tmp_path) @@ -566,6 +586,23 @@ def test_legacy_preparation_state_requires_a_fresh_preparation_id(tmp_path: Path assert _git(harness_root, "branch", "--list", "ofw/itsm-hermes-demo") == "" +def test_invalid_utf8_preparation_state_returns_typed_failure(tmp_path: Path) -> None: + harness_root = _harness_repository(tmp_path) + harbor = _fake_harbor(tmp_path) + benchmark_root, config = _benchmark_repository(tmp_path, harbor) + worktree_parent = tmp_path / "worktrees" + worktree_parent.mkdir() + request = _request(harness_root, worktree_parent, benchmark_root, harbor, config) + control = GitWorktreeGateway().control_directory(harness_root, request.experiment_id) + control.mkdir(parents=True) + (control / "state.json").write_bytes(b"\xff") + + result = _service().prepare(request) + + assert result.error_code is PreparationErrorCode.INVALID_BASELINE_RESULT + assert _git(harness_root, "branch", "--list", "ofw/itsm-hermes-demo") == "" + + def test_prepare_workspace_input_accepts_json_path_strings(tmp_path: Path) -> None: config = PrepareWorkspaceInput.model_validate_json( f"""{{ From 48d7363d348d49baf6496a0100054ce246d35e4c Mon Sep 17 00:00:00 2001 From: divo12 Date: Thu, 3 Sep 2026 14:58:29 +0530 Subject: [PATCH 05/25] fix: bind hypotheses to curated evidence --- .../openflywheel/.codex-plugin/plugin.json | 2 +- .../openflywheel/program_templates/base.md | 4 +- .../skills/hypothesis-former/SKILL.md | 20 +- pyproject.toml | 2 +- src/ofw/evaluation/failure_curation.py | 117 ++++++-- src/ofw/evaluation/failure_patterns.py | 2 +- src/ofw/evaluation/failure_workspace.py | 264 ++++-------------- src/ofw/evolution/hypothesis.py | 115 +++++++- src/ofw/evolution/hypothesis_repository.py | 67 +++-- src/ofw/preparation/contracts.py | 6 +- src/ofw/preparation/policy.py | 6 +- src/ofw/preparation/service.py | 7 +- src/ofw/preparation/templates/base.md | 4 +- src/ofw/safe_file.py | 12 + tests/test_experiment_policy.py | 28 ++ tests/test_failure_workspace.py | 66 ----- tests/test_hypothesis.py | 213 +++++++++++++- tests/test_openflywheel_mcp.py | 5 + tests/test_plugin_packaging.py | 41 ++- tests/test_safe_file.py | 31 ++ 20 files changed, 646 insertions(+), 366 deletions(-) diff --git a/plugins/openflywheel/.codex-plugin/plugin.json b/plugins/openflywheel/.codex-plugin/plugin.json index 635d45e..6393349 100644 --- a/plugins/openflywheel/.codex-plugin/plugin.json +++ b/plugins/openflywheel/.codex-plugin/plugin.json @@ -1,6 +1,6 @@ { "name": "openflywheel", - "version": "0.7.1", + "version": "0.8.0", "description": "Prepare ITSM-bench workspaces and record evidence-backed harness hypotheses.", "author": { "name": "OpenFlyWheel" diff --git a/plugins/openflywheel/program_templates/base.md b/plugins/openflywheel/program_templates/base.md index 369dcdd..3a20d41 100644 --- a/plugins/openflywheel/program_templates/base.md +++ b/plugins/openflywheel/program_templates/base.md @@ -48,8 +48,8 @@ State the failure pattern, supporting trace and verifier evidence, proposed harn expected improvement, and possible regressions. Stop if the evidence cannot distinguish between materially different changes. -Use `$hypothesis-former` with only exact supported pattern and diagnosis receipt IDs, then call -`record_hypothesis`. Retain the stable hypothesis receipt and stop before candidate editing. +Use `$hypothesis-former` with one curation receipt and group ID plus every exact supported +pattern and diagnosis receipt ID in that group, then call `record_hypothesis`. Retain the stable hypothesis receipt and stop before candidate editing. Candidate editing requires a later package and must not begin in this prepared program. ## Package boundary diff --git a/plugins/openflywheel/skills/hypothesis-former/SKILL.md b/plugins/openflywheel/skills/hypothesis-former/SKILL.md index 64b96f7..d8b19ed 100644 --- a/plugins/openflywheel/skills/hypothesis-former/SKILL.md +++ b/plugins/openflywheel/skills/hypothesis-former/SKILL.md @@ -1,20 +1,22 @@ --- name: hypothesis-former -description: Record one OpenFlywheel harness hypothesis from exact supported failure-pattern and diagnosis receipts in a prepared experiment. Use after exact pattern mining and before any candidate edit; do not use with inconclusive evidence or to modify harness files. +description: Record one OpenFlywheel harness hypothesis from one curated failure group and its exact supported pattern and diagnosis receipts. Use after failure curation and before any candidate edit; do not use with incomplete evidence or to modify harness files. --- # Hypothesis Former -Use only selected pattern IDs and their exact diagnosis artifact IDs from the same prepared -experiment. Keep the global diagnosis set at fifty IDs or fewer. Read the compact diagnoses, -then propose one bounded statement, rationale, expected effect, regression-risk list, component -taxonomy, and one or more exact target paths. +Select one group from the retained failure-curation receipt. Use every diagnosis artifact ID in +that group, partitioned under its exact mined pattern IDs; do not add or omit IDs. Keep the global +diagnosis set at fifty IDs or fewer. Read the compact diagnoses, then propose one bounded +statement, rationale, expected effect, regression-risk list, component taxonomy, and one or more +exact target paths within the group's target component. Call `record_hypothesis` once with the prepared worktree, experiment ID, current initialization -commit, exact pattern-to-diagnosis assignments, and proposed target. The tool recomputes the -patterns and rejects missing, extra, misassigned, or inconclusive evidence. It also proves each -target is exactly present in the authoritative editable allowlist; `component_kind` describes the -change and never grants path access. +commit, curation ID, selected curation group ID, exact pattern-to-diagnosis assignments, and +proposed target. The tool reloads the curation, requires the complete selected group, recomputes +the patterns, and rejects missing, extra, misassigned, or inconclusive evidence. It also proves +each target is exactly present in the authoritative editable allowlist; `component_kind` must +match the curated component and never grants path access. Retain the returned hypothesis ID and artifact path. Stop before editing any harness file. On an MCP timeout, retry the identical request once because publication is idempotent; if the retry diff --git a/pyproject.toml b/pyproject.toml index a3db39c..ceb748f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "hatchling.build" [project] name = "openflywheel" -version = "0.7.1" +version = "0.8.0" description = "A governed self-improving agent harness" requires-python = ">=3.11" dependencies = [ diff --git a/src/ofw/evaluation/failure_curation.py b/src/ofw/evaluation/failure_curation.py index f5e2b2b..6ae94ec 100644 --- a/src/ofw/evaluation/failure_curation.py +++ b/src/ofw/evaluation/failure_curation.py @@ -16,11 +16,11 @@ from ofw.observability.langfuse.domain import ObservationId, ScoreId, TraceId _ARTIFACT_ID_PATTERN = r"^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$" -_PATTERN_KEY_PATTERN = r"[a-z0-9]+(?:-[a-z0-9]+)*" -_TEXT_PATTERN = r"[^\x00]+" +_PATTERN_KEY_PATTERN = r"^[a-z0-9]+(?:-[a-z0-9]+)*$" +_TEXT_PATTERN = r"^[^\x00]+$" ArtifactIdentifier = Annotated[str, Field(pattern=_ARTIFACT_ID_PATTERN)] -DigestValue = Annotated[str, Field(pattern=r"sha256:[0-9a-f]{64}")] +DigestValue = Annotated[str, Field(pattern=r"^sha256:[0-9a-f]{64}$")] PatternKey = Annotated[ str, Field(min_length=1, max_length=80, pattern=_PATTERN_KEY_PATTERN), @@ -292,6 +292,17 @@ class FailureCurationArtifact(StrictModel): groups: tuple[FailureGroupArtifact, ...] = Field(max_length=25) deferred: tuple[DeferredFailureArtifact, ...] = Field(max_length=50) + def require_valid_identity(self) -> None: + _require_group_artifact_identities(self.groups) + _require_complete_partition(self.source_artifact_ids, _artifact_assignment_ids(self)) + expected = _failure_curation_id( + self.source_artifact_ids, + tuple(group.group_id for group in self.groups), + _artifact_deferred_evidence(self.deferred), + ) + if self.curation_id != expected: + raise ValueError("curation_id does not match canonical curation content") + @classmethod def from_curation(cls, curation: FailureCuration) -> FailureCurationArtifact: return cls( @@ -490,36 +501,108 @@ def _group_id( item: FailureGroupInput, issue_type: FailureType, members: tuple[FailureGroupMember, ...], +) -> str: + evidence = tuple( + (member.artifact_id, member.artifact_digest.value) for member in members + ) + return _failure_group_id( + item.pattern_key, + item.title, + item.mechanism, + item.prevention, + item.target_component, + issue_type, + evidence, + ) + + +def _curation_id( + source_ids: tuple[str, ...], + groups: tuple[FailureGroup, ...], + deferred: tuple[DeferredFailure, ...], +) -> str: + deferred_evidence = tuple( + (item.source.artifact_id, item.source.artifact_digest.value, item.reason) + for item in deferred + ) + return _failure_curation_id( + source_ids, + tuple(group.id for group in groups), + deferred_evidence, + ) + + +def _failure_group_id( + pattern_key: str, + title: str, + mechanism: str, + prevention: str, + target_component: ComponentKind, + issue_type: FailureType, + members: tuple[tuple[str, str], ...], ) -> str: identity = "\0".join( ( "ofw.failure-group", - item.pattern_key, - item.title, - item.mechanism, - item.prevention, - item.target_component.value, + pattern_key, + title, + mechanism, + prevention, + target_component.value, issue_type.value, - *(f"{member.artifact_id}:{member.artifact_digest.value}" for member in members), + *(f"{artifact_id}:{digest}" for artifact_id, digest in members), ) ) return str(uuid5(NAMESPACE_URL, identity)) -def _curation_id( +def _require_group_artifact_identity(group: FailureGroupArtifact) -> None: + members = tuple((item.artifact_id, item.artifact_digest) for item in group.members) + expected = _failure_group_id( + group.pattern_key, + group.title, + group.mechanism, + group.prevention, + group.target_component, + group.issue_type, + members, + ) + if group.group_id != expected: + raise ValueError("group_id does not match canonical group content") + + +def _require_group_artifact_identities(groups: tuple[FailureGroupArtifact, ...]) -> None: + for group in groups: + _require_group_artifact_identity(group) + + +def _artifact_assignment_ids(artifact: FailureCurationArtifact) -> tuple[str, ...]: + grouped = tuple( + item.artifact_id for group in artifact.groups for item in group.members + ) + return grouped + tuple(item.source.artifact_id for item in artifact.deferred) + + +def _artifact_deferred_evidence( + deferred: tuple[DeferredFailureArtifact, ...], +) -> tuple[tuple[str, str, str], ...]: + return tuple( + (item.source.artifact_id, item.source.artifact_digest, item.reason) + for item in deferred + ) + + +def _failure_curation_id( source_ids: tuple[str, ...], - groups: tuple[FailureGroup, ...], - deferred: tuple[DeferredFailure, ...], + group_ids: tuple[str, ...], + deferred: tuple[tuple[str, str, str], ...], ) -> str: identity = "\0".join( ( "ofw.failure-curation", *source_ids, - *(group.id for group in groups), - *( - f"{item.source.artifact_id}:{item.source.artifact_digest.value}:{item.reason}" - for item in deferred - ), + *group_ids, + *(f"{artifact_id}:{digest}:{reason}" for artifact_id, digest, reason in deferred), ) ) return str(uuid5(NAMESPACE_URL, identity)) diff --git a/src/ofw/evaluation/failure_patterns.py b/src/ofw/evaluation/failure_patterns.py index 10a4f19..f4dd7e5 100644 --- a/src/ofw/evaluation/failure_patterns.py +++ b/src/ofw/evaluation/failure_patterns.py @@ -15,7 +15,7 @@ from ofw.evaluation.failure import FailureDiagnosis, FailureEvidenceStatus, FailureType _ARTIFACT_ID_PATTERN = r"^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$" -_PATTERN_ID_PATTERN = r"sha256:[0-9a-f]{64}" +_PATTERN_ID_PATTERN = r"^sha256:[0-9a-f]{64}$" _ABSOLUTE_PATH = re.compile(r"(?:/[\w.+-]+){2,}") _OPAQUE_ID = re.compile( r"\b(?:(?:request|trace|span|observation|score|session|run)(?:[_-]?id)?|rq)" diff --git a/src/ofw/evaluation/failure_workspace.py b/src/ofw/evaluation/failure_workspace.py index a768ae5..c65d47e 100644 --- a/src/ofw/evaluation/failure_workspace.py +++ b/src/ofw/evaluation/failure_workspace.py @@ -3,10 +3,7 @@ from __future__ import annotations import hashlib -import os import re -from collections.abc import Iterator -from contextlib import contextmanager from dataclasses import dataclass from datetime import datetime, timedelta from enum import StrEnum @@ -48,12 +45,13 @@ from ofw.safe_file import ( SafeFileErrorCode, SafeFileFailure, + open_child_directory, + open_directory_chain, publish_idempotent, read_bounded, - write_new_file, ) -_IDENTIFIER_PATTERN = r"[A-Za-z0-9][A-Za-z0-9._:@/-]*" +_IDENTIFIER_PATTERN = r"^[A-Za-z0-9][A-Za-z0-9._:@/-]*$" _ARTIFACT_ID_PATTERN = r"^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$" _ARTIFACT_ID = re.compile(_ARTIFACT_ID_PATTERN) _ARTIFACT_LIMIT_BYTES = 64 * 1024 @@ -62,7 +60,6 @@ _CURATION_DIRECTORY = "failure-curations" _IGNORE_CONTENT = "*\n" _WORKSPACE_MARKERS = ("PROGRAM.md", "experiment_config.yaml") -_DIRECTORY_FLAGS = os.O_RDONLY | os.O_DIRECTORY | os.O_NOFOLLOW Identifier = Annotated[ str, @@ -179,7 +176,7 @@ class FailureRecordObservation(StrictModel): class FailureArtifact(StrictModel): schema_version: Literal[2] = 2 artifact_id: str = Field(pattern=_ARTIFACT_ID_PATTERN) - content_digest: str = Field(pattern=r"sha256:[0-9a-f]{64}") + content_digest: str = Field(pattern=r"^sha256:[0-9a-f]{64}$") trace_id: Identifier task_id: Identifier verifier_id: Identifier @@ -270,13 +267,6 @@ class FailureArtifactReceipt: relative_path: Path -@dataclass(frozen=True, slots=True) -class _DirectoryChainIdentity: - root: tuple[int, int] - workspace: tuple[int, int] - artifacts: tuple[int, int] - - class FailureWorkspaceErrorCode(StrEnum): INVALID_WORKSPACE = "invalid_workspace" ARTIFACT_CONFLICT = "artifact_conflict" @@ -346,10 +336,18 @@ def read( ) -> tuple[FailureDiagnosisRecord, ...]: try: return self._read(root, artifact_ids) - except FailurePatternMiningError: - raise + except FileNotFoundError: + raise FailurePatternMiningError( + FailurePatternMiningErrorCode.ARTIFACT_NOT_FOUND, + _FAILURE_DIRECTORY, + ) from None except FailureWorkspaceFailure as error: raise _pattern_read_error(error) from None + except SafeFileFailure: + raise FailurePatternMiningError( + FailurePatternMiningErrorCode.INVALID_WORKSPACE, + _WORKSPACE_DIRECTORY, + ) from None except (OSError, RuntimeError, UnicodeError): raise FailurePatternMiningError( FailurePatternMiningErrorCode.READ_FAILED, @@ -362,16 +360,10 @@ def _read( artifact_ids: tuple[str, ...], ) -> tuple[FailureDiagnosisRecord, ...]: prepared_root = _prepared_root(root) - workspace, failures = _workspace_artifact_paths(prepared_root, _FAILURE_DIRECTORY) - directory_identity = _existing_workspace_directories( + with open_directory_chain( prepared_root, - workspace, - failures, - ) - with _artifact_directory_handle( - prepared_root, - _FAILURE_DIRECTORY, - directory_identity, + (_WORKSPACE_DIRECTORY, _FAILURE_DIRECTORY), + create=False, ) as directory: return tuple(_read_artifact(directory, artifact_id) for artifact_id in artifact_ids) @@ -382,8 +374,6 @@ class FileFailureCurationWorkspace: def load(self, root: Path, artifact_ids: tuple[str, ...]) -> tuple[FailureSource, ...]: try: return self._load(root, artifact_ids) - except FailureCurationFailure: - raise except FailureWorkspaceFailure as error: code = ( FailureCurationErrorCode.SOURCE_INVALID @@ -391,6 +381,11 @@ def load(self, root: Path, artifact_ids: tuple[str, ...]) -> tuple[FailureSource else FailureCurationErrorCode.INVALID_WORKSPACE ) raise FailureCurationFailure(code, error.subject) from None + except SafeFileFailure: + raise FailureCurationFailure( + FailureCurationErrorCode.INVALID_WORKSPACE, + _WORKSPACE_DIRECTORY, + ) from None except (OSError, RuntimeError, UnicodeError): raise FailureCurationFailure( FailureCurationErrorCode.SOURCE_INVALID, @@ -422,29 +417,19 @@ def _load( artifact_ids: tuple[str, ...], ) -> tuple[FailureSource, ...]: prepared_root = _prepared_root(root) - workspace, failures = _workspace_artifact_paths(prepared_root, _FAILURE_DIRECTORY) try: - identity = _existing_directory_identity(prepared_root, workspace, failures) - with _artifact_directory_handle( + with open_directory_chain( prepared_root, - _FAILURE_DIRECTORY, - identity, + (_WORKSPACE_DIRECTORY, _FAILURE_DIRECTORY), + create=False, ) as directory: return tuple( _read_failure_source(directory, artifact_id) for artifact_id in artifact_ids ) except FileNotFoundError: - missing = next( - ( - artifact_id - for artifact_id in artifact_ids - if not (failures / f"{artifact_id}.json").exists() - ), - artifact_ids[0], - ) raise FailureCurationFailure( FailureCurationErrorCode.SOURCE_NOT_FOUND, - missing, + artifact_ids[0], ) from None @@ -578,36 +563,33 @@ def _is_prepared_root(root: Path) -> bool: return all((root / name).is_file() for name in _WORKSPACE_MARKERS) -def _workspace_artifact_paths(root: Path, directory_name: str) -> tuple[Path, Path]: - workspace = root / _WORKSPACE_DIRECTORY - failures = workspace / directory_name - _require_contained(root, workspace.resolve(strict=False)) - _require_contained(root, failures.resolve(strict=False)) - _require_directory_if_present(workspace) - return workspace, failures - - def _store_artifact(root: Path, directory_name: str, artifact_id: str, content: bytes) -> Path: prepared_root = _prepared_root(root) - workspace, artifacts = _workspace_artifact_paths(prepared_root, directory_name) _validate_artifact_size(content) - identity = _prepare_workspace_directories(prepared_root, workspace, artifacts) - path = artifacts / f"{artifact_id}.json" - with _artifact_directory_handle(prepared_root, directory_name, identity) as directory: - _publish_or_validate(directory, path.name, content, artifact_id) - return path.relative_to(prepared_root) - - -def _require_directory_if_present(path: Path) -> None: - if path.exists() and not path.is_dir(): - _invalid_workspace(_WORKSPACE_DIRECTORY) - - -def _require_contained(root: Path, path: Path) -> None: + name = f"{artifact_id}.json" try: - path.relative_to(root) - except ValueError: - _invalid_workspace(_WORKSPACE_DIRECTORY) + with open_directory_chain( + prepared_root, + (_WORKSPACE_DIRECTORY,), + create=True, + ) as workspace: + publish_idempotent( + workspace, + ".gitignore", + _IGNORE_CONTENT.encode("utf-8"), + maximum_bytes=16, + subject=_WORKSPACE_DIRECTORY, + ) + with open_child_directory(workspace, directory_name, create=True) as directory: + _publish_or_validate(directory, name, content, artifact_id) + except SafeFileFailure as error: + code = ( + FailureWorkspaceErrorCode.INVALID_WORKSPACE + if error.code is SafeFileErrorCode.INVALID_FILE + else FailureWorkspaceErrorCode.WRITE_FAILED + ) + raise FailureWorkspaceFailure(code, artifact_id) from None + return Path(_WORKSPACE_DIRECTORY, directory_name, name) def _invalid_workspace(subject: str) -> Never: @@ -625,58 +607,6 @@ def _validate_artifact_size(content: bytes) -> None: ) -def _prepare_workspace_directories( - root: Path, - workspace: Path, - failures: Path, -) -> _DirectoryChainIdentity: - failures.mkdir(parents=True, exist_ok=True) - _require_contained(root, workspace.resolve(strict=True)) - _require_contained(root, failures.resolve(strict=True)) - with _directory_handle(workspace) as directory: - _require_directory_identity(directory, workspace) - _write_ignore_file(directory) - return _DirectoryChainIdentity( - root=_path_identity(root), - workspace=_path_identity(workspace), - artifacts=_path_identity(failures), - ) - - -def _existing_directory_identity( - root: Path, - workspace: Path, - artifacts: Path, -) -> _DirectoryChainIdentity: - _require_contained(root, workspace.resolve(strict=True)) - _require_contained(root, artifacts.resolve(strict=True)) - return _DirectoryChainIdentity( - root=_path_identity(root), - workspace=_path_identity(workspace), - artifacts=_path_identity(artifacts), - ) - - -def _existing_workspace_directories( - root: Path, - workspace: Path, - failures: Path, -) -> _DirectoryChainIdentity: - if not workspace.is_dir() or not failures.is_dir(): - raise FailurePatternMiningError( - FailurePatternMiningErrorCode.ARTIFACT_NOT_FOUND, - _FAILURE_DIRECTORY, - ) - return _existing_directory_identity(root, workspace, failures) - - -def _write_ignore_file(directory: int) -> None: - try: - write_new_file(directory, ".gitignore", _IGNORE_CONTENT.encode("utf-8")) - except FileExistsError: - return - - def _publish_or_validate( directory: int, name: str, @@ -769,99 +699,3 @@ def _read_existing(directory: int, name: str, artifact_id: str) -> bytes: artifact_id, ) from None raise OSError("unsafe failure artifact") from None - - -@contextmanager -def _directory_handle(path: Path) -> Iterator[int]: - descriptor = os.open(path, _DIRECTORY_FLAGS) - try: - yield descriptor - finally: - os.close(descriptor) - - -@contextmanager -def _child_directory_handle(parent: int, name: str) -> Iterator[int]: - descriptor = os.open(name, _DIRECTORY_FLAGS, dir_fd=parent) - try: - yield descriptor - finally: - os.close(descriptor) - - -@contextmanager -def _artifact_directory_handle( - root: Path, - directory_name: str, - expected: _DirectoryChainIdentity, -) -> Iterator[int]: - with ( - _directory_handle(root) as root_directory, - _child_directory_handle(root_directory, _WORKSPACE_DIRECTORY) as workspace, - _child_directory_handle(workspace, directory_name) as artifacts, - ): - _require_directory_chain( - root, - root_directory, - workspace, - directory_name, - artifacts, - expected, - ) - yield artifacts - _require_directory_chain( - root, - root_directory, - workspace, - directory_name, - artifacts, - expected, - ) - - -def _require_directory_chain( - root: Path, - root_directory: int, - workspace: int, - directory_name: str, - artifacts: int, - expected: _DirectoryChainIdentity, -) -> None: - _require_directory_identity(root_directory, root, expected.root) - _require_child_identity(root_directory, _WORKSPACE_DIRECTORY, workspace, expected.workspace) - _require_child_identity(workspace, directory_name, artifacts, expected.artifacts) - - -def _require_directory_identity( - descriptor: int, - path: Path, - expected: tuple[int, int] | None = None, -) -> None: - opened = _descriptor_identity(descriptor) - current = _path_identity(path) - if opened != current or (expected is not None and opened != expected): - raise OSError("workspace directory changed during failure recording") - - -def _require_child_identity( - parent: int, - name: str, - descriptor: int, - expected: tuple[int, int], -) -> None: - opened = _descriptor_identity(descriptor) - current = _stat_identity(os.stat(name, dir_fd=parent, follow_symlinks=False)) - if opened != current or opened != expected: - raise OSError("workspace directory changed during failure recording") - - -def _descriptor_identity(descriptor: int) -> tuple[int, int]: - return _stat_identity(os.fstat(descriptor)) - - -def _path_identity(path: Path) -> tuple[int, int]: - return _stat_identity(os.stat(path, follow_symlinks=False)) - - -def _stat_identity(value: os.stat_result) -> tuple[int, int]: - return value.st_dev, value.st_ino diff --git a/src/ofw/evolution/hypothesis.py b/src/ofw/evolution/hypothesis.py index 5b66e7d..34efa4f 100644 --- a/src/ofw/evolution/hypothesis.py +++ b/src/ofw/evolution/hypothesis.py @@ -7,11 +7,12 @@ from dataclasses import dataclass from enum import StrEnum from pathlib import Path -from typing import Annotated, Literal +from typing import Annotated, Literal, Protocol from pydantic import BaseModel, ConfigDict, Field, field_validator, model_validator from ofw.contracts import ComponentKind +from ofw.evaluation.failure_curation import FailureCurationArtifact, FailureGroupArtifact from ofw.evaluation.failure_patterns import ( FailurePatternMiningError, FailurePatternMiningObservation, @@ -19,13 +20,17 @@ MineFailurePatternsInput, ) from ofw.preparation.contracts import contained_relative_path -from ofw.preparation.policy import ExperimentPolicyErrorCode, ExperimentPolicyFailure +from ofw.preparation.policy import ( + ExperimentPolicyErrorCode, + ExperimentPolicyFailure, + ExperimentPolicySnapshot, +) -_ARTIFACT_ID_PATTERN = r"[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}" -_PATTERN_ID_PATTERN = r"sha256:[0-9a-f]{64}" +_ARTIFACT_ID_PATTERN = r"^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$" +_PATTERN_ID_PATTERN = r"^sha256:[0-9a-f]{64}$" _HYPOTHESIS_ID_PATTERN = re.compile(r"sha256:[0-9a-f]{64}") -_COMMIT_PATTERN = r"[0-9a-f]{40}" -_EXPERIMENT_PATTERN = r"[a-z0-9]+(?:-[a-z0-9]+)*" +_COMMIT_PATTERN = r"^[0-9a-f]{40}$" +_EXPERIMENT_PATTERN = r"^[a-z0-9]+(?:-[a-z0-9]+)*$" HypothesisTextValue = Annotated[str, Field(min_length=1, max_length=4000)] ArtifactIdsValue = Annotated[ tuple[Annotated[str, Field(pattern=_ARTIFACT_ID_PATTERN)], ...], @@ -70,6 +75,8 @@ class RecordHypothesisInput(StrictModel): workspace_root: Path = Field(strict=False) experiment_id: str = Field(min_length=1, max_length=80, pattern=_EXPERIMENT_PATTERN) source_commit: str = Field(pattern=_COMMIT_PATTERN) + curation_id: str = Field(pattern=_ARTIFACT_ID_PATTERN) + curation_group_id: str = Field(pattern=_ARTIFACT_ID_PATTERN) patterns: tuple[FailurePatternReferenceInput, ...] = Field( strict=False, min_length=1, @@ -146,6 +153,8 @@ class HarnessHypothesis: id: HypothesisId experiment_id: str source_commit: str + curation_id: str + curation_group_id: str patterns: tuple[FailurePatternReference, ...] statement: str rationale: str @@ -158,6 +167,8 @@ class _HypothesisContent(StrictModel): schema_version: Literal[1] = 1 experiment_id: str = Field(min_length=1, max_length=80, pattern=_EXPERIMENT_PATTERN) source_commit: str = Field(pattern=_COMMIT_PATTERN) + curation_id: str = Field(pattern=_ARTIFACT_ID_PATTERN) + curation_group_id: str = Field(pattern=_ARTIFACT_ID_PATTERN) patterns: tuple[FailurePatternReferenceInput, ...] = Field(min_length=1, max_length=50) statement: HypothesisTextValue rationale: HypothesisTextValue @@ -175,6 +186,8 @@ def from_hypothesis(cls, hypothesis: HarnessHypothesis) -> HypothesisArtifact: hypothesis_id=hypothesis.id.value, experiment_id=hypothesis.experiment_id, source_commit=hypothesis.source_commit, + curation_id=hypothesis.curation_id, + curation_group_id=hypothesis.curation_group_id, patterns=tuple( FailurePatternReferenceInput( pattern_id=pattern.pattern_id, @@ -203,8 +216,10 @@ class HypothesisObservation(StrictModel): next_actions: tuple[str, ...] = Field(max_length=2) artifacts: tuple[str, ...] = Field(min_length=2, max_length=2) hypothesis_id: str = Field(pattern=_PATTERN_ID_PATTERN) - experiment_id: str = Field(min_length=1, max_length=80) + experiment_id: str = Field(min_length=1, max_length=80, pattern=_EXPERIMENT_PATTERN) source_commit: str = Field(pattern=_COMMIT_PATTERN) + curation_id: str = Field(pattern=_ARTIFACT_ID_PATTERN) + curation_group_id: str = Field(pattern=_ARTIFACT_ID_PATTERN) relative_path: Path pattern_count: int = Field(strict=True, ge=1, le=50) diagnosis_count: int = Field(strict=True, ge=1, le=50) @@ -217,6 +232,10 @@ class HypothesisErrorCode(StrEnum): STALE_POLICY = "stale_policy" STALE_COMMIT = "stale_commit" DIRTY_WORKSPACE = "dirty_workspace" + CURATION_NOT_FOUND = "curation_not_found" + CURATION_INVALID = "curation_invalid" + CURATION_GROUP_NOT_FOUND = "curation_group_not_found" + CURATION_EVIDENCE_MISMATCH = "curation_evidence_mismatch" PATTERN_EVIDENCE_MISMATCH = "pattern_evidence_mismatch" INCONCLUSIVE_EVIDENCE = "inconclusive_evidence" TARGET_NOT_EDITABLE = "target_not_editable" @@ -237,10 +256,26 @@ def __init__(self, code: HypothesisErrorCode, subject: str) -> None: super().__init__(f"{code.value}: {subject}") +class HypothesisGateway(Protocol): + def load_policy(self, root: Path, experiment_id: str) -> ExperimentPolicySnapshot: ... + + def load_curation(self, root: Path, curation_id: str) -> FailureCurationArtifact: ... + + def validate_workspace( + self, + root: Path, + policy: ExperimentPolicySnapshot, + source_commit: str, + paths: tuple[Path, ...], + ) -> None: ... + + def store(self, root: Path, hypothesis: HarnessHypothesis) -> Path: ... + + @dataclass(frozen=True, slots=True) class HypothesisService: pattern_miner: FailurePatternMiningService - repository: FileHypothesisRepository + repository: HypothesisGateway def record(self, request: RecordHypothesisInput) -> HypothesisObservation: policy = _load_policy(self.repository, request) @@ -253,7 +288,9 @@ def record(self, request: RecordHypothesisInput) -> HypothesisObservation: request.source_commit, paths, ) + group = _load_curation_group(self.repository, request) patterns = _canonical_patterns(request.patterns) + _validate_curated_evidence(request, group, patterns) mined = _mine_patterns(self.pattern_miner, request.workspace_root, patterns) _validate_mined_patterns(patterns, mined) hypothesis = _hypothesis(request, patterns, paths) @@ -262,7 +299,7 @@ def record(self, request: RecordHypothesisInput) -> HypothesisObservation: def _load_policy( - repository: FileHypothesisRepository, + repository: HypothesisGateway, request: RecordHypothesisInput, ) -> ExperimentPolicySnapshot: try: @@ -276,6 +313,23 @@ def _load_policy( raise HypothesisFailure(code, request.experiment_id) from None +def _load_curation_group( + repository: HypothesisGateway, + request: RecordHypothesisInput, +) -> FailureGroupArtifact: + curation = repository.load_curation(request.workspace_root, request.curation_id) + group = next( + (item for item in curation.groups if item.group_id == request.curation_group_id), + None, + ) + if group is None: + raise HypothesisFailure( + HypothesisErrorCode.CURATION_GROUP_NOT_FOUND, + request.curation_group_id, + ) + return group + + def _validate_policy( request: RecordHypothesisInput, policy: ExperimentPolicySnapshot, @@ -302,6 +356,39 @@ def _canonical_patterns( ) +def _validate_curated_evidence( + request: RecordHypothesisInput, + group: FailureGroupArtifact, + patterns: tuple[FailurePatternReference, ...], +) -> None: + if _declared_artifact_ids(patterns) != _curated_artifact_ids(group): + raise HypothesisFailure( + HypothesisErrorCode.CURATION_EVIDENCE_MISMATCH, + request.curation_group_id, + ) + if request.target.component_kind is not group.target_component: + raise HypothesisFailure( + HypothesisErrorCode.CURATION_EVIDENCE_MISMATCH, + request.curation_group_id, + ) + + +def _declared_artifact_ids( + patterns: tuple[FailurePatternReference, ...], +) -> tuple[str, ...]: + return tuple( + sorted( + artifact_id + for pattern in patterns + for artifact_id in pattern.diagnosis_artifact_ids + ) + ) + + +def _curated_artifact_ids(group: FailureGroupArtifact) -> tuple[str, ...]: + return tuple(sorted(member.artifact_id for member in group.members)) + + def _pattern_reference_id(pattern: FailurePatternReferenceInput) -> str: return pattern.pattern_id @@ -370,6 +457,8 @@ def _hypothesis( hypothesis_id, request.experiment_id, request.source_commit, + request.curation_id, + request.curation_group_id, patterns, request.statement, request.rationale, @@ -388,6 +477,8 @@ def _content( return _HypothesisContent( experiment_id=request.experiment_id, source_commit=request.source_commit, + curation_id=request.curation_id, + curation_group_id=request.curation_group_id, patterns=tuple( FailurePatternReferenceInput( pattern_id=pattern.pattern_id, @@ -416,6 +507,8 @@ def _observation(hypothesis: HarnessHypothesis, relative_path: Path) -> Hypothes hypothesis_id=hypothesis.id.value, experiment_id=hypothesis.experiment_id, source_commit=hypothesis.source_commit, + curation_id=hypothesis.curation_id, + curation_group_id=hypothesis.curation_group_id, relative_path=relative_path, pattern_count=len(hypothesis.patterns), diagnosis_count=diagnosis_count, @@ -438,7 +531,3 @@ def _normalized_text(value: str) -> str: if not normalized: raise ValueError("hypothesis text must not be blank") return normalized - - -from ofw.evolution.hypothesis_repository import FileHypothesisRepository # noqa: E402 -from ofw.preparation.policy import ExperimentPolicySnapshot # noqa: E402 diff --git a/src/ofw/evolution/hypothesis_repository.py b/src/ofw/evolution/hypothesis_repository.py index 1e852a2..8644022 100644 --- a/src/ofw/evolution/hypothesis_repository.py +++ b/src/ofw/evolution/hypothesis_repository.py @@ -6,6 +6,9 @@ import subprocess # nosec B404 from pathlib import Path +from pydantic import ValidationError + +from ofw.evaluation.failure_curation import FailureCurationArtifact from ofw.evolution.hypothesis import ( HarnessHypothesis, HypothesisArtifact, @@ -19,8 +22,10 @@ from ofw.safe_file import ( SafeFileErrorCode, SafeFileFailure, + open_child_directory, open_directory_chain, publish_idempotent, + read_bounded, ) _HYPOTHESIS_LIMIT_BYTES = 64 * 1024 @@ -33,6 +38,30 @@ class FileHypothesisRepository: def load_policy(self, root: Path, experiment_id: str) -> ExperimentPolicySnapshot: return FileExperimentPolicyRepository().load(root, experiment_id) + def load_curation(self, root: Path, curation_id: str) -> FailureCurationArtifact: + prepared_root = _prepared_root(root) + try: + with open_directory_chain( + prepared_root, + (".workspace", "failure-curations"), + create=False, + ) as directory: + content = read_bounded( + directory, + f"{curation_id}.json", + maximum_bytes=_HYPOTHESIS_LIMIT_BYTES, + subject=curation_id, + ) + artifact = FailureCurationArtifact.model_validate_json(content) + artifact.require_valid_identity() + except FileNotFoundError: + raise HypothesisFailure(HypothesisErrorCode.CURATION_NOT_FOUND, curation_id) from None + except (SafeFileFailure, ValidationError, ValueError): + raise HypothesisFailure(HypothesisErrorCode.CURATION_INVALID, curation_id) from None + if artifact.curation_id != curation_id: + raise HypothesisFailure(HypothesisErrorCode.CURATION_INVALID, curation_id) + return artifact + def validate_workspace( self, root: Path, @@ -55,19 +84,20 @@ def store(self, root: Path, hypothesis: HarnessHypothesis) -> Path: artifact = HypothesisArtifact.from_hypothesis(hypothesis) content = (artifact.model_dump_json(indent=2) + "\n").encode("utf-8") try: - _ensure_workspace_ignore(prepared_root) with open_directory_chain( prepared_root, - (".workspace", "hypotheses"), + (".workspace",), create=True, - ) as directory: - publish_idempotent( - directory, - f"{hypothesis.id.value}.json", - content, - maximum_bytes=_HYPOTHESIS_LIMIT_BYTES, - subject=hypothesis.id.value, - ) + ) as workspace: + _publish_workspace_ignore(workspace) + with open_child_directory(workspace, "hypotheses", create=True) as directory: + publish_idempotent( + directory, + f"{hypothesis.id.value}.json", + content, + maximum_bytes=_HYPOTHESIS_LIMIT_BYTES, + subject=hypothesis.id.value, + ) except SafeFileFailure as error: raise _storage_failure(error, hypothesis.id.value) from None except OSError: @@ -106,15 +136,14 @@ def _require_regular_target(root: Path, relative: Path) -> None: raise HypothesisFailure(HypothesisErrorCode.INVALID_TARGET, relative.as_posix()) -def _ensure_workspace_ignore(root: Path) -> None: - with open_directory_chain(root, (".workspace",), create=True) as workspace: - publish_idempotent( - workspace, - ".gitignore", - b"*\n", - maximum_bytes=16, - subject=".workspace", - ) +def _publish_workspace_ignore(workspace: int) -> None: + publish_idempotent( + workspace, + ".gitignore", + b"*\n", + maximum_bytes=16, + subject=".workspace", + ) def _git(root: Path, *arguments: str) -> str: diff --git a/src/ofw/preparation/contracts.py b/src/ofw/preparation/contracts.py index 4903e97..46b2a24 100644 --- a/src/ofw/preparation/contracts.py +++ b/src/ofw/preparation/contracts.py @@ -9,9 +9,9 @@ from pydantic import BaseModel, BeforeValidator, ConfigDict, Field, field_validator -_EXPERIMENT_PATTERN = r"[a-z0-9]+(?:-[a-z0-9]+)*" -_REF_PATTERN = r"[A-Za-z0-9][A-Za-z0-9._/@-]*" -_COMMIT_PATTERN = r"[0-9a-f]{40}" +_EXPERIMENT_PATTERN = r"^[a-z0-9]+(?:-[a-z0-9]+)*$" +_REF_PATTERN = r"^[A-Za-z0-9][A-Za-z0-9._/@-]*$" +_COMMIT_PATTERN = r"^[0-9a-f]{40}$" def _normalized_score(value: object) -> float: diff --git a/src/ofw/preparation/policy.py b/src/ofw/preparation/policy.py index 5a3cdcb..5f15fca 100644 --- a/src/ofw/preparation/policy.py +++ b/src/ofw/preparation/policy.py @@ -34,9 +34,9 @@ ) _POLICY_LIMIT_BYTES = 256 * 1024 -_COMMIT_PATTERN = r"[0-9a-f]{40}" -_DIGEST_PATTERN = r"sha256:[0-9a-f]{64}" -_TASK_ID_PATTERN = r"[^\x00]+" +_COMMIT_PATTERN = r"^[0-9a-f]{40}$" +_DIGEST_PATTERN = r"^sha256:[0-9a-f]{64}$" +_TASK_ID_PATTERN = r"^[^\x00]+$" TaskIds = Annotated[ tuple[Annotated[str, Field(min_length=1, max_length=256, pattern=_TASK_ID_PATTERN)], ...], Field(min_length=1, max_length=500), diff --git a/src/ofw/preparation/service.py b/src/ofw/preparation/service.py index f43c929..130e7c5 100644 --- a/src/ofw/preparation/service.py +++ b/src/ofw/preparation/service.py @@ -35,12 +35,13 @@ build_experiment_policy, ) -_COMMIT_PATTERN = r"[0-9a-f]{40}" +_COMMIT_PATTERN = r"^[0-9a-f]{40}$" +_DIGEST_PATTERN = r"^sha256:[0-9a-f]{64}$" class _PreparationStateWire(StrictModel): schema_version: int = Field(default=2, ge=2, le=2) - request_digest: str = Field(pattern=r"sha256:[0-9a-f]{64}") + request_digest: str = Field(pattern=_DIGEST_PATTERN) phase: PreparationPhase branch_name: str worktree_path: Path @@ -51,7 +52,7 @@ class _PreparationStateWire(StrictModel): log_path: Path model: str = Field(min_length=1, max_length=256) task_ids: tuple[str, ...] = Field(min_length=1, max_length=500) - benchmark_config_digest: str = Field(pattern=r"sha256:[0-9a-f]{64}") + benchmark_config_digest: str = Field(pattern=_DIGEST_PATTERN) verifier: str = Field(min_length=1, max_length=128) environment: str = Field(min_length=1, max_length=128) policy_published: bool = False diff --git a/src/ofw/preparation/templates/base.md b/src/ofw/preparation/templates/base.md index 369dcdd..3a20d41 100644 --- a/src/ofw/preparation/templates/base.md +++ b/src/ofw/preparation/templates/base.md @@ -48,8 +48,8 @@ State the failure pattern, supporting trace and verifier evidence, proposed harn expected improvement, and possible regressions. Stop if the evidence cannot distinguish between materially different changes. -Use `$hypothesis-former` with only exact supported pattern and diagnosis receipt IDs, then call -`record_hypothesis`. Retain the stable hypothesis receipt and stop before candidate editing. +Use `$hypothesis-former` with one curation receipt and group ID plus every exact supported +pattern and diagnosis receipt ID in that group, then call `record_hypothesis`. Retain the stable hypothesis receipt and stop before candidate editing. Candidate editing requires a later package and must not begin in this prepared program. ## Package boundary diff --git a/src/ofw/safe_file.py b/src/ofw/safe_file.py index 60be33a..a466c62 100644 --- a/src/ofw/safe_file.py +++ b/src/ofw/safe_file.py @@ -40,6 +40,18 @@ def open_directory(path: Path) -> Iterator[int]: yield descriptor +@contextmanager +def open_child_directory(parent: int, name: str, *, create: bool) -> Iterator[int]: + """Open one child directory and reject replacement while its descriptor is in use.""" + descriptor = _open_child(parent, name, create=create) + try: + _require_child_identity(parent, name, descriptor) + yield descriptor + _require_child_identity(parent, name, descriptor) + finally: + os.close(descriptor) + + @contextmanager def open_directory_chain( root: Path, diff --git a/tests/test_experiment_policy.py b/tests/test_experiment_policy.py index 8a99594..01f5cea 100644 --- a/tests/test_experiment_policy.py +++ b/tests/test_experiment_policy.py @@ -178,6 +178,34 @@ def test_policy_schema_rejects_extra_fields_and_tampered_controls(tmp_path: Path ExperimentPolicySnapshot.model_validate_json(escaped) +@pytest.mark.parametrize( + ("field", "malformed"), + ( + ("base_commit", "x" + "1" * 40 + "y"), + ("initialization_commit", "x" + "2" * 40 + "y"), + ("benchmark_config_digest", "xsha256:" + "3" * 64 + "y"), + ), +) +def test_policy_schema_rejects_prefixed_and_suffixed_identifiers( + tmp_path: Path, + field: str, + malformed: str, +) -> None: + _, policy = _snapshot(tmp_path) + originals: dict[str, str] = { + "base_commit": policy.base_commit, + "initialization_commit": policy.initialization_commit, + "benchmark_config_digest": policy.benchmark_config_digest, + } + payload = policy.model_dump_json().replace( + f'"{field}":"{originals[field]}"', + f'"{field}":"{malformed}"', + ) + + with pytest.raises(ValidationError, match=field): + ExperimentPolicySnapshot.model_validate_json(payload) + + def test_policy_publish_is_atomic_idempotent_and_conflict_detecting(tmp_path: Path) -> None: root, policy = _snapshot(tmp_path) control = _control_directory(root) diff --git a/tests/test_failure_workspace.py b/tests/test_failure_workspace.py index 0c170ad..6a1886e 100644 --- a/tests/test_failure_workspace.py +++ b/tests/test_failure_workspace.py @@ -14,7 +14,6 @@ import pytest from pydantic import ValidationError -import ofw.evaluation.failure_workspace as failure_workspace_module from ofw.contracts import ComponentKind, Sha256Digest from ofw.evaluation.failure import FailureEvidenceStatus, FailureType from ofw.evaluation.failure_curation import ( @@ -407,71 +406,6 @@ def test_workspace_symlink_cannot_escape_the_prepared_root(tmp_path: Path) -> No assert tuple(outside.iterdir()) == () -def test_symlink_swap_after_validation_cannot_redirect_the_artifact( - tmp_path: Path, - monkeypatch: pytest.MonkeyPatch, -) -> None: - root = _prepared_workspace(tmp_path) - outside = tmp_path / "outside" - outside.mkdir() - original = failure_workspace_module._prepare_workspace_directories - - def prepare_then_swap( - prepared_root: Path, - workspace: Path, - failures: Path, - ) -> failure_workspace_module._DirectoryChainIdentity: - identity = original(prepared_root, workspace, failures) - failures.rmdir() - failures.symlink_to(outside, target_is_directory=True) - return identity - - monkeypatch.setattr( - failure_workspace_module, - "_prepare_workspace_directories", - prepare_then_swap, - ) - - with pytest.raises(FailureWorkspaceFailure): - FailureWorkspaceService(FileFailureWorkspace()).record(_request(root)) - - assert tuple(outside.iterdir()) == () - - -def test_workspace_directory_swap_after_validation_cannot_receive_the_artifact( - tmp_path: Path, - monkeypatch: pytest.MonkeyPatch, -) -> None: - root = _prepared_workspace(tmp_path) - replacement = tmp_path / "replacement" - displaced = tmp_path / "validated-workspace" - (replacement / "failures").mkdir(parents=True) - original = failure_workspace_module._prepare_workspace_directories - - def prepare_then_swap( - prepared_root: Path, - workspace: Path, - failures: Path, - ) -> failure_workspace_module._DirectoryChainIdentity: - identity = original(prepared_root, workspace, failures) - workspace.rename(displaced) - replacement.rename(workspace) - return identity - - monkeypatch.setattr( - failure_workspace_module, - "_prepare_workspace_directories", - prepare_then_swap, - ) - - with pytest.raises(FailureWorkspaceFailure) as raised: - FailureWorkspaceService(FileFailureWorkspace()).record(_request(root)) - - assert raised.value.code is FailureWorkspaceErrorCode.WRITE_FAILED - assert not tuple(displaced.rglob("*.json")) - assert not tuple((root / ".workspace").rglob("*.json")) - - def test_curates_recorded_failures_without_copying_trace_content(tmp_path: Path) -> None: root = _prepared_workspace(tmp_path) artifact_ids, _ = _record_failures(root) diff --git a/tests/test_hypothesis.py b/tests/test_hypothesis.py index cbe2878..1a8c5f6 100644 --- a/tests/test_hypothesis.py +++ b/tests/test_hypothesis.py @@ -13,10 +13,17 @@ from ofw.contracts import ComponentKind from ofw.evaluation.failure import FailureEvidenceStatus, FailureType +from ofw.evaluation.failure_curation import ( + FailureCurationArtifact, + FailureCurationService, + FailureGroupInput, + RecordFailureCurationInput, +) from ofw.evaluation.failure_patterns import FailurePatternMiningService, failure_pattern_id from ofw.evaluation.failure_workspace import ( FailedOutcomeInput, FailureWorkspaceService, + FileFailureCurationWorkspace, FileFailureWorkspace, RecordFailureInput, ) @@ -152,11 +159,49 @@ def _service() -> HypothesisService: ) -def _request(root: Path, commit: str, artifacts: tuple[str, ...]) -> RecordHypothesisInput: +def _curation(root: Path, artifacts: tuple[str, ...]) -> tuple[str, str]: + observation = FailureCurationService(FileFailureCurationWorkspace()).record( + RecordFailureCurationInput( + workspace_root=root, + source_artifact_ids=artifacts, + groups=( + FailureGroupInput( + pattern_key="premature-completion", + title="Premature completion", + mechanism="The agent finalizes before verification.", + prevention="Require verification before completion.", + target_component=ComponentKind.PROMPT, + failure_artifact_ids=artifacts, + ), + ), + deferred=(), + ) + ) + artifact = FailureCurationArtifact.model_validate_json( + (root / observation.relative_path).read_text(encoding="utf-8") + ) + return observation.curation_id, artifact.groups[0].group_id + + +def _request( + root: Path, + commit: str, + artifacts: tuple[str, ...], + *, + curation_artifacts: tuple[str, ...] | None = None, +) -> RecordHypothesisInput: + selected_curation_artifacts = curation_artifacts or artifacts + if len(selected_curation_artifacts) >= 2: + curation_id, curation_group_id = _curation(root, selected_curation_artifacts) + else: + curation_id = "00000000-0000-0000-0000-000000000010" + curation_group_id = "00000000-0000-0000-0000-000000000011" return RecordHypothesisInput( workspace_root=root, experiment_id="experiment-one", source_commit=commit, + curation_id=curation_id, + curation_group_id=curation_group_id, patterns=( FailurePatternReferenceInput( pattern_id=failure_pattern_id( @@ -185,6 +230,8 @@ def _with_patterns( workspace_root=request.workspace_root, experiment_id=request.experiment_id, source_commit=request.source_commit, + curation_id=request.curation_id, + curation_group_id=request.curation_group_id, patterns=patterns, statement=request.statement, rationale=request.rationale, @@ -202,6 +249,8 @@ def _with_target( workspace_root=request.workspace_root, experiment_id=request.experiment_id, source_commit=request.source_commit, + curation_id=request.curation_id, + curation_group_id=request.curation_group_id, patterns=request.patterns, statement=request.statement, rationale=request.rationale, @@ -211,6 +260,25 @@ def _with_target( ) +def _with_curation_group( + request: RecordHypothesisInput, + curation_group_id: str, +) -> RecordHypothesisInput: + return RecordHypothesisInput( + workspace_root=request.workspace_root, + experiment_id=request.experiment_id, + source_commit=request.source_commit, + curation_id=request.curation_id, + curation_group_id=curation_group_id, + patterns=request.patterns, + statement=request.statement, + rationale=request.rationale, + target=request.target, + expected_effect=request.expected_effect, + regression_risks=request.regression_risks, + ) + + def _evidence_case( case: str, first: str, @@ -238,6 +306,8 @@ def test_record_hypothesis_recomputes_evidence_and_is_deterministic(tmp_path: Pa assert first == repeated assert first.status is HypothesisStatus.SUCCESS assert first.source_commit == commit + assert first.curation_id == request.curation_id + assert first.curation_group_id == request.curation_group_id assert first.pattern_count == 1 assert first.diagnosis_count == 2 assert first.target_paths == (Path("prompt.md"),) @@ -271,7 +341,8 @@ def test_hypothesis_fails_closed_on_inexact_evidence(tmp_path: Path, case: str) other = _diagnosis(root, "3", root_cause="The tool result is ignored after mutation.") inconclusive = _diagnosis(root, "4", supported=False) artifacts = _evidence_case(case, first, second, other, inconclusive) - request = _request(root, commit, artifacts) + curation_artifacts = (first, other) if case == "misassigned" else (first, second) + request = _request(root, commit, artifacts, curation_artifacts=curation_artifacts) if case == "misassigned": request = _with_patterns( request, @@ -297,9 +368,9 @@ def test_hypothesis_fails_closed_on_inexact_evidence(tmp_path: Path, case: str) _service().record(request) expected = ( - HypothesisErrorCode.INCONCLUSIVE_EVIDENCE - if case == "inconclusive" - else HypothesisErrorCode.PATTERN_EVIDENCE_MISMATCH + HypothesisErrorCode.PATTERN_EVIDENCE_MISMATCH + if case == "misassigned" + else HypothesisErrorCode.CURATION_EVIDENCE_MISMATCH ) assert raised.value.code is expected @@ -340,6 +411,7 @@ def test_hypothesis_rejects_stale_or_drifted_workspace(tmp_path: Path, drift: st def test_hypothesis_rejects_semantically_tampered_diagnosis(tmp_path: Path) -> None: root, commit = _workspace(tmp_path) artifacts = (_diagnosis(root, "1"), _diagnosis(root, "2")) + request = _request(root, commit, artifacts) path = root / ".workspace/failures" / f"{artifacts[0]}.json" content = path.read_text(encoding="utf-8") path.write_text( @@ -351,7 +423,7 @@ def test_hypothesis_rejects_semantically_tampered_diagnosis(tmp_path: Path) -> N ) with pytest.raises(HypothesisFailure) as raised: - _service().record(_request(root, commit, artifacts)) + _service().record(request) assert raised.value.code is HypothesisErrorCode.PATTERN_EVIDENCE_MISMATCH @@ -414,6 +486,8 @@ def test_hypothesis_input_rejects_duplicates_escapes_empty_and_extra_fields(tmp_ workspace_root=request.workspace_root, experiment_id=request.experiment_id, source_commit=request.source_commit, + curation_id=request.curation_id, + curation_group_id=request.curation_group_id, patterns=request.patterns, statement=request.statement, rationale=request.rationale, @@ -423,12 +497,67 @@ def test_hypothesis_input_rejects_duplicates_escapes_empty_and_extra_fields(tmp_ ) +@pytest.mark.parametrize( + ("field", "malformed"), + ( + ("experiment_id", "Xexperiment-one!"), + ("source_commit", "x" + "1" * 40 + "y"), + ("curation_id", "x00000000-0000-0000-0000-000000000001y"), + ("curation_group_id", "x00000000-0000-0000-0000-000000000002y"), + ), +) +def test_hypothesis_input_rejects_prefixed_and_suffixed_identifiers( + tmp_path: Path, + field: str, + malformed: str, +) -> None: + root, commit = _workspace(tmp_path) + artifact = _diagnosis(root, "1") + request = _request(root, commit, (artifact,)) + originals: dict[str, str] = { + "experiment_id": request.experiment_id, + "source_commit": request.source_commit, + "curation_id": request.curation_id, + "curation_group_id": request.curation_group_id, + } + payload = request.model_dump_json().replace( + f'"{field}":"{originals[field]}"', + f'"{field}":"{malformed}"', + ) + + with pytest.raises(ValidationError, match=field): + RecordHypothesisInput.model_validate_json(payload) + + +@pytest.mark.parametrize( + ("field", "malformed"), + ( + ("pattern_id", "xsha256:" + "1" * 64 + "y"), + ("diagnosis_artifact_ids", ("x00000000-0000-0000-0000-000000000001y",)), + ), +) +def test_pattern_reference_rejects_prefixed_and_suffixed_identifiers( + field: str, + malformed: str | tuple[str, ...], +) -> None: + payload: dict[str, str | tuple[str, ...]] = { + "pattern_id": "sha256:" + "1" * 64, + "diagnosis_artifact_ids": ("00000000-0000-0000-0000-000000000001",), + } + payload[field] = malformed + + with pytest.raises(ValidationError): + FailurePatternReferenceInput.model_validate(payload) + + def test_hypothesis_input_accepts_its_exact_global_maximum() -> None: artifacts = tuple(f"00000000-0000-0000-0000-{index:012x}" for index in range(50)) request = RecordHypothesisInput( workspace_root=Path("/prepared"), experiment_id="maximum", source_commit="1" * 40, + curation_id="00000000-0000-0000-0000-000000000001", + curation_group_id="00000000-0000-0000-0000-000000000002", patterns=( FailurePatternReferenceInput( pattern_id="sha256:" + "1" * 64, @@ -450,6 +579,78 @@ def test_hypothesis_input_accepts_its_exact_global_maximum() -> None: assert len(request.regression_risks) == 10 +def test_hypothesis_rejects_an_incomplete_curation_group(tmp_path: Path) -> None: + root, commit = _workspace(tmp_path) + first = _diagnosis(root, "1") + second = _diagnosis(root, "2") + request = _request(root, commit, (first, second)) + incomplete = _with_patterns( + request, + ( + FailurePatternReferenceInput( + pattern_id=request.patterns[0].pattern_id, + diagnosis_artifact_ids=(first,), + ), + ), + ) + + with pytest.raises(HypothesisFailure) as raised: + _service().record(incomplete) + + assert raised.value.code is HypothesisErrorCode.CURATION_EVIDENCE_MISMATCH + + +def test_hypothesis_rejects_a_missing_curation_group(tmp_path: Path) -> None: + root, commit = _workspace(tmp_path) + artifacts = (_diagnosis(root, "1"), _diagnosis(root, "2")) + request = _with_curation_group( + _request(root, commit, artifacts), + "00000000-0000-0000-0000-000000000099", + ) + + with pytest.raises(HypothesisFailure) as raised: + _service().record(request) + + assert raised.value.code is HypothesisErrorCode.CURATION_GROUP_NOT_FOUND + + +def test_hypothesis_rejects_a_target_outside_the_curated_component(tmp_path: Path) -> None: + root, commit = _workspace(tmp_path) + artifacts = (_diagnosis(root, "1"), _diagnosis(root, "2")) + request = _with_target( + _request(root, commit, artifacts), + HarnessChangeTargetInput( + component_kind=ComponentKind.TOOL, + relative_paths=(Path("prompt.md"),), + ), + ) + + with pytest.raises(HypothesisFailure) as raised: + _service().record(request) + + assert raised.value.code is HypothesisErrorCode.CURATION_EVIDENCE_MISMATCH + + +def test_hypothesis_rejects_a_tampered_curation_receipt(tmp_path: Path) -> None: + root, commit = _workspace(tmp_path) + artifacts = (_diagnosis(root, "1"), _diagnosis(root, "2")) + request = _request(root, commit, artifacts) + path = root / ".workspace/failure-curations" / f"{request.curation_id}.json" + content = path.read_text(encoding="utf-8") + path.write_text( + content.replace( + "The agent finalizes before verification.", + "The curation was changed after recording.", + ), + encoding="utf-8", + ) + + with pytest.raises(HypothesisFailure) as raised: + _service().record(request) + + assert raised.value.code is HypothesisErrorCode.CURATION_INVALID + + def test_hypothesis_missing_policy_is_typed(tmp_path: Path) -> None: root, commit = _workspace(tmp_path) (root / ".git/ofw/preparations/experiment-one/policy.json").unlink() diff --git a/tests/test_openflywheel_mcp.py b/tests/test_openflywheel_mcp.py index e6adfd2..12e84b5 100644 --- a/tests/test_openflywheel_mcp.py +++ b/tests/test_openflywheel_mcp.py @@ -79,6 +79,7 @@ _FAILURE_ARTIFACT_ID = "00000000-0000-0000-0000-000000000001" _SECOND_FAILURE_ARTIFACT_ID = "00000000-0000-0000-0000-000000000002" _CURATION_ID = "00000000-0000-0000-0000-000000000003" +_CURATION_GROUP_ID = "00000000-0000-0000-0000-000000000004" _PATTERN_ID = "sha256:" + "1" * 64 _HYPOTHESIS_ID = "sha256:" + "2" * 64 _COMMIT = "1" * 40 @@ -340,6 +341,8 @@ def _hypothesis_request(root: Path) -> RecordHypothesisInput: workspace_root=root, experiment_id="experiment-one", source_commit=_COMMIT, + curation_id=_CURATION_ID, + curation_group_id=_CURATION_GROUP_ID, patterns=( FailurePatternReferenceInput( pattern_id=_PATTERN_ID, @@ -367,6 +370,8 @@ def _hypothesis_observation() -> HypothesisObservation: hypothesis_id=_HYPOTHESIS_ID, experiment_id="experiment-one", source_commit=_COMMIT, + curation_id=_CURATION_ID, + curation_group_id=_CURATION_GROUP_ID, relative_path=relative_path, pattern_count=1, diagnosis_count=1, diff --git a/tests/test_plugin_packaging.py b/tests/test_plugin_packaging.py index 3b01148..b09b535 100644 --- a/tests/test_plugin_packaging.py +++ b/tests/test_plugin_packaging.py @@ -1,5 +1,7 @@ """Portable Codex plugin MCP launch contract.""" +import re +import subprocess from pathlib import Path from typing import Literal @@ -23,15 +25,44 @@ class _McpManifest(BaseModel): def test_openflywheel_mcp_uses_pinned_portable_runtime() -> None: - path = Path(__file__).parents[1] / "plugins/openflywheel/.mcp.json" + root = Path(__file__).parents[1] + path = root / "plugins/openflywheel/.mcp.json" manifest = _McpManifest.model_validate_json(path.read_text(encoding="utf-8")) server = manifest.mcpServers["openflywheel"] - - assert ( - "git+https://github.com/divo12/OpenFlyWheel.git@b4c785d9472ca1d6c245c41fa4d9985f0cef97fc" - in server.args + source = server.args[1] + match = re.fullmatch( + r"git\+https://github\.com/divo12/OpenFlyWheel\.git@([0-9a-f]{40})", + source, ) + + assert match is not None + runtime = subprocess.run( + ("git", "show", f"{match.group(1)}:src/ofw/mcp.py"), + cwd=root, + check=True, + capture_output=True, + text=True, + ).stdout + assert "def record_hypothesis(" in runtime assert "openflywheel-mcp" in server.args assert "PLUGIN_ROOT" not in path.read_text(encoding="utf-8") assert "OPENFLYWHEEL_ROOT" not in path.read_text(encoding="utf-8") assert "LANGFUSE_SECRET_KEY" in server.env_vars + + +def test_plugin_and_runtime_package_versions_match() -> None: + root = Path(__file__).parents[1] + plugin_version_line = next( + line + for line in (root / "plugins/openflywheel/.codex-plugin/plugin.json") + .read_text(encoding="utf-8") + .splitlines() + if line.strip().startswith('"version":') + ) + package_version_line = next( + line + for line in (root / "pyproject.toml").read_text(encoding="utf-8").splitlines() + if line.startswith("version = ") + ) + + assert plugin_version_line.split('"')[3] == package_version_line.split('"')[1] diff --git a/tests/test_safe_file.py b/tests/test_safe_file.py index 5b4a498..1e87829 100644 --- a/tests/test_safe_file.py +++ b/tests/test_safe_file.py @@ -10,12 +10,43 @@ from ofw.safe_file import ( SafeFileErrorCode, SafeFileFailure, + open_child_directory, open_directory, publish_idempotent, read_bounded, ) +def test_child_directory_swap_is_detected_without_redirecting_publication( + tmp_path: Path, +) -> None: + parent_path = tmp_path / "parent" + child_path = parent_path / "child" + moved_path = parent_path / "moved" + replacement_path = parent_path / "replacement" + child_path.mkdir(parents=True) + replacement_path.mkdir() + + with ( + pytest.raises(SafeFileFailure) as raised, + open_directory(parent_path) as parent, + open_child_directory(parent, "child", create=False) as child, + ): + child_path.rename(moved_path) + replacement_path.rename(child_path) + publish_idempotent( + child, + "artifact.json", + b"{}\n", + maximum_bytes=16, + subject="artifact", + ) + + assert raised.value.code is SafeFileErrorCode.DIRECTORY_CHANGED + assert not (child_path / "artifact.json").exists() + assert (moved_path / "artifact.json").read_bytes() == b"{}\n" + + def test_regular_file_reader_rejects_device() -> None: with open_directory(Path("/dev")) as directory, pytest.raises(SafeFileFailure) as raised: read_bounded(directory, "null", maximum_bytes=16, subject="device") From abee2da959c14d261f7e846b614683babb0a1bd9 Mon Sep 17 00:00:00 2001 From: divo12 Date: Thu, 3 Sep 2026 14:58:48 +0530 Subject: [PATCH 06/25] fix: pin plugin to hypothesis runtime --- plugins/openflywheel/.mcp.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/openflywheel/.mcp.json b/plugins/openflywheel/.mcp.json index 00d2fb5..3e05ef6 100644 --- a/plugins/openflywheel/.mcp.json +++ b/plugins/openflywheel/.mcp.json @@ -4,7 +4,7 @@ "command": "uvx", "args": [ "--from", - "git+https://github.com/divo12/OpenFlyWheel.git@b4c785d9472ca1d6c245c41fa4d9985f0cef97fc", + "git+https://github.com/divo12/OpenFlyWheel.git@48d7363d348d49baf6496a0100054ce246d35e4c", "--with", "mcp>=1.13,<2", "openflywheel-mcp" From 8495ce1db822f84a89a341adca0562fb6eaed50a Mon Sep 17 00:00:00 2001 From: divo12 Date: Thu, 3 Sep 2026 15:01:25 +0530 Subject: [PATCH 07/25] ci: fetch pinned runtime history --- .github/workflows/ci.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 348407f..6ebe419 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -14,6 +14,8 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v7 + with: + fetch-depth: 0 - name: Install Python and dependencies run: | python -m pip install uv From a43b8a2e1ae37643fc2c19107346c38f99069b0f Mon Sep 17 00:00:00 2001 From: divo12 Date: Thu, 3 Sep 2026 15:03:04 +0530 Subject: [PATCH 08/25] test: avoid git history in plugin runtime check --- tests/test_plugin_packaging.py | 19 ++++--------------- 1 file changed, 4 insertions(+), 15 deletions(-) diff --git a/tests/test_plugin_packaging.py b/tests/test_plugin_packaging.py index b09b535..c31dffa 100644 --- a/tests/test_plugin_packaging.py +++ b/tests/test_plugin_packaging.py @@ -1,7 +1,5 @@ """Portable Codex plugin MCP launch contract.""" -import re -import subprocess from pathlib import Path from typing import Literal @@ -29,20 +27,11 @@ def test_openflywheel_mcp_uses_pinned_portable_runtime() -> None: path = root / "plugins/openflywheel/.mcp.json" manifest = _McpManifest.model_validate_json(path.read_text(encoding="utf-8")) server = manifest.mcpServers["openflywheel"] - source = server.args[1] - match = re.fullmatch( - r"git\+https://github\.com/divo12/OpenFlyWheel\.git@([0-9a-f]{40})", - source, + assert ( + "git+https://github.com/divo12/OpenFlyWheel.git@48d7363d348d49baf6496a0100054ce246d35e4c" + in server.args ) - - assert match is not None - runtime = subprocess.run( - ("git", "show", f"{match.group(1)}:src/ofw/mcp.py"), - cwd=root, - check=True, - capture_output=True, - text=True, - ).stdout + runtime = (root / "src/ofw/mcp.py").read_text(encoding="utf-8") assert "def record_hypothesis(" in runtime assert "openflywheel-mcp" in server.args assert "PLUGIN_ROOT" not in path.read_text(encoding="utf-8") From 3175f240a601bcb70a6925d6cb67326850f24700 Mon Sep 17 00:00:00 2001 From: divo12 Date: Thu, 3 Sep 2026 15:07:27 +0530 Subject: [PATCH 09/25] ci: keep runtime check shallow-safe --- .github/workflows/ci.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 6ebe419..348407f 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -14,8 +14,6 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v7 - with: - fetch-depth: 0 - name: Install Python and dependencies run: | python -m pip install uv From fe5d0972ef4733553454814e96910ae51ab6a5a3 Mon Sep 17 00:00:00 2001 From: divo12 Date: Thu, 3 Sep 2026 17:34:07 +0530 Subject: [PATCH 10/25] feat: persist baseline provenance and hypothesis predictions --- .../openflywheel/program_templates/base.md | 6 +- .../skills/hypothesis-former/SKILL.md | 7 ++- .../skills/workspace-init/SKILL.md | 5 +- src/ofw/evolution/hypothesis.py | 34 +++++++++++ src/ofw/preparation/contracts.py | 1 + src/ofw/preparation/policy.py | 4 ++ src/ofw/preparation/service.py | 39 ++++++++++++ src/ofw/preparation/templates/base.md | 6 +- src/ofw/preparation/worktree.py | 1 + tests/test_experiment_policy.py | 1 + tests/test_hypothesis.py | 39 ++++++++++++ tests/test_openflywheel_mcp.py | 4 ++ tests/test_workspace_preparation.py | 60 +++++++++++++++++++ 13 files changed, 199 insertions(+), 8 deletions(-) diff --git a/plugins/openflywheel/program_templates/base.md b/plugins/openflywheel/program_templates/base.md index 3a20d41..1fd3954 100644 --- a/plugins/openflywheel/program_templates/base.md +++ b/plugins/openflywheel/program_templates/base.md @@ -8,7 +8,8 @@ Record one evidence-backed hypothesis for the connected agent harness under the experiment policy, then stop before candidate editing. The baseline has already been recorded. Begin at step 2; do not rerun the unchanged -baseline. +baseline. Its provenance is recorded in the policy (`baseline_reused` is explicit when an +existing terminal Harbor job was adopted). ## Authority @@ -49,7 +50,8 @@ expected improvement, and possible regressions. Stop if the evidence cannot dist between materially different changes. Use `$hypothesis-former` with one curation receipt and group ID plus every exact supported -pattern and diagnosis receipt ID in that group, then call `record_hypothesis`. Retain the stable hypothesis receipt and stop before candidate editing. +pattern and diagnosis receipt ID in that group, explicit predicted task IDs, and at-risk task IDs, then call +`record_hypothesis`. Retain the stable hypothesis receipt and stop before candidate editing. Candidate editing requires a later package and must not begin in this prepared program. ## Package boundary diff --git a/plugins/openflywheel/skills/hypothesis-former/SKILL.md b/plugins/openflywheel/skills/hypothesis-former/SKILL.md index d8b19ed..ee368cc 100644 --- a/plugins/openflywheel/skills/hypothesis-former/SKILL.md +++ b/plugins/openflywheel/skills/hypothesis-former/SKILL.md @@ -9,11 +9,12 @@ Select one group from the retained failure-curation receipt. Use every diagnosis that group, partitioned under its exact mined pattern IDs; do not add or omit IDs. Keep the global diagnosis set at fifty IDs or fewer. Read the compact diagnoses, then propose one bounded statement, rationale, expected effect, regression-risk list, component taxonomy, and one or more -exact target paths within the group's target component. +exact target paths within the group's target component, explicit predicted task IDs, and at-risk +task IDs. Keep those prediction sets disjoint so the next iteration can falsify the decision. Call `record_hypothesis` once with the prepared worktree, experiment ID, current initialization -commit, curation ID, selected curation group ID, exact pattern-to-diagnosis assignments, and -proposed target. The tool reloads the curation, requires the complete selected group, recomputes +commit, curation ID, selected curation group ID, exact pattern-to-diagnosis assignments, explicit +predicted and at-risk task IDs, and proposed target. The tool reloads the curation, requires the complete selected group, recomputes the patterns, and rejects missing, extra, misassigned, or inconclusive evidence. It also proves each target is exactly present in the authoritative editable allowlist; `component_kind` must match the curated component and never grants path access. diff --git a/plugins/openflywheel/skills/workspace-init/SKILL.md b/plugins/openflywheel/skills/workspace-init/SKILL.md index 54dae4b..3505c15 100644 --- a/plugins/openflywheel/skills/workspace-init/SKILL.md +++ b/plugins/openflywheel/skills/workspace-init/SKILL.md @@ -25,7 +25,10 @@ when the evidence supports one. Collect, in order: directories. 2. Optimization goal, primary metric, target, and stopping condition. Keep quality, cost, and latency constraints separate rather than hiding them in one average. -3. ITSM-bench root, Harbor executable, Harbor configuration, and expected task count. +3. ITSM-bench root, Harbor executable, Harbor configuration, and expected task count. If an + exact terminal Harbor job already exists at `benchmark_root/jobs/`, ask whether + to adopt it; set `reuse_existing_baseline=true` only after explicit confirmation so policy is + published without launching a duplicate baseline. 4. Experiment ID and maximum baseline duration. Read and report the frozen model from the Harbor configuration. `prepare_workspace` fixes diff --git a/src/ofw/evolution/hypothesis.py b/src/ofw/evolution/hypothesis.py index 34efa4f..bd1338a 100644 --- a/src/ofw/evolution/hypothesis.py +++ b/src/ofw/evolution/hypothesis.py @@ -31,6 +31,7 @@ _HYPOTHESIS_ID_PATTERN = re.compile(r"sha256:[0-9a-f]{64}") _COMMIT_PATTERN = r"^[0-9a-f]{40}$" _EXPERIMENT_PATTERN = r"^[a-z0-9]+(?:-[a-z0-9]+)*$" +_TASK_ID_PATTERN = r"^[^\x00]+$" HypothesisTextValue = Annotated[str, Field(min_length=1, max_length=4000)] ArtifactIdsValue = Annotated[ tuple[Annotated[str, Field(pattern=_ARTIFACT_ID_PATTERN)], ...], @@ -40,6 +41,14 @@ tuple[Annotated[Path, Field(strict=False)], ...], Field(strict=False, min_length=1, max_length=50), ] +TaskIdsValue = Annotated[ + tuple[Annotated[str, Field(min_length=1, max_length=256, pattern=_TASK_ID_PATTERN)], ...], + Field(strict=False, min_length=1, max_length=50), +] +RiskTaskIdsValue = Annotated[ + tuple[Annotated[str, Field(min_length=1, max_length=256, pattern=_TASK_ID_PATTERN)], ...], + Field(strict=False, max_length=50), +] class StrictModel(BaseModel): @@ -77,6 +86,8 @@ class RecordHypothesisInput(StrictModel): source_commit: str = Field(pattern=_COMMIT_PATTERN) curation_id: str = Field(pattern=_ARTIFACT_ID_PATTERN) curation_group_id: str = Field(pattern=_ARTIFACT_ID_PATTERN) + predicted_task_ids: TaskIdsValue + at_risk_task_ids: RiskTaskIdsValue = () patterns: tuple[FailurePatternReferenceInput, ...] = Field( strict=False, min_length=1, @@ -121,6 +132,7 @@ def validate_evidence_bound(self) -> RecordHypothesisInput: ) _require_unique_patterns(pattern_ids) _require_bounded_unique_artifacts(artifact_ids) + _require_disjoint_task_predictions(self.predicted_task_ids, self.at_risk_task_ids) return self @@ -155,6 +167,8 @@ class HarnessHypothesis: source_commit: str curation_id: str curation_group_id: str + predicted_task_ids: tuple[str, ...] + at_risk_task_ids: tuple[str, ...] patterns: tuple[FailurePatternReference, ...] statement: str rationale: str @@ -169,6 +183,8 @@ class _HypothesisContent(StrictModel): source_commit: str = Field(pattern=_COMMIT_PATTERN) curation_id: str = Field(pattern=_ARTIFACT_ID_PATTERN) curation_group_id: str = Field(pattern=_ARTIFACT_ID_PATTERN) + predicted_task_ids: TaskIdsValue + at_risk_task_ids: RiskTaskIdsValue = () patterns: tuple[FailurePatternReferenceInput, ...] = Field(min_length=1, max_length=50) statement: HypothesisTextValue rationale: HypothesisTextValue @@ -188,6 +204,8 @@ def from_hypothesis(cls, hypothesis: HarnessHypothesis) -> HypothesisArtifact: source_commit=hypothesis.source_commit, curation_id=hypothesis.curation_id, curation_group_id=hypothesis.curation_group_id, + predicted_task_ids=hypothesis.predicted_task_ids, + at_risk_task_ids=hypothesis.at_risk_task_ids, patterns=tuple( FailurePatternReferenceInput( pattern_id=pattern.pattern_id, @@ -220,6 +238,8 @@ class HypothesisObservation(StrictModel): source_commit: str = Field(pattern=_COMMIT_PATTERN) curation_id: str = Field(pattern=_ARTIFACT_ID_PATTERN) curation_group_id: str = Field(pattern=_ARTIFACT_ID_PATTERN) + predicted_task_ids: TaskIdsValue + at_risk_task_ids: RiskTaskIdsValue = () relative_path: Path pattern_count: int = Field(strict=True, ge=1, le=50) diagnosis_count: int = Field(strict=True, ge=1, le=50) @@ -389,6 +409,14 @@ def _curated_artifact_ids(group: FailureGroupArtifact) -> tuple[str, ...]: return tuple(sorted(member.artifact_id for member in group.members)) +def _require_disjoint_task_predictions( + predicted: tuple[str, ...], + at_risk: tuple[str, ...], +) -> None: + if set(predicted) & set(at_risk): + raise ValueError("predicted_task_ids and at_risk_task_ids must be disjoint") + + def _pattern_reference_id(pattern: FailurePatternReferenceInput) -> str: return pattern.pattern_id @@ -459,6 +487,8 @@ def _hypothesis( request.source_commit, request.curation_id, request.curation_group_id, + request.predicted_task_ids, + request.at_risk_task_ids, patterns, request.statement, request.rationale, @@ -479,6 +509,8 @@ def _content( source_commit=request.source_commit, curation_id=request.curation_id, curation_group_id=request.curation_group_id, + predicted_task_ids=request.predicted_task_ids, + at_risk_task_ids=request.at_risk_task_ids, patterns=tuple( FailurePatternReferenceInput( pattern_id=pattern.pattern_id, @@ -509,6 +541,8 @@ def _observation(hypothesis: HarnessHypothesis, relative_path: Path) -> Hypothes source_commit=hypothesis.source_commit, curation_id=hypothesis.curation_id, curation_group_id=hypothesis.curation_group_id, + predicted_task_ids=hypothesis.predicted_task_ids, + at_risk_task_ids=hypothesis.at_risk_task_ids, relative_path=relative_path, pattern_count=len(hypothesis.patterns), diagnosis_count=diagnosis_count, diff --git a/src/ofw/preparation/contracts.py b/src/ofw/preparation/contracts.py index 46b2a24..521d0df 100644 --- a/src/ofw/preparation/contracts.py +++ b/src/ofw/preparation/contracts.py @@ -82,6 +82,7 @@ class PrepareWorkspaceInput(StrictModel): quality_target: NormalizedScore max_iterations: IterationCount no_improvement_limit: IterationCount + reuse_existing_baseline: bool = False max_cost_per_task_usd: PositiveMetric | None = None max_latency_seconds: PositiveMetric | None = None max_baseline_seconds: DurationSeconds diff --git a/src/ofw/preparation/policy.py b/src/ofw/preparation/policy.py index 5f15fca..7f6929a 100644 --- a/src/ofw/preparation/policy.py +++ b/src/ofw/preparation/policy.py @@ -54,6 +54,7 @@ class _ExperimentPolicyContent(StrictModel): quality_target: NormalizedScore max_iterations: IterationCount no_improvement_limit: IterationCount + baseline_reused: bool = False max_cost_per_task_usd: PositiveMetric | None = None max_latency_seconds: PositiveMetric | None = None max_baseline_seconds: int = Field(strict=True, ge=60, le=172800) @@ -174,6 +175,7 @@ def build_experiment_policy( quality_target=request.quality_target, max_iterations=request.max_iterations, no_improvement_limit=request.no_improvement_limit, + baseline_reused=request.reuse_existing_baseline, max_cost_per_task_usd=request.max_cost_per_task_usd, max_latency_seconds=request.max_latency_seconds, max_baseline_seconds=request.max_baseline_seconds, @@ -197,6 +199,7 @@ def _content_from_snapshot(policy: ExperimentPolicySnapshot) -> _ExperimentPolic quality_target=policy.quality_target, max_iterations=policy.max_iterations, no_improvement_limit=policy.no_improvement_limit, + baseline_reused=policy.baseline_reused, max_cost_per_task_usd=policy.max_cost_per_task_usd, max_latency_seconds=policy.max_latency_seconds, max_baseline_seconds=policy.max_baseline_seconds, @@ -219,6 +222,7 @@ def _snapshot_from_content(content: _ExperimentPolicyContent) -> ExperimentPolic quality_target=content.quality_target, max_iterations=content.max_iterations, no_improvement_limit=content.no_improvement_limit, + baseline_reused=content.baseline_reused, max_cost_per_task_usd=content.max_cost_per_task_usd, max_latency_seconds=content.max_latency_seconds, max_baseline_seconds=content.max_baseline_seconds, diff --git a/src/ofw/preparation/service.py b/src/ofw/preparation/service.py index 130e7c5..decbecc 100644 --- a/src/ofw/preparation/service.py +++ b/src/ofw/preparation/service.py @@ -55,6 +55,7 @@ class _PreparationStateWire(StrictModel): benchmark_config_digest: str = Field(pattern=_DIGEST_PATTERN) verifier: str = Field(min_length=1, max_length=128) environment: str = Field(min_length=1, max_length=128) + baseline_reused: bool = False policy_published: bool = False process_id: int | None = Field(default=None, strict=True, ge=1) started_at: datetime @@ -133,8 +134,32 @@ def _start( ) state = _initial_state(request, state_directory, digest, prepared, configuration) _write_state(state_directory, state) + if request.reuse_existing_baseline: + return self._adopt_existing(request, state_directory, state) return self._resume_start(request, state_directory, state) + def _adopt_existing( + self, + request: PrepareWorkspaceInput, + state_directory: Path, + state: _PreparationStateWire, + ) -> WorkspacePreparationObservation: + summary = self._runner.summarize(_run_from_state(request, state)) + if summary is None: + raise PreparationFailure( + PreparationErrorCode.INVALID_BASELINE_RESULT, + "existing baseline", + ) + _publish_policy( + state_directory, + request, + _prepared_from_state(state), + _configuration_from_state(state), + ) + ready = _ready_state(_state_with_policy(state), summary, request.expected_task_count) + _write_state(state_directory, ready) + return _ready_observation(request, ready) + def _resume_start( self, request: PrepareWorkspaceInput, @@ -176,9 +201,19 @@ def _poll( PreparationErrorCode.REQUEST_CONFLICT, request.experiment_id, ) + if state.baseline_reused: + return self._adopt_existing(request, state_directory, state) continuation = self._continue_existing(request, state_directory, state) if continuation is not None: return continuation + return self._poll_running_baseline(request, state_directory, state) + + def _poll_running_baseline( + self, + request: PrepareWorkspaceInput, + state_directory: Path, + state: _PreparationStateWire, + ) -> WorkspacePreparationObservation: run = _run_from_state(request, state) summary = self._runner.summarize(run) if summary is not None: @@ -267,6 +302,7 @@ def _initial_state( benchmark_config_digest=configuration.benchmark_config_digest, verifier=configuration.verifier, environment=configuration.environment, + baseline_reused=request.reuse_existing_baseline, started_at=started_at, deadline_at=started_at + timedelta(seconds=request.max_baseline_seconds), ) @@ -423,6 +459,7 @@ def _terminal_state( benchmark_config_digest=state.benchmark_config_digest, verifier=state.verifier, environment=state.environment, + baseline_reused=state.baseline_reused, policy_published=state.policy_published, process_id=state.process_id, started_at=state.started_at, @@ -467,6 +504,7 @@ def _state_with_process( benchmark_config_digest=state.benchmark_config_digest, verifier=state.verifier, environment=state.environment, + baseline_reused=state.baseline_reused, policy_published=state.policy_published, process_id=process_id, started_at=state.started_at, @@ -490,6 +528,7 @@ def _state_with_policy(state: _PreparationStateWire) -> _PreparationStateWire: benchmark_config_digest=state.benchmark_config_digest, verifier=state.verifier, environment=state.environment, + baseline_reused=state.baseline_reused, policy_published=True, process_id=state.process_id, started_at=state.started_at, diff --git a/src/ofw/preparation/templates/base.md b/src/ofw/preparation/templates/base.md index 3a20d41..1fd3954 100644 --- a/src/ofw/preparation/templates/base.md +++ b/src/ofw/preparation/templates/base.md @@ -8,7 +8,8 @@ Record one evidence-backed hypothesis for the connected agent harness under the experiment policy, then stop before candidate editing. The baseline has already been recorded. Begin at step 2; do not rerun the unchanged -baseline. +baseline. Its provenance is recorded in the policy (`baseline_reused` is explicit when an +existing terminal Harbor job was adopted). ## Authority @@ -49,7 +50,8 @@ expected improvement, and possible regressions. Stop if the evidence cannot dist between materially different changes. Use `$hypothesis-former` with one curation receipt and group ID plus every exact supported -pattern and diagnosis receipt ID in that group, then call `record_hypothesis`. Retain the stable hypothesis receipt and stop before candidate editing. +pattern and diagnosis receipt ID in that group, explicit predicted task IDs, and at-risk task IDs, then call +`record_hypothesis`. Retain the stable hypothesis receipt and stop before candidate editing. Candidate editing requires a later package and must not begin in this prepared program. ## Package boundary diff --git a/src/ofw/preparation/worktree.py b/src/ofw/preparation/worktree.py index 25cf4b6..85e0d78 100644 --- a/src/ofw/preparation/worktree.py +++ b/src/ofw/preparation/worktree.py @@ -191,6 +191,7 @@ def _render_experiment_config( f" harbor_config: {_yaml(request.harbor_config.as_posix())}\n" f" job_name: {_yaml(request.experiment_id)}\n" f" expected_task_count: {baseline.task_count}\n" + f" reuse_existing_baseline: {str(request.reuse_existing_baseline).lower()}\n" "harness:\n" f" branch: {_yaml(branch_name)}\n" f" base_commit: {_yaml(base_commit)}\n" diff --git a/tests/test_experiment_policy.py b/tests/test_experiment_policy.py index 01f5cea..ba522fb 100644 --- a/tests/test_experiment_policy.py +++ b/tests/test_experiment_policy.py @@ -117,6 +117,7 @@ def test_policy_is_derived_from_validated_preparation_inputs(tmp_path: Path) -> assert policy.environment == "itsm-bench" assert policy.concurrency == 1 assert policy.max_retries == 0 + assert policy.baseline_reused is False assert policy.controls_digest.startswith("sha256:") assert policy.controls_digest == policy.recomputed_controls_digest() diff --git a/tests/test_hypothesis.py b/tests/test_hypothesis.py index 1a8c5f6..9a9f433 100644 --- a/tests/test_hypothesis.py +++ b/tests/test_hypothesis.py @@ -202,6 +202,8 @@ def _request( source_commit=commit, curation_id=curation_id, curation_group_id=curation_group_id, + predicted_task_ids=("task-1", "task-2"), + at_risk_task_ids=("task-3",), patterns=( FailurePatternReferenceInput( pattern_id=failure_pattern_id( @@ -232,6 +234,8 @@ def _with_patterns( source_commit=request.source_commit, curation_id=request.curation_id, curation_group_id=request.curation_group_id, + predicted_task_ids=request.predicted_task_ids, + at_risk_task_ids=request.at_risk_task_ids, patterns=patterns, statement=request.statement, rationale=request.rationale, @@ -251,6 +255,8 @@ def _with_target( source_commit=request.source_commit, curation_id=request.curation_id, curation_group_id=request.curation_group_id, + predicted_task_ids=request.predicted_task_ids, + at_risk_task_ids=request.at_risk_task_ids, patterns=request.patterns, statement=request.statement, rationale=request.rationale, @@ -270,6 +276,8 @@ def _with_curation_group( source_commit=request.source_commit, curation_id=request.curation_id, curation_group_id=curation_group_id, + predicted_task_ids=request.predicted_task_ids, + at_risk_task_ids=request.at_risk_task_ids, patterns=request.patterns, statement=request.statement, rationale=request.rationale, @@ -488,6 +496,8 @@ def test_hypothesis_input_rejects_duplicates_escapes_empty_and_extra_fields(tmp_ source_commit=request.source_commit, curation_id=request.curation_id, curation_group_id=request.curation_group_id, + predicted_task_ids=request.predicted_task_ids, + at_risk_task_ids=request.at_risk_task_ids, patterns=request.patterns, statement=request.statement, rationale=request.rationale, @@ -558,6 +568,8 @@ def test_hypothesis_input_accepts_its_exact_global_maximum() -> None: source_commit="1" * 40, curation_id="00000000-0000-0000-0000-000000000001", curation_group_id="00000000-0000-0000-0000-000000000002", + predicted_task_ids=("task-1",), + at_risk_task_ids=("task-2",), patterns=( FailurePatternReferenceInput( pattern_id="sha256:" + "1" * 64, @@ -579,6 +591,33 @@ def test_hypothesis_input_accepts_its_exact_global_maximum() -> None: assert len(request.regression_risks) == 10 +def test_hypothesis_predictions_must_be_disjoint() -> None: + with pytest.raises(ValidationError): + RecordHypothesisInput( + workspace_root=Path("/prepared"), + experiment_id="experiment-one", + source_commit="1" * 40, + curation_id="00000000-0000-0000-0000-000000000001", + curation_group_id="00000000-0000-0000-0000-000000000002", + predicted_task_ids=("task-1",), + at_risk_task_ids=("task-1",), + patterns=( + FailurePatternReferenceInput( + pattern_id="sha256:" + "1" * 64, + diagnosis_artifact_ids=("00000000-0000-0000-0000-000000000001",), + ), + ), + statement="s", + rationale="r", + target=HarnessChangeTargetInput( + component_kind=ComponentKind.PROMPT, + relative_paths=(Path("prompt.md"),), + ), + expected_effect="e", + regression_risks=(), + ) + + def test_hypothesis_rejects_an_incomplete_curation_group(tmp_path: Path) -> None: root, commit = _workspace(tmp_path) first = _diagnosis(root, "1") diff --git a/tests/test_openflywheel_mcp.py b/tests/test_openflywheel_mcp.py index 12e84b5..85d39d8 100644 --- a/tests/test_openflywheel_mcp.py +++ b/tests/test_openflywheel_mcp.py @@ -343,6 +343,8 @@ def _hypothesis_request(root: Path) -> RecordHypothesisInput: source_commit=_COMMIT, curation_id=_CURATION_ID, curation_group_id=_CURATION_GROUP_ID, + predicted_task_ids=("task-1",), + at_risk_task_ids=("task-2",), patterns=( FailurePatternReferenceInput( pattern_id=_PATTERN_ID, @@ -372,6 +374,8 @@ def _hypothesis_observation() -> HypothesisObservation: source_commit=_COMMIT, curation_id=_CURATION_ID, curation_group_id=_CURATION_GROUP_ID, + predicted_task_ids=("task-1",), + at_risk_task_ids=("task-2",), relative_path=relative_path, pattern_count=1, diagnosis_count=1, diff --git a/tests/test_workspace_preparation.py b/tests/test_workspace_preparation.py index a0ffda8..4af4779 100644 --- a/tests/test_workspace_preparation.py +++ b/tests/test_workspace_preparation.py @@ -57,6 +57,24 @@ def summarize(self, run: BaselineRun) -> BaselineSummary | None: return None +class _AdoptingRunner: + def validate(self, request: PrepareWorkspaceInput) -> BaselineConfiguration: + return BaselineConfiguration( + model="openai/gpt-5.4-mini", + task_ids=("task-pass", "task-fail"), + benchmark_config_digest="sha256:" + "1" * 64, + verifier="itsm-bench", + environment="itsm-bench", + ) + + def start(self, run: BaselineRun) -> int: + raise AssertionError(f"adopted baseline must not launch Harbor: {run.job_path}") + + def summarize(self, run: BaselineRun) -> BaselineSummary | None: + assert run.job_path.name == "itsm-hermes-demo" + return BaselineSummary(2, 1, 1, 0, 0) + + def _git(root: Path, *arguments: str) -> str: return subprocess.run( ("git", "-C", str(root), *arguments), @@ -175,6 +193,7 @@ def _request( *, goal: str = "Reach full ITSM verifier pass rate.", expected_task_count: int = 2, + reuse_existing_baseline: bool = False, ) -> PrepareWorkspaceInput: return PrepareWorkspaceInput( experiment_id="itsm-hermes-demo", @@ -193,6 +212,7 @@ def _request( max_cost_per_task_usd=1.0, max_latency_seconds=600.0, max_baseline_seconds=60, + reuse_existing_baseline=reuse_existing_baseline, ) @@ -312,6 +332,46 @@ def test_prepare_workspace_creates_isolated_branch_commit_and_baseline( assert (benchmark_root / "invocations.txt").read_text(encoding="utf-8") == "run\n" +def test_prepare_workspace_adopts_existing_terminal_baseline_without_launching( + tmp_path: Path, +) -> None: + harness_root = _harness_repository(tmp_path) + harbor = _fake_harbor(tmp_path) + benchmark_root, config = _benchmark_repository(tmp_path, harbor) + worktree_parent = tmp_path / "worktrees" + worktree_parent.mkdir() + request = _request( + harness_root, + worktree_parent, + benchmark_root, + harbor, + config, + reuse_existing_baseline=True, + ) + job = benchmark_root / "jobs/itsm-hermes-demo" + job.mkdir(parents=True) + (job / "result.json").write_text('{"finished_at":"2026-08-27T20:01:02Z"}', encoding="utf-8") + + service = WorkspacePreparationService( + runner=_AdoptingRunner(), + workspace=GitWorktreeGateway(), + base_program="# Base program\n", + itsm_program="## ITSM\n", + ) + + result = service.prepare(request) + + assert result.status is PreparationStatus.SUCCESS + assert result.phase is PreparationPhase.READY + assert result.terminal_trials == 2 + assert result.verifier_passes == 1 + assert result.verifier_failures == 1 + assert not (benchmark_root / "invocations.txt").exists() + policy = FileExperimentPolicyRepository().load(harness_root, request.experiment_id) + assert policy.task_ids == ("task-pass", "task-fail") + assert policy.baseline_reused is True + + def test_prepare_workspace_rejects_reused_id_with_different_configuration( tmp_path: Path, monkeypatch: pytest.MonkeyPatch, From a63fc061442265ca9b2e75fc056675ec2525e956 Mon Sep 17 00:00:00 2001 From: divo12 Date: Thu, 3 Sep 2026 17:34:43 +0530 Subject: [PATCH 11/25] chore: pin plugin runtime to experiment fix --- plugins/openflywheel/.mcp.json | 2 +- tests/test_plugin_packaging.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/plugins/openflywheel/.mcp.json b/plugins/openflywheel/.mcp.json index 3e05ef6..1078b2a 100644 --- a/plugins/openflywheel/.mcp.json +++ b/plugins/openflywheel/.mcp.json @@ -4,7 +4,7 @@ "command": "uvx", "args": [ "--from", - "git+https://github.com/divo12/OpenFlyWheel.git@48d7363d348d49baf6496a0100054ce246d35e4c", + "git+https://github.com/divo12/OpenFlyWheel.git@fe5d0972ef4733553454814e96910ae51ab6a5a3", "--with", "mcp>=1.13,<2", "openflywheel-mcp" diff --git a/tests/test_plugin_packaging.py b/tests/test_plugin_packaging.py index c31dffa..0826b94 100644 --- a/tests/test_plugin_packaging.py +++ b/tests/test_plugin_packaging.py @@ -28,7 +28,7 @@ def test_openflywheel_mcp_uses_pinned_portable_runtime() -> None: manifest = _McpManifest.model_validate_json(path.read_text(encoding="utf-8")) server = manifest.mcpServers["openflywheel"] assert ( - "git+https://github.com/divo12/OpenFlyWheel.git@48d7363d348d49baf6496a0100054ce246d35e4c" + "git+https://github.com/divo12/OpenFlyWheel.git@fe5d0972ef4733553454814e96910ae51ab6a5a3" in server.args ) runtime = (root / "src/ofw/mcp.py").read_text(encoding="utf-8") From b4b30d3893303198719e7d431f179b3f1232be84 Mon Sep 17 00:00:00 2001 From: divo12 Date: Wed, 2 Sep 2026 19:14:30 +0530 Subject: [PATCH 12/25] feat: execute isolated candidates --- .../openflywheel/program_templates/base.md | 28 +- src/ofw/__init__.py | 20 + src/ofw/evolution/__init__.py | 28 + src/ofw/evolution/candidate.py | 285 ++++++ src/ofw/evolution/candidate_git.py | 289 ++++++ src/ofw/evolution/candidate_langfuse.py | 51 ++ src/ofw/evolution/candidate_service.py | 669 ++++++++++++++ src/ofw/evolution/hypothesis.py | 26 +- src/ofw/evolution/hypothesis_repository.py | 21 + src/ofw/mcp.py | 30 +- src/ofw/preparation/__init__.py | 8 + src/ofw/preparation/contracts.py | 44 + src/ofw/preparation/harbor.py | 251 +++++- src/ofw/preparation/templates/base.md | 28 +- tests/test_candidate_execution.py | 825 ++++++++++++++++++ tests/test_harbor_experiment.py | 268 ++++++ tests/test_openflywheel_mcp.py | 59 ++ tests/test_program_templates.py | 6 +- tests/test_typing.py | 13 + 19 files changed, 2901 insertions(+), 48 deletions(-) create mode 100644 src/ofw/evolution/candidate.py create mode 100644 src/ofw/evolution/candidate_git.py create mode 100644 src/ofw/evolution/candidate_langfuse.py create mode 100644 src/ofw/evolution/candidate_service.py create mode 100644 tests/test_candidate_execution.py create mode 100644 tests/test_harbor_experiment.py diff --git a/plugins/openflywheel/program_templates/base.md b/plugins/openflywheel/program_templates/base.md index 1fd3954..c440f02 100644 --- a/plugins/openflywheel/program_templates/base.md +++ b/plugins/openflywheel/program_templates/base.md @@ -4,8 +4,8 @@ This file is generated by `prepare_workspace`. Do not edit it directly. ## Mission -Record one evidence-backed hypothesis for the connected agent harness under the canonical -experiment policy, then stop before candidate editing. +Record one evidence-backed hypothesis, execute its isolated candidate under the canonical +experiment policy, then stop before admission. The baseline has already been recorded. Begin at step 2; do not rerun the unchanged baseline. Its provenance is recorded in the policy (`baseline_reused` is explicit when an @@ -27,9 +27,9 @@ existing terminal Harbor job was adopted). ## Editable and frozen surfaces -Target only exact paths allowed by the canonical experiment policy. Do not edit them in this -program. Never target the benchmark, held-out tasks, verifier, model, reasoning budget, -observability identity, or this program. +Target only exact paths allowed by the canonical experiment policy. Edit them only in the +candidate worktree returned by `execute_candidate`. Never target the benchmark, held-out tasks, +verifier, model, reasoning budget, observability identity, or this program. Keep one focused hypothesis per iteration. Do not mix prompt, tool, middleware, and control flow changes unless the evidence requires the combination. @@ -51,10 +51,20 @@ between materially different changes. Use `$hypothesis-former` with one curation receipt and group ID plus every exact supported pattern and diagnosis receipt ID in that group, explicit predicted task IDs, and at-risk task IDs, then call -`record_hypothesis`. Retain the stable hypothesis receipt and stop before candidate editing. -Candidate editing requires a later package and must not begin in this prepared program. +`record_hypothesis`. Retain the stable hypothesis receipt before candidate execution. + +### 4. Execute one candidate + +Call `execute_candidate` with the prepared workspace, experiment and hypothesis receipts, sibling +candidate-worktree parent, and Harbor runtime locations. The first call creates the isolated +worktree from the accepted experiment commit. Edit only the exact hypothesis targets in the +returned candidate worktree, then call `execute_candidate` again with the identical request. + +Poll identical requests while the candidate is running. Retain its candidate ID, Git commit, +trace-mapping blockers, and authoritative outcome receipts. Do not copy trace payloads locally, +change frozen controls, rerun an empty candidate, or edit the accepted experiment worktree. ## Package boundary -Report the hypothesis receipt and its exact evidence and target paths. Do not edit, commit, -run, gate, publish, or install a candidate; those capabilities are not part of this program. +Report the hypothesis, candidate, commit, blocker, and outcome receipts. Stop before admission: +do not gate, accept, merge, publish, push, or install the candidate. diff --git a/src/ofw/__init__.py b/src/ofw/__init__.py index 077b3bb..8f642c9 100644 --- a/src/ofw/__init__.py +++ b/src/ofw/__init__.py @@ -45,6 +45,16 @@ VerifierVerdict, ) from ofw.evolution import ( + CandidateBlocker, + CandidateBlockerCode, + CandidateErrorCode, + CandidateExecutionInput, + CandidateExecutionObservation, + CandidateFailure, + CandidateId, + CandidateOutcomeReceipt, + CandidatePhase, + CandidateStatus, FailurePatternReference, FailurePatternReferenceInput, HarnessChangeTarget, @@ -76,6 +86,16 @@ ) __all__ = [ + "CandidateBlocker", + "CandidateBlockerCode", + "CandidateErrorCode", + "CandidateExecutionInput", + "CandidateExecutionObservation", + "CandidateFailure", + "CandidateId", + "CandidateOutcomeReceipt", + "CandidatePhase", + "CandidateStatus", "CollectionError", "CollectionErrorCode", "ComponentKind", diff --git a/src/ofw/evolution/__init__.py b/src/ofw/evolution/__init__.py index a05c3c2..866f0ca 100644 --- a/src/ofw/evolution/__init__.py +++ b/src/ofw/evolution/__init__.py @@ -1,5 +1,20 @@ """Prepared-experiment evolution contracts.""" +from ofw.evolution.candidate import ( + CandidateBlocker, + CandidateBlockerCode, + CandidateErrorCode, + CandidateExecutionInput, + CandidateExecutionObservation, + CandidateFailure, + CandidateId, + CandidateOutcomeReceipt, + CandidatePhase, + CandidateStatus, +) +from ofw.evolution.candidate_git import CandidateGitGateway +from ofw.evolution.candidate_langfuse import LangfuseCandidateTraceLocator +from ofw.evolution.candidate_service import CandidateExecutionService from ofw.evolution.hypothesis import ( FailurePatternReference, FailurePatternReferenceInput, @@ -17,6 +32,18 @@ from ofw.evolution.hypothesis_repository import FileHypothesisRepository __all__ = [ + "CandidateBlocker", + "CandidateBlockerCode", + "CandidateErrorCode", + "CandidateExecutionInput", + "CandidateExecutionObservation", + "CandidateExecutionService", + "CandidateFailure", + "CandidateGitGateway", + "CandidateId", + "CandidateOutcomeReceipt", + "CandidatePhase", + "CandidateStatus", "FailurePatternReference", "FailurePatternReferenceInput", "FileHypothesisRepository", @@ -29,5 +56,6 @@ "HypothesisObservation", "HypothesisService", "HypothesisStatus", + "LangfuseCandidateTraceLocator", "RecordHypothesisInput", ] diff --git a/src/ofw/evolution/candidate.py b/src/ofw/evolution/candidate.py new file mode 100644 index 0000000..1e8c85e --- /dev/null +++ b/src/ofw/evolution/candidate.py @@ -0,0 +1,285 @@ +"""Immutable candidate-execution contracts.""" + +from __future__ import annotations + +import hashlib +import re +from dataclasses import dataclass +from datetime import datetime +from enum import StrEnum +from pathlib import Path +from typing import Literal, Protocol + +from pydantic import Field, field_validator + +from ofw.evaluation.langfuse import OutcomeScoreSubmission +from ofw.evaluation.outcome import OutcomeEvaluation, VerifierVerdict +from ofw.evolution.hypothesis import HarnessHypothesis, StrictModel +from ofw.preparation.contracts import ( + ExperimentControls, + ExperimentRun, + ExperimentSummary, + PathValue, + contained_relative_path, +) +from ofw.preparation.policy import ExperimentPolicySnapshot + +_DIGEST_PATTERN = re.compile(r"sha256:[0-9a-f]{64}") + + +class _CandidateIdentity(StrictModel): + schema_version: Literal[1] = 1 + policy_digest: str = Field(pattern=r"sha256:[0-9a-f]{64}") + hypothesis_id: str = Field(pattern=r"sha256:[0-9a-f]{64}") + source_commit: str = Field(pattern=r"[0-9a-f]{40}") + candidate_tree: str = Field(pattern=r"[0-9a-f]{40}") + controls_digest: str = Field(pattern=r"sha256:[0-9a-f]{64}") + + +@dataclass(frozen=True, slots=True) +class CandidateId: + value: str + + def __post_init__(self) -> None: + if _DIGEST_PATTERN.fullmatch(self.value) is None: + raise ValueError("invalid candidate id") + + def __str__(self) -> str: + return self.value + + @classmethod + def build( + cls, + *, + policy_digest: str, + hypothesis_id: str, + source_commit: str, + candidate_tree: str, + controls_digest: str, + ) -> CandidateId: + identity = _CandidateIdentity( + policy_digest=policy_digest, + hypothesis_id=hypothesis_id, + source_commit=source_commit, + candidate_tree=candidate_tree, + controls_digest=controls_digest, + ) + digest = hashlib.sha256(identity.model_dump_json().encode("utf-8")).hexdigest() + return cls(f"sha256:{digest}") + + +def candidate_policy_digest(policy: ExperimentPolicySnapshot) -> str: + digest = hashlib.sha256(policy.model_dump_json().encode("utf-8")).hexdigest() + return f"sha256:{digest}" + + +class CandidateErrorCode(StrEnum): + INVALID_WORKSPACE = "invalid_workspace" + WORKTREE_EXISTS = "worktree_exists" + STALE_COMMIT = "stale_commit" + STALE_POLICY = "stale_policy" + EMPTY_CANDIDATE = "empty_candidate" + OUT_OF_SCOPE = "out_of_scope" + UNSAFE_PATH = "unsafe_path" + MANAGED_PATH = "managed_path" + CREDENTIAL_PATH = "credential_path" + CONTROLS_DRIFT = "controls_drift" + REQUEST_CONFLICT = "request_conflict" + MISSING_ENVIRONMENT = "missing_environment" + LAUNCH_FAILED = "launch_failed" + INVALID_RESULT = "invalid_result" + OUTCOME_STORE_FAILED = "outcome_store_failed" + GIT_FAILED = "git_failed" + + +class CandidateFailure(Exception): + __slots__ = ("code", "subject") + + def __init__(self, code: CandidateErrorCode, subject: str) -> None: + self.code = code + self.subject = subject + super().__init__(f"{code.value}: {subject}") + + +@dataclass(frozen=True, slots=True) +class CandidateWorkspace: + accepted_root: Path + worktree_path: Path + source_commit: str + + +@dataclass(frozen=True, slots=True) +class CandidateTree: + tree_id: str + changed_paths: tuple[Path, ...] + + +@dataclass(frozen=True, slots=True) +class CandidateCommit: + commit: str + + +class CandidatePhase(StrEnum): + EDITING = "editing" + RUNNING = "running" + COMPLETE = "complete" + FAILED = "failed" + + +class CandidateStatus(StrEnum): + SUCCESS = "success" + WARNING = "warning" + ERROR = "error" + + +class CandidateBlockerCode(StrEnum): + TRACE_NOT_FOUND = "trace_not_found" + TRACE_AMBIGUOUS = "trace_ambiguous" + UNVERIFIED = "unverified" + UNSUPPORTED_REWARD = "unsupported_reward" + + +class CandidateExecutionInput(StrictModel): + workspace_root: PathValue + worktree_parent: PathValue + benchmark_root: PathValue + harbor_executable: PathValue + harbor_config: PathValue + experiment_id: str = Field(pattern=r"[a-z0-9]+(?:-[a-z0-9]+)*", max_length=80) + hypothesis_id: str = Field(pattern=r"sha256:[0-9a-f]{64}") + + @field_validator( + "workspace_root", + "worktree_parent", + "benchmark_root", + "harbor_executable", + ) + @classmethod + def validate_absolute(cls, value: Path) -> Path: + if not value.is_absolute(): + raise ValueError("path must be absolute") + return value + + @field_validator("harbor_config") + @classmethod + def validate_config(cls, value: Path) -> Path: + return contained_relative_path(value, "harbor_config") + + +class CandidateOutcomeReceipt(StrictModel): + task_id: str = Field(min_length=1, max_length=256) + trace_id: str = Field(min_length=1, max_length=256) + score_id: str = Field(min_length=1, max_length=256) + verdict: VerifierVerdict + + +class CandidateBlocker(StrictModel): + task_id: str = Field(min_length=1, max_length=256) + code: CandidateBlockerCode + subject: str = Field(min_length=1, max_length=256) + + +class CandidateExecutionObservation(StrictModel): + status: CandidateStatus + summary: str = Field(min_length=1, max_length=256) + next_actions: tuple[str, ...] = Field(max_length=2) + artifacts: tuple[str, ...] = Field(max_length=10) + phase: CandidatePhase + experiment_id: str = Field(min_length=1, max_length=80) + hypothesis_id: str = Field(pattern=r"sha256:[0-9a-f]{64}") + source_commit: str | None = Field(default=None, pattern=r"[0-9a-f]{40}") + candidate_id: str | None = Field(default=None, pattern=r"sha256:[0-9a-f]{64}") + candidate_tree: str | None = Field(default=None, pattern=r"[0-9a-f]{40}") + candidate_commit: str | None = Field(default=None, pattern=r"[0-9a-f]{40}") + worktree_path: Path | None = None + job_path: Path | None = None + session_id: str | None = Field(default=None, max_length=199) + terminal_trials: int | None = Field(default=None, ge=0, le=500) + verifier_passes: int | None = Field(default=None, ge=0, le=500) + verifier_failures: int | None = Field(default=None, ge=0, le=500) + unverified_trials: int | None = Field(default=None, ge=0, le=500) + outcome_receipts: tuple[CandidateOutcomeReceipt, ...] = Field(max_length=500) + blockers: tuple[CandidateBlocker, ...] = Field(max_length=500) + next_poll_after_seconds: int | None = Field(default=None, ge=1, le=300) + error_code: CandidateErrorCode | None = None + + +@dataclass(frozen=True, slots=True) +class TraceMatchRequest: + task_id: str + session_id: str + environment: str + release: str + started_at: datetime + finished_at: datetime + + +@dataclass(frozen=True, slots=True) +class TraceMatch: + trace_id: str | None + blocker: CandidateBlockerCode | None + + def __post_init__(self) -> None: + if (self.trace_id is None) == (self.blocker is None): + raise ValueError("trace match requires exactly one result") + + +class CandidateWorkspaceGateway(Protocol): + def control_directory(self, root: Path, hypothesis_id: str) -> Path: ... + + def prepare( + self, + accepted_root: Path, + worktree_parent: Path, + policy: ExperimentPolicySnapshot, + hypothesis: HarnessHypothesis, + ) -> CandidateWorkspace: ... + + def inspect( + self, + workspace: CandidateWorkspace, + policy: ExperimentPolicySnapshot, + hypothesis: HarnessHypothesis, + ) -> CandidateTree: ... + + def commit( + self, + workspace: CandidateWorkspace, + tree: CandidateTree, + candidate_id: CandidateId, + experiment_id: str, + ) -> CandidateCommit: ... + + def validate_accepted( + self, + root: Path, + policy: ExperimentPolicySnapshot, + hypothesis: HarnessHypothesis, + ) -> None: ... + + +class CandidateHypothesisRepository(Protocol): + def load_policy(self, root: Path, experiment_id: str) -> ExperimentPolicySnapshot: ... + + def load(self, root: Path, hypothesis_id: str) -> HarnessHypothesis: ... + + +class CandidateExperimentRunner(Protocol): + def validate( + self, + benchmark_root: Path, + harbor_executable: Path, + harbor_config: Path, + ) -> ExperimentControls: ... + + def start(self, run: ExperimentRun) -> int: ... + + def summarize(self, run: ExperimentRun) -> ExperimentSummary | None: ... + + +class CandidateTraceLocator(Protocol): + def locate(self, request: TraceMatchRequest) -> TraceMatch: ... + + +class CandidateOutcomeStore(Protocol): + def store(self, outcome: OutcomeEvaluation) -> OutcomeScoreSubmission: ... diff --git a/src/ofw/evolution/candidate_git.py b/src/ofw/evolution/candidate_git.py new file mode 100644 index 0000000..feabeec --- /dev/null +++ b/src/ofw/evolution/candidate_git.py @@ -0,0 +1,289 @@ +"""Git isolation and exact-path sealing for one harness candidate.""" + +from __future__ import annotations + +import stat +import subprocess # nosec B404 +from pathlib import Path, PurePosixPath + +from ofw.evolution.candidate import ( + CandidateCommit, + CandidateErrorCode, + CandidateFailure, + CandidateId, + CandidateTree, + CandidateWorkspace, +) +from ofw.evolution.hypothesis import HarnessHypothesis +from ofw.preparation.policy import ExperimentPolicySnapshot + +_MANAGED_PATHS = frozenset(("PROGRAM.md", "experiment_config.yaml")) +_CREDENTIAL_NAMES = frozenset( + ( + "credentials.json", + "credentials.yaml", + "credentials.yml", + "secrets.json", + "secrets.yaml", + "secrets.yml", + ) +) + + +class CandidateGitGateway: + """Create a detached candidate worktree and commit only hypothesis targets.""" + + def control_directory(self, root: Path, hypothesis_id: str) -> Path: + common = Path(_git(root, "rev-parse", "--git-common-dir")) + if not common.is_absolute(): + common = root / common + return common.resolve() / "ofw" / "candidates" / hypothesis_id.removeprefix("sha256:") + + def validate_accepted( + self, + root: Path, + policy: ExperimentPolicySnapshot, + hypothesis: HarnessHypothesis, + ) -> None: + _validate_authority(_directory(root, "workspace_root"), policy, hypothesis) + + def prepare( + self, + accepted_root: Path, + worktree_parent: Path, + policy: ExperimentPolicySnapshot, + hypothesis: HarnessHypothesis, + ) -> CandidateWorkspace: + root = _directory(accepted_root, "workspace_root") + parent = _directory(worktree_parent, "worktree_parent") + _validate_authority(root, policy, hypothesis) + worktree = parent / _worktree_name(root, policy, hypothesis) + if worktree.exists(): + raise CandidateFailure(CandidateErrorCode.WORKTREE_EXISTS, str(worktree)) + _git(root, "worktree", "add", "--detach", str(worktree), policy.initialization_commit) + return CandidateWorkspace( + accepted_root=root, + worktree_path=worktree, + source_commit=policy.initialization_commit, + ) + + def inspect( + self, + workspace: CandidateWorkspace, + policy: ExperimentPolicySnapshot, + hypothesis: HarnessHypothesis, + ) -> CandidateTree: + _validate_authority(workspace.accepted_root, policy, hypothesis) + _require_head(workspace.worktree_path, workspace.source_commit) + changed = _changed_paths(workspace.worktree_path) + if not changed: + raise CandidateFailure(CandidateErrorCode.EMPTY_CANDIDATE, hypothesis.id.value) + _validate_changes(workspace.worktree_path, changed, policy, hypothesis) + _git( + workspace.worktree_path, + "add", + "--all", + "--", + *(path.as_posix() for path in hypothesis.target.relative_paths), + ) + return CandidateTree( + tree_id=_git(workspace.worktree_path, "write-tree"), + changed_paths=changed, + ) + + def commit( + self, + workspace: CandidateWorkspace, + tree: CandidateTree, + candidate_id: CandidateId, + experiment_id: str, + ) -> CandidateCommit: + _require_head(workspace.worktree_path, workspace.source_commit) + if ( + _changed_paths(workspace.worktree_path) != tree.changed_paths + or not _git_succeeds(workspace.worktree_path, "diff", "--quiet") + or _git(workspace.worktree_path, "write-tree") != tree.tree_id + ): + raise CandidateFailure(CandidateErrorCode.STALE_COMMIT, experiment_id) + message = ( + "feat(ofw): record candidate execution\n\n" + f"OFW-Experiment: {experiment_id}\n" + f"OFW-Run: {candidate_id.value}" + ) + _git(workspace.worktree_path, "commit", "-m", message) + commit = _git(workspace.worktree_path, "rev-parse", "HEAD") + if _changed_paths(workspace.worktree_path): + raise CandidateFailure(CandidateErrorCode.STALE_COMMIT, experiment_id) + return CandidateCommit(commit=commit) + + +def _validate_authority( + root: Path, + policy: ExperimentPolicySnapshot, + hypothesis: HarnessHypothesis, +) -> None: + _require_experiment(policy, hypothesis) + _require_source(policy, hypothesis) + _require_head(root, policy.initialization_commit) + _require_branch(root, policy) + _require_targets(policy, hypothesis) + + +def _require_experiment( + policy: ExperimentPolicySnapshot, + hypothesis: HarnessHypothesis, +) -> None: + if hypothesis.experiment_id != policy.experiment_id: + raise CandidateFailure(CandidateErrorCode.STALE_POLICY, hypothesis.id.value) + + +def _require_source( + policy: ExperimentPolicySnapshot, + hypothesis: HarnessHypothesis, +) -> None: + if hypothesis.source_commit != policy.initialization_commit: + raise CandidateFailure(CandidateErrorCode.STALE_COMMIT, hypothesis.id.value) + + +def _require_branch(root: Path, policy: ExperimentPolicySnapshot) -> None: + if _git(root, "branch", "--show-current") != policy.branch_name: + raise CandidateFailure(CandidateErrorCode.STALE_POLICY, policy.experiment_id) + + +def _require_targets( + policy: ExperimentPolicySnapshot, + hypothesis: HarnessHypothesis, +) -> None: + if any(path not in policy.editable_paths for path in hypothesis.target.relative_paths): + raise CandidateFailure(CandidateErrorCode.STALE_POLICY, hypothesis.id.value) + + +def _require_head(root: Path, expected: str) -> None: + if _git(root, "rev-parse", "HEAD") != expected: + raise CandidateFailure(CandidateErrorCode.STALE_COMMIT, expected) + + +def _changed_paths(root: Path) -> tuple[Path, ...]: + tracked = _git_paths(root, "diff", "--name-only", "--no-renames", "-z", "HEAD") + untracked = _git_paths(root, "ls-files", "--others", "--exclude-standard", "-z") + ignored = _git_paths(root, "ls-files", "--others", "--ignored", "--exclude-standard", "-z") + paths = tracked | untracked | ignored + return tuple(sorted(paths, key=Path.as_posix)) + + +def _git_paths(root: Path, *arguments: str) -> set[Path]: + output = _git_bytes(root, *arguments) + return {_path(value) for value in output.split(b"\0") if value} + + +def _path(value: bytes) -> Path: + try: + text = value.decode("utf-8") + except UnicodeError: + raise CandidateFailure(CandidateErrorCode.UNSAFE_PATH, "encoding") from None + pure = PurePosixPath(text) + if pure.is_absolute() or pure == PurePosixPath(".") or ".." in pure.parts: + raise CandidateFailure(CandidateErrorCode.UNSAFE_PATH, text) + return Path(*pure.parts) + + +def _validate_changes( + root: Path, + changed: tuple[Path, ...], + policy: ExperimentPolicySnapshot, + hypothesis: HarnessHypothesis, +) -> None: + targets = frozenset(hypothesis.target.relative_paths) + editable = frozenset(policy.editable_paths) + for path in changed: + _validate_reserved_path(path) + if path not in editable or path not in targets: + raise CandidateFailure(CandidateErrorCode.OUT_OF_SCOPE, path.as_posix()) + _require_regular_path(root, path) + + +def _validate_reserved_path(path: Path) -> None: + text = path.as_posix() + if text in _MANAGED_PATHS or path.parts[0] == ".workspace": + raise CandidateFailure(CandidateErrorCode.MANAGED_PATH, text) + if any(_credential_name(part) for part in path.parts): + raise CandidateFailure(CandidateErrorCode.CREDENTIAL_PATH, text) + + +def _credential_name(name: str) -> bool: + lowered = name.lower() + return lowered == ".env" or lowered.startswith(".env.") or lowered in _CREDENTIAL_NAMES + + +def _require_regular_path(root: Path, relative: Path) -> None: + try: + _require_regular_parents(root, relative) + path = root / relative + metadata = path.lstat() + path.resolve(strict=True).relative_to(root.resolve(strict=True)) + except (OSError, ValueError): + raise CandidateFailure(CandidateErrorCode.UNSAFE_PATH, relative.as_posix()) from None + if path.is_symlink() or not stat.S_ISREG(metadata.st_mode): + raise CandidateFailure(CandidateErrorCode.UNSAFE_PATH, relative.as_posix()) + + +def _require_regular_parents(root: Path, relative: Path) -> None: + current = root + for part in relative.parts[:-1]: + current /= part + metadata = current.lstat() + if current.is_symlink() or not stat.S_ISDIR(metadata.st_mode): + raise OSError + + +def _directory(path: Path, subject: str) -> Path: + try: + resolved = path.resolve(strict=True) + except (OSError, RuntimeError): + raise CandidateFailure(CandidateErrorCode.INVALID_WORKSPACE, subject) from None + if not resolved.is_dir(): + raise CandidateFailure(CandidateErrorCode.INVALID_WORKSPACE, subject) + return resolved + + +def _worktree_name( + root: Path, + policy: ExperimentPolicySnapshot, + hypothesis: HarnessHypothesis, +) -> str: + suffix = hypothesis.id.value.removeprefix("sha256:")[:12] + return f"{root.name}-ofw-{policy.experiment_id}-{suffix}" + + +def _git(root: Path, *arguments: str) -> str: + result = subprocess.run( + ("git", "-C", str(root), *arguments), + check=False, + capture_output=True, + text=True, + ) + if result.returncode != 0: + raise CandidateFailure(CandidateErrorCode.GIT_FAILED, arguments[0]) + return result.stdout.strip() + + +def _git_bytes(root: Path, *arguments: str) -> bytes: + result = subprocess.run( + ("git", "-C", str(root), *arguments), + check=False, + capture_output=True, + ) + if result.returncode != 0: + raise CandidateFailure(CandidateErrorCode.GIT_FAILED, arguments[0]) + return result.stdout + + +def _git_succeeds(root: Path, *arguments: str) -> bool: + return ( + subprocess.run( + ("git", "-C", str(root), *arguments), + check=False, + capture_output=True, + ).returncode + == 0 + ) diff --git a/src/ofw/evolution/candidate_langfuse.py b/src/ofw/evolution/candidate_langfuse.py new file mode 100644 index 0000000..616b9ca --- /dev/null +++ b/src/ofw/evolution/candidate_langfuse.py @@ -0,0 +1,51 @@ +"""Exact read-only Langfuse trace mapping for candidate trials.""" + +from __future__ import annotations + +from ofw.evolution.candidate import ( + CandidateBlockerCode, + TraceMatch, + TraceMatchRequest, +) +from ofw.observability.langfuse.contracts import TraceWindow +from ofw.observability.langfuse.domain import ObservationRecord, TraceId +from ofw.observability.langfuse.trace_query import ( + ObservationFieldGroup, + ObservationRead, + ObservationReader, +) + + +class LangfuseCandidateTraceLocator: + """Accept one trace only when the complete bounded result is unambiguous.""" + + def __init__(self, reader: ObservationReader) -> None: + self._reader = reader + + def locate(self, request: TraceMatchRequest) -> TraceMatch: + page = self._reader.read_observations( + ObservationRead( + trace_id=None, + fields=( + ObservationFieldGroup.CORE, + ObservationFieldGroup.BASIC, + ObservationFieldGroup.TRACE_CONTEXT, + ), + limit=2, + window=TraceWindow(request.started_at, request.finished_at), + session_id=request.session_id, + environment=request.environment, + release=request.release, + is_root_observation=True, + ) + ) + trace_ids = _trace_ids(page.records) + if not trace_ids: + return TraceMatch(None, CandidateBlockerCode.TRACE_NOT_FOUND) + if page.cursor is not None or len(set(trace_ids)) != 1: + return TraceMatch(None, CandidateBlockerCode.TRACE_AMBIGUOUS) + return TraceMatch(TraceId(trace_ids[0]).value, None) + + +def _trace_ids(records: tuple[ObservationRecord, ...]) -> tuple[str, ...]: + return tuple(record.trace_id.value for record in records if record.trace_id is not None) diff --git a/src/ofw/evolution/candidate_service.py b/src/ofw/evolution/candidate_service.py new file mode 100644 index 0000000..3bd0956 --- /dev/null +++ b/src/ofw/evolution/candidate_service.py @@ -0,0 +1,669 @@ +"""Re-entrant candidate worktree sealing, execution, and outcome recording.""" + +from __future__ import annotations + +import hashlib +import os +import tempfile +from collections.abc import Iterator +from contextlib import contextmanager +from dataclasses import dataclass +from datetime import UTC, datetime, timedelta +from pathlib import Path +from typing import Literal + +from pydantic import Field + +from ofw.evaluation.outcome import ( + EvidenceReference, + OutcomeEvaluation, + TaskId, + VerifierId, + VerifierVerdict, +) +from ofw.evolution.candidate import ( + CandidateBlocker, + CandidateBlockerCode, + CandidateErrorCode, + CandidateExecutionInput, + CandidateExecutionObservation, + CandidateExperimentRunner, + CandidateFailure, + CandidateHypothesisRepository, + CandidateId, + CandidateOutcomeReceipt, + CandidateOutcomeStore, + CandidatePhase, + CandidateStatus, + CandidateTraceLocator, + CandidateWorkspace, + CandidateWorkspaceGateway, + TraceMatchRequest, + candidate_policy_digest, +) +from ofw.evolution.hypothesis import HarnessHypothesis, HypothesisFailure, StrictModel +from ofw.observability.langfuse.domain import TraceId +from ofw.preparation.contracts import ( + ExperimentControls, + ExperimentRun, + ExperimentSummary, + ExperimentTrial, + PreparationErrorCode, + PreparationFailure, +) +from ofw.preparation.policy import ExperimentPolicyFailure, ExperimentPolicySnapshot + + +class _CandidateState(StrictModel): + schema_version: Literal[1] = 1 + request_digest: str = Field(pattern=r"sha256:[0-9a-f]{64}") + phase: CandidatePhase + source_commit: str = Field(pattern=r"[0-9a-f]{40}") + policy_digest: str = Field(pattern=r"sha256:[0-9a-f]{64}") + controls_digest: str = Field(pattern=r"sha256:[0-9a-f]{64}") + worktree_path: Path + candidate_id: str | None = Field(default=None, pattern=r"sha256:[0-9a-f]{64}") + candidate_tree: str | None = Field(default=None, pattern=r"[0-9a-f]{40}") + candidate_commit: str | None = Field(default=None, pattern=r"[0-9a-f]{40}") + job_path: Path | None = None + log_path: Path | None = None + process_id: int | None = Field(default=None, ge=1) + started_at: datetime | None = None + deadline_at: datetime | None = None + outcome_receipts: tuple[CandidateOutcomeReceipt, ...] = Field(max_length=500) + blockers: tuple[CandidateBlocker, ...] = Field(max_length=500) + + +@dataclass(frozen=True, slots=True) +class _OutcomeReduction: + receipts: tuple[CandidateOutcomeReceipt, ...] + blockers: tuple[CandidateBlocker, ...] + + +class CandidateExecutionService: + def __init__( + self, + *, + workspace: CandidateWorkspaceGateway, + hypotheses: CandidateHypothesisRepository, + runner: CandidateExperimentRunner, + trace_locator: CandidateTraceLocator, + outcome_store: CandidateOutcomeStore, + ) -> None: + self._workspace = workspace + self._hypotheses = hypotheses + self._runner = runner + self._trace_locator = trace_locator + self._outcome_store = outcome_store + + def execute(self, request: CandidateExecutionInput) -> CandidateExecutionObservation: + try: + return self._execute(request) + except CandidateFailure as error: + return _failure_observation(request, error) + except PreparationFailure as error: + return _failure_observation(request, _runner_failure(error)) + + def _execute(self, request: CandidateExecutionInput) -> CandidateExecutionObservation: + policy, hypothesis = self._authority(request) + control = self._workspace.control_directory( + request.workspace_root, + request.hypothesis_id, + ) + control.mkdir(parents=True, exist_ok=True) + with _candidate_lock(control): + state = _read_state(control) + if state is None: + return self._prepare(request, policy, hypothesis, control) + _validate_state(request, policy, state) + self._workspace.validate_accepted(request.workspace_root, policy, hypothesis) + if state.phase is CandidatePhase.COMPLETE: + return _complete_observation(request, state) + if state.phase is CandidatePhase.RUNNING: + return self._poll(request, policy, hypothesis, control, state) + return self._launch(request, policy, hypothesis, control, state) + + def _authority( + self, + request: CandidateExecutionInput, + ) -> tuple[ExperimentPolicySnapshot, HarnessHypothesis]: + try: + policy = self._hypotheses.load_policy(request.workspace_root, request.experiment_id) + hypothesis = self._hypotheses.load(request.workspace_root, request.hypothesis_id) + except (ExperimentPolicyFailure, HypothesisFailure): + raise CandidateFailure(CandidateErrorCode.STALE_POLICY, request.experiment_id) from None + if hypothesis.experiment_id != policy.experiment_id: + raise CandidateFailure(CandidateErrorCode.STALE_POLICY, request.hypothesis_id) + if hypothesis.source_commit != policy.initialization_commit: + raise CandidateFailure(CandidateErrorCode.STALE_COMMIT, request.hypothesis_id) + return policy, hypothesis + + def _prepare( + self, + request: CandidateExecutionInput, + policy: ExperimentPolicySnapshot, + hypothesis: HarnessHypothesis, + control: Path, + ) -> CandidateExecutionObservation: + prepared = self._workspace.prepare( + request.workspace_root, + request.worktree_parent, + policy, + hypothesis, + ) + state = _CandidateState( + request_digest=_request_digest(request), + phase=CandidatePhase.EDITING, + source_commit=prepared.source_commit, + policy_digest=candidate_policy_digest(policy), + controls_digest=policy.controls_digest, + worktree_path=prepared.worktree_path, + outcome_receipts=(), + blockers=(), + ) + _write_state(control, state) + return _editing_observation(request, state) + + def _launch( + self, + request: CandidateExecutionInput, + policy: ExperimentPolicySnapshot, + hypothesis: HarnessHypothesis, + control: Path, + state: _CandidateState, + ) -> CandidateExecutionObservation: + workspace = _git_workspace(request, state) + tree = self._workspace.inspect(workspace, policy, hypothesis) + controls = self._validated_controls(request, policy) + candidate_id = CandidateId.build( + policy_digest=state.policy_digest, + hypothesis_id=request.hypothesis_id, + source_commit=state.source_commit, + candidate_tree=tree.tree_id, + controls_digest=state.controls_digest, + ) + committed = self._workspace.commit( + workspace, + tree, + candidate_id, + request.experiment_id, + ) + run = _new_run(request, control, workspace, controls, candidate_id, committed.commit) + started_at = datetime.now(UTC) + running = _running_state( + state, + candidate_id, + tree.tree_id, + committed.commit, + run, + started_at, + None, + policy.max_baseline_seconds, + ) + _write_state(control, running) + process_id = self._runner.start(run) + launched = _running_state( + state, + candidate_id, + tree.tree_id, + committed.commit, + run, + started_at, + process_id, + policy.max_baseline_seconds, + ) + _write_state(control, launched) + return _running_observation(request, launched) + + def _poll( + self, + request: CandidateExecutionInput, + policy: ExperimentPolicySnapshot, + hypothesis: HarnessHypothesis, + control: Path, + state: _CandidateState, + ) -> CandidateExecutionObservation: + del hypothesis + controls = self._validated_controls(request, policy) + run = _run_from_state(request, state, controls) + summary = self._runner.summarize(run) + if summary is None: + _require_before_deadline(state) + return _running_observation(request, state) + reduction = _record_outcomes( + summary, + run, + controls, + self._trace_locator, + self._outcome_store, + ) + complete = _complete_state(state, reduction) + _write_state(control, complete) + return _complete_observation(request, complete) + + def _validated_controls( + self, + request: CandidateExecutionInput, + policy: ExperimentPolicySnapshot, + ) -> ExperimentControls: + actual = self._runner.validate( + request.benchmark_root, + request.harbor_executable, + request.harbor_config, + ) + if actual != _policy_controls(policy): + raise CandidateFailure(CandidateErrorCode.CONTROLS_DRIFT, policy.experiment_id) + return actual + + +def _record_outcomes( + summary: ExperimentSummary, + run: ExperimentRun, + controls: ExperimentControls, + trace_locator: CandidateTraceLocator, + outcome_store: CandidateOutcomeStore, +) -> _OutcomeReduction: + receipts: list[CandidateOutcomeReceipt] = [] + blockers: list[CandidateBlocker] = [] + for trial in summary.trials: + result = _authoritative_result(trial) + if isinstance(result, CandidateBlocker): + blockers.append(result) + continue + match = trace_locator.locate(_trace_request(trial, run, controls)) + if match.trace_id is None: + blockers.append(_trace_blocker(trial, match.blocker)) + continue + outcome = _outcome(trial, controls, match.trace_id, result) + try: + submission = outcome_store.store(outcome) + except Exception: + raise CandidateFailure( + CandidateErrorCode.OUTCOME_STORE_FAILED, + trial.task_id, + ) from None + receipts.append( + CandidateOutcomeReceipt( + task_id=trial.task_id, + trace_id=match.trace_id, + score_id=submission.score_id.value, + verdict=result[0], + ) + ) + return _OutcomeReduction(tuple(receipts), tuple(blockers)) + + +def _authoritative_result( + trial: ExperimentTrial, +) -> tuple[VerifierVerdict, float | None] | CandidateBlocker: + if trial.exception: + return _blocker(trial, CandidateBlockerCode.UNVERIFIED, "agent_exception") + reward = _reward_result(trial) + if reward is not None: + return reward + return _verdict_result(trial) + + +def _reward_result( + trial: ExperimentTrial, +) -> tuple[VerifierVerdict, float] | CandidateBlocker | None: + if trial.reward == 1.0: + return VerifierVerdict.PASS, 1.0 + if trial.reward == 0.0: + return VerifierVerdict.FAIL, 0.0 + if trial.reward is not None: + return _blocker(trial, CandidateBlockerCode.UNSUPPORTED_REWARD, str(trial.reward)) + return None + + +def _verdict_result( + trial: ExperimentTrial, +) -> tuple[VerifierVerdict, None] | CandidateBlocker: + if trial.verdict in (VerifierVerdict.ABSTAIN.value, VerifierVerdict.ERROR.value): + return VerifierVerdict(trial.verdict), None + return _blocker(trial, CandidateBlockerCode.UNVERIFIED, "missing_verifier_result") + + +def _trace_request( + trial: ExperimentTrial, + run: ExperimentRun, + controls: ExperimentControls, +) -> TraceMatchRequest: + return TraceMatchRequest( + task_id=trial.task_id, + session_id=run.session_id, + environment=controls.environment, + release=run.release, + started_at=trial.started_at, + finished_at=trial.finished_at, + ) + + +def _trace_blocker( + trial: ExperimentTrial, + code: CandidateBlockerCode | None, +) -> CandidateBlocker: + if code is None: + raise CandidateFailure(CandidateErrorCode.INVALID_RESULT, trial.task_id) + return _blocker(trial, code, "trace_mapping") + + +def _blocker( + trial: ExperimentTrial, + code: CandidateBlockerCode, + subject: str, +) -> CandidateBlocker: + return CandidateBlocker(task_id=trial.task_id, code=code, subject=subject) + + +def _outcome( + trial: ExperimentTrial, + controls: ExperimentControls, + trace_id: str, + result: tuple[VerifierVerdict, float | None], +) -> OutcomeEvaluation: + verdict, score = result + return OutcomeEvaluation( + trace_id=TraceId(trace_id), + task_id=TaskId(trial.task_id), + verifier_id=VerifierId(f"{controls.verifier}@{trial.task_checksum}"), + evaluated_at=trial.evaluated_at, + verdict=verdict, + score=score, + evidence=tuple(EvidenceReference(value) for value in trial.evidence), + ) + + +def _policy_controls(policy: ExperimentPolicySnapshot) -> ExperimentControls: + return ExperimentControls( + model=policy.model, + task_ids=policy.task_ids, + benchmark_config_digest=policy.benchmark_config_digest, + verifier=policy.verifier, + environment=policy.environment, + concurrency=policy.concurrency, + max_retries=policy.max_retries, + ) + + +def _new_run( + request: CandidateExecutionInput, + control: Path, + workspace: CandidateWorkspace, + controls: ExperimentControls, + candidate_id: CandidateId, + candidate_commit: str, +) -> ExperimentRun: + run_id = f"ofw-candidate-{candidate_id.value.removeprefix('sha256:')[:24]}" + return ExperimentRun( + run_id=run_id, + benchmark_root=request.benchmark_root, + harbor_executable=request.harbor_executable, + harbor_config=request.benchmark_root / request.harbor_config, + job_path=request.benchmark_root / "jobs" / run_id, + log_path=control / "candidate.log", + source_root=workspace.worktree_path, + release=candidate_commit, + session_id=candidate_id.value, + controls=controls, + ) + + +def _run_from_state( + request: CandidateExecutionInput, + state: _CandidateState, + controls: ExperimentControls, +) -> ExperimentRun: + candidate_id = state.candidate_id + candidate_commit = state.candidate_commit + job_path = state.job_path + log_path = state.log_path + if candidate_id is None or candidate_commit is None or job_path is None or log_path is None: + raise CandidateFailure(CandidateErrorCode.INVALID_RESULT, request.hypothesis_id) + return ExperimentRun( + run_id=job_path.name, + benchmark_root=request.benchmark_root, + harbor_executable=request.harbor_executable, + harbor_config=request.benchmark_root / request.harbor_config, + job_path=job_path, + log_path=log_path, + source_root=state.worktree_path, + release=candidate_commit, + session_id=candidate_id, + controls=controls, + ) + + +def _git_workspace( + request: CandidateExecutionInput, + state: _CandidateState, +) -> CandidateWorkspace: + return CandidateWorkspace( + accepted_root=request.workspace_root.resolve(), + worktree_path=state.worktree_path, + source_commit=state.source_commit, + ) + + +def _running_state( + previous: _CandidateState, + candidate_id: CandidateId, + tree: str, + commit: str, + run: ExperimentRun, + started_at: datetime, + process_id: int | None, + timeout_seconds: int, +) -> _CandidateState: + return _CandidateState( + request_digest=previous.request_digest, + phase=CandidatePhase.RUNNING, + source_commit=previous.source_commit, + policy_digest=previous.policy_digest, + controls_digest=previous.controls_digest, + worktree_path=previous.worktree_path, + candidate_id=candidate_id.value, + candidate_tree=tree, + candidate_commit=commit, + job_path=run.job_path, + log_path=run.log_path, + process_id=process_id, + started_at=started_at, + deadline_at=started_at + timedelta(seconds=timeout_seconds), + outcome_receipts=(), + blockers=(), + ) + + +def _complete_state(state: _CandidateState, reduction: _OutcomeReduction) -> _CandidateState: + return _CandidateState( + request_digest=state.request_digest, + phase=CandidatePhase.COMPLETE, + source_commit=state.source_commit, + policy_digest=state.policy_digest, + controls_digest=state.controls_digest, + worktree_path=state.worktree_path, + candidate_id=state.candidate_id, + candidate_tree=state.candidate_tree, + candidate_commit=state.candidate_commit, + job_path=state.job_path, + log_path=state.log_path, + process_id=state.process_id, + started_at=state.started_at, + deadline_at=state.deadline_at, + outcome_receipts=reduction.receipts, + blockers=reduction.blockers, + ) + + +def _validate_state( + request: CandidateExecutionInput, + policy: ExperimentPolicySnapshot, + state: _CandidateState, +) -> None: + if state.request_digest != _request_digest(request): + raise CandidateFailure(CandidateErrorCode.REQUEST_CONFLICT, request.hypothesis_id) + if state.policy_digest != candidate_policy_digest(policy): + raise CandidateFailure(CandidateErrorCode.STALE_POLICY, request.experiment_id) + if state.controls_digest != policy.controls_digest: + raise CandidateFailure(CandidateErrorCode.CONTROLS_DRIFT, request.experiment_id) + + +def _require_before_deadline(state: _CandidateState) -> None: + if state.deadline_at is not None and datetime.now(UTC) > state.deadline_at: + raise CandidateFailure(CandidateErrorCode.INVALID_RESULT, "candidate_timeout") + + +def _editing_observation( + request: CandidateExecutionInput, + state: _CandidateState, +) -> CandidateExecutionObservation: + return CandidateExecutionObservation( + status=CandidateStatus.WARNING, + summary="The isolated candidate worktree is ready for the hypothesis edit.", + next_actions=( + "Edit only the exact hypothesis targets, then call execute_candidate again.", + ), + artifacts=(str(state.worktree_path),), + phase=CandidatePhase.EDITING, + experiment_id=request.experiment_id, + hypothesis_id=request.hypothesis_id, + source_commit=state.source_commit, + worktree_path=state.worktree_path, + outcome_receipts=(), + blockers=(), + ) + + +def _running_observation( + request: CandidateExecutionInput, + state: _CandidateState, +) -> CandidateExecutionObservation: + return CandidateExecutionObservation( + status=CandidateStatus.WARNING, + summary="The deterministic candidate Harbor run is still running.", + next_actions=("Poll execute_candidate with the identical request.",), + artifacts=_artifacts(state), + phase=CandidatePhase.RUNNING, + experiment_id=request.experiment_id, + hypothesis_id=request.hypothesis_id, + source_commit=state.source_commit, + candidate_id=state.candidate_id, + candidate_tree=state.candidate_tree, + candidate_commit=state.candidate_commit, + worktree_path=state.worktree_path, + job_path=state.job_path, + session_id=state.candidate_id, + outcome_receipts=(), + blockers=(), + next_poll_after_seconds=30, + ) + + +def _complete_observation( + request: CandidateExecutionInput, + state: _CandidateState, +) -> CandidateExecutionObservation: + passes = sum(item.verdict is VerifierVerdict.PASS for item in state.outcome_receipts) + failures = sum(item.verdict is VerifierVerdict.FAIL for item in state.outcome_receipts) + terminal = len(state.outcome_receipts) + len(state.blockers) + return CandidateExecutionObservation( + status=CandidateStatus.SUCCESS, + summary="The candidate run is complete with authoritative outcome receipts.", + next_actions=("Retain the candidate and outcome receipts for the admission gate.",), + artifacts=_artifacts(state), + phase=CandidatePhase.COMPLETE, + experiment_id=request.experiment_id, + hypothesis_id=request.hypothesis_id, + source_commit=state.source_commit, + candidate_id=state.candidate_id, + candidate_tree=state.candidate_tree, + candidate_commit=state.candidate_commit, + worktree_path=state.worktree_path, + job_path=state.job_path, + session_id=state.candidate_id, + terminal_trials=terminal, + verifier_passes=passes, + verifier_failures=failures, + unverified_trials=len(state.blockers), + outcome_receipts=state.outcome_receipts, + blockers=state.blockers, + ) + + +def _failure_observation( + request: CandidateExecutionInput, + error: CandidateFailure, +) -> CandidateExecutionObservation: + return CandidateExecutionObservation( + status=CandidateStatus.ERROR, + summary=f"Candidate execution stopped: {error.code.value}.", + next_actions=("Correct the typed boundary failure without forcing Git state.",), + artifacts=(), + phase=CandidatePhase.FAILED, + experiment_id=request.experiment_id, + hypothesis_id=request.hypothesis_id, + outcome_receipts=(), + blockers=(), + error_code=error.code, + ) + + +def _artifacts(state: _CandidateState) -> tuple[str, ...]: + values = ( + state.candidate_id, + state.candidate_commit, + None if state.job_path is None else str(state.job_path), + ) + return tuple(value for value in values if value is not None) + + +def _request_digest(request: CandidateExecutionInput) -> str: + digest = hashlib.sha256(request.model_dump_json().encode("utf-8")).hexdigest() + return f"sha256:{digest}" + + +@contextmanager +def _candidate_lock(control: Path) -> Iterator[None]: + lock = control / ".lock" + try: + lock.mkdir() + except FileExistsError: + raise CandidateFailure(CandidateErrorCode.REQUEST_CONFLICT, control.name) from None + try: + yield + finally: + lock.rmdir() + + +def _read_state(control: Path) -> _CandidateState | None: + path = control / "state.json" + if not path.exists(): + return None + try: + return _CandidateState.model_validate_json(path.read_text(encoding="utf-8")) + except (OSError, ValueError): + raise CandidateFailure(CandidateErrorCode.INVALID_RESULT, "state.json") from None + + +def _write_state(control: Path, state: _CandidateState) -> None: + path = control / "state.json" + with tempfile.NamedTemporaryFile( + mode="w", + encoding="utf-8", + dir=control, + delete=False, + ) as temporary: + temporary.write(state.model_dump_json(indent=2) + "\n") + temporary_path = Path(temporary.name) + os.replace(temporary_path, path) + + +def _runner_failure(error: PreparationFailure) -> CandidateFailure: + if error.code is PreparationErrorCode.MISSING_ENVIRONMENT: + code = CandidateErrorCode.MISSING_ENVIRONMENT + elif error.code is PreparationErrorCode.LAUNCH_FAILED: + code = CandidateErrorCode.LAUNCH_FAILED + else: + code = CandidateErrorCode.INVALID_RESULT + return CandidateFailure(code, error.subject) diff --git a/src/ofw/evolution/hypothesis.py b/src/ofw/evolution/hypothesis.py index bd1338a..4b14c28 100644 --- a/src/ofw/evolution/hypothesis.py +++ b/src/ofw/evolution/hypothesis.py @@ -223,6 +223,28 @@ def from_hypothesis(cls, hypothesis: HarnessHypothesis) -> HypothesisArtifact: regression_risks=hypothesis.regression_risks, ) + def to_hypothesis(self) -> HarnessHypothesis: + return HarnessHypothesis( + id=HypothesisId(self.hypothesis_id), + experiment_id=self.experiment_id, + source_commit=self.source_commit, + patterns=tuple( + FailurePatternReference( + pattern.pattern_id, + pattern.diagnosis_artifact_ids, + ) + for pattern in self.patterns + ), + statement=self.statement, + rationale=self.rationale, + target=HarnessChangeTarget( + self.target.component_kind, + self.target.relative_paths, + ), + expected_effect=self.expected_effect, + regression_risks=self.regression_risks, + ) + class HypothesisStatus(StrEnum): SUCCESS = "success" @@ -428,9 +450,7 @@ def _mine_patterns( ) -> FailurePatternMiningObservation: artifact_ids = tuple( sorted( - artifact_id - for pattern in patterns - for artifact_id in pattern.diagnosis_artifact_ids + artifact_id for pattern in patterns for artifact_id in pattern.diagnosis_artifact_ids ) ) try: diff --git a/src/ofw/evolution/hypothesis_repository.py b/src/ofw/evolution/hypothesis_repository.py index 8644022..1322adc 100644 --- a/src/ofw/evolution/hypothesis_repository.py +++ b/src/ofw/evolution/hypothesis_repository.py @@ -62,6 +62,27 @@ def load_curation(self, root: Path, curation_id: str) -> FailureCurationArtifact raise HypothesisFailure(HypothesisErrorCode.CURATION_INVALID, curation_id) return artifact + def load(self, root: Path, hypothesis_id: str) -> HarnessHypothesis: + prepared_root = _prepared_root(root) + try: + with open_directory_chain( + prepared_root, + (".workspace", "hypotheses"), + create=False, + ) as directory: + content = read_bounded( + directory, + f"{hypothesis_id}.json", + maximum_bytes=_HYPOTHESIS_LIMIT_BYTES, + subject=hypothesis_id, + ) + artifact = HypothesisArtifact.model_validate_json(content) + except (FileNotFoundError, SafeFileFailure, ValidationError, ValueError, OSError): + raise HypothesisFailure(HypothesisErrorCode.STALE_POLICY, hypothesis_id) from None + if artifact.hypothesis_id != hypothesis_id: + raise HypothesisFailure(HypothesisErrorCode.STALE_POLICY, hypothesis_id) + return artifact.to_hypothesis() + def validate_workspace( self, root: Path, diff --git a/src/ofw/mcp.py b/src/ofw/mcp.py index e684793..c4a4d77 100644 --- a/src/ofw/mcp.py +++ b/src/ofw/mcp.py @@ -45,9 +45,14 @@ VerifierVerdict, ) from ofw.evolution import ( + CandidateExecutionInput, + CandidateExecutionObservation, + CandidateExecutionService, + CandidateGitGateway, FileHypothesisRepository, HypothesisObservation, HypothesisService, + LangfuseCandidateTraceLocator, RecordHypothesisInput, ) from ofw.observability.langfuse.contracts import LangfuseProject @@ -70,7 +75,7 @@ WorkspacePreparationObservation, WorkspacePreparationService, ) -from ofw.preparation.harbor import HarborBaselineRunner +from ofw.preparation.harbor import HarborBaselineRunner, HarborExperimentRunner from ofw.preparation.worktree import GitWorktreeGateway QueryInput = TypeVar("QueryInput") @@ -92,8 +97,8 @@ instructions=( "Prepare isolated ITSM harness workspaces, read bounded Langfuse trace evidence, and " "record authoritative outcomes, compact failure diagnoses, exact patterns, and " - "evidence-backed hypotheses. Never infer outcomes, mutate traces, copy trace payloads " - "into local storage, or edit candidates while recording a hypothesis." + "evidence-backed hypotheses and isolated candidates. Never infer outcomes, mutate " + "traces, copy trace payloads into local storage, or broaden candidate edit authority." ), log_level="DEBUG", ) @@ -170,6 +175,17 @@ def _hypothesis_service() -> HypothesisService: ) +def _candidate_service() -> CandidateExecutionService: + client = _client() + return CandidateExecutionService( + workspace=CandidateGitGateway(), + hypotheses=FileHypothesisRepository(), + runner=HarborExperimentRunner(), + trace_locator=LangfuseCandidateTraceLocator(client), + outcome_store=_outcome_store(), + ) + + def _program_template(name: str) -> str: content = files("ofw.preparation.templates").joinpath(name).read_bytes() if len(content) > _PROGRAM_TEMPLATE_LIMIT_BYTES: @@ -317,6 +333,14 @@ def record_hypothesis(request: RecordHypothesisInput) -> HypothesisObservation: return _hypothesis_service().record(request) +@server.tool(annotations=record_write, structured_output=True) +def execute_candidate( + request: CandidateExecutionInput, +) -> CandidateExecutionObservation: + """Create, seal, launch, or poll one exact hypothesis candidate.""" + return _candidate_service().execute(request) + + def main() -> None: """Run the OpenFlywheel MCP server over stdio.""" server.run(transport="stdio") diff --git a/src/ofw/preparation/__init__.py b/src/ofw/preparation/__init__.py index f2fd7b8..96ce3d6 100644 --- a/src/ofw/preparation/__init__.py +++ b/src/ofw/preparation/__init__.py @@ -5,6 +5,10 @@ BaselineRun, BaselineRunner, BaselineSummary, + ExperimentControls, + ExperimentRun, + ExperimentSummary, + ExperimentTrial, PreparationErrorCode, PreparationFailure, PreparationPhase, @@ -27,6 +31,10 @@ "BaselineRun", "BaselineRunner", "BaselineSummary", + "ExperimentControls", + "ExperimentRun", + "ExperimentSummary", + "ExperimentTrial", "ExperimentPolicyErrorCode", "ExperimentPolicyFailure", "ExperimentPolicySnapshot", diff --git a/src/ofw/preparation/contracts.py b/src/ofw/preparation/contracts.py index 521d0df..95e4b45 100644 --- a/src/ofw/preparation/contracts.py +++ b/src/ofw/preparation/contracts.py @@ -3,6 +3,7 @@ from __future__ import annotations from dataclasses import dataclass +from datetime import datetime from enum import StrEnum from pathlib import Path from typing import Annotated, Protocol @@ -207,6 +208,49 @@ class BaselineSummary: unsupported_reward_trials: int +@dataclass(frozen=True, slots=True) +class ExperimentControls: + model: str + task_ids: tuple[str, ...] + benchmark_config_digest: str + verifier: str + environment: str + concurrency: int + max_retries: int + + +@dataclass(frozen=True, slots=True) +class ExperimentRun: + run_id: str + benchmark_root: Path + harbor_executable: Path + harbor_config: Path + job_path: Path + log_path: Path + source_root: Path + release: str + session_id: str + controls: ExperimentControls + + +@dataclass(frozen=True, slots=True) +class ExperimentTrial: + task_id: str + task_checksum: str + exception: bool + verdict: str | None + reward: float | None + started_at: datetime + finished_at: datetime + evaluated_at: datetime + evidence: tuple[str, ...] + + +@dataclass(frozen=True, slots=True) +class ExperimentSummary: + trials: tuple[ExperimentTrial, ...] + + @dataclass(frozen=True, slots=True) class PreparedGitWorkspace: branch_name: str diff --git a/src/ofw/preparation/harbor.py b/src/ofw/preparation/harbor.py index d1d10e3..056ce41 100644 --- a/src/ofw/preparation/harbor.py +++ b/src/ofw/preparation/harbor.py @@ -6,6 +6,7 @@ import os import subprocess # nosec B404 from dataclasses import dataclass +from datetime import datetime from pathlib import Path from pydantic import BaseModel, ConfigDict, Field, JsonValue, ValidationError @@ -14,6 +15,10 @@ BaselineConfiguration, BaselineRun, BaselineSummary, + ExperimentControls, + ExperimentRun, + ExperimentSummary, + ExperimentTrial, PreparationErrorCode, PreparationFailure, PrepareWorkspaceInput, @@ -58,8 +63,21 @@ class _HarborVerifierResultWire(_WireModel): verdict: str | None = None +class _HarborExecutionWire(_WireModel): + started_at: datetime + finished_at: datetime + + +class _HarborVerifierWire(_WireModel): + finished_at: datetime + + class _HarborTrialResultWire(_WireModel): + task_name: str | None = None + task_checksum: str | None = None exception_info: JsonValue | None = None + agent_execution: _HarborExecutionWire | None = None + verifier: _HarborVerifierWire | None = None verifier_result: _HarborVerifierResultWire | None = None @@ -72,18 +90,20 @@ class _Credentials: langfuse_base_url: str -class HarborBaselineRunner: - """Launch one sequential ITSM Harbor job and parse its bounded results.""" +class HarborExperimentRunner: + """Validate, launch, and normalize one deterministic Harbor experiment.""" - def validate(self, request: PrepareWorkspaceInput) -> BaselineConfiguration: - _executable(request.harbor_executable) - config_path = _contained(request.benchmark_root, request.harbor_config) + def validate( + self, + benchmark_root: Path, + harbor_executable: Path, + harbor_config: Path, + *, + require_credentials: bool = True, + ) -> ExperimentControls: + _executable(harbor_executable) + config_path = _contained(benchmark_root, harbor_config) config, config_content = _parse_config(config_path) - if len(config.tasks) != request.expected_task_count: - raise PreparationFailure( - PreparationErrorCode.TASK_COUNT_MISMATCH, - str(len(config.tasks)), - ) agent = config.agents[0] if agent.name != _AGENT_NAME: raise PreparationFailure( @@ -96,9 +116,10 @@ def validate(self, request: PrepareWorkspaceInput) -> BaselineConfiguration: PreparationErrorCode.INVALID_HARBOR_CONFIG, "tasks", ) - _validate_source_adapter(request.benchmark_root) - _credentials() - return BaselineConfiguration( + _validate_source_adapter(benchmark_root) + if require_credentials: + _credentials() + return ExperimentControls( model=agent.model_name, task_ids=task_ids, benchmark_config_digest=( @@ -106,24 +127,27 @@ def validate(self, request: PrepareWorkspaceInput) -> BaselineConfiguration: ), verifier="itsm-bench", environment="itsm-bench", + concurrency=1, + max_retries=0, ) - def start(self, run: BaselineRun) -> int: + def start(self, run: ExperimentRun) -> int: if run.job_path.exists(): raise PreparationFailure(PreparationErrorCode.LAUNCH_FAILED, "job_path") + _require_run_controls(self, run) command = ( str(_executable(run.harbor_executable)), "run", "--config", str(run.harbor_config), "--job-name", - run.experiment_id, + run.run_id, "--jobs-dir", str(run.job_path.parent), "--n-concurrent", - "1", + str(run.controls.concurrency), "--max-retries", - "0", + str(run.controls.max_retries), "--yes", ) run.log_path.parent.mkdir(parents=True, exist_ok=True) @@ -145,6 +169,67 @@ def start(self, run: BaselineRun) -> int: ) from error return process.pid + def summarize(self, run: ExperimentRun) -> ExperimentSummary | None: + root = _finished_job_result(run.job_path) + if root is None: + return None + trials = _experiment_trials(run) + if len(trials) > root.n_total_trials: + raise PreparationFailure( + PreparationErrorCode.INVALID_BASELINE_RESULT, + "trial count", + ) + if len(trials) != root.n_total_trials: + raise PreparationFailure( + PreparationErrorCode.INVALID_BASELINE_RESULT, + "terminal trial count", + ) + if _trial_ids(trials) != run.controls.task_ids: + raise PreparationFailure( + PreparationErrorCode.INVALID_BASELINE_RESULT, + "task ids", + ) + return ExperimentSummary(trials) + + +def _trial_ids(trials: tuple[ExperimentTrial, ...]) -> tuple[str, ...]: + return tuple(trial.task_id for trial in trials) + + +class HarborBaselineRunner: + """Compatibility adapter preserving baseline preparation behavior.""" + + def __init__(self) -> None: + self._runner = HarborExperimentRunner() + self._controls: ExperimentControls | None = None + + def validate(self, request: PrepareWorkspaceInput) -> BaselineConfiguration: + controls = self._runner.validate( + request.benchmark_root, + request.harbor_executable, + request.harbor_config, + require_credentials=False, + ) + if len(controls.task_ids) != request.expected_task_count: + raise PreparationFailure( + PreparationErrorCode.TASK_COUNT_MISMATCH, + str(len(controls.task_ids)), + ) + _credentials() + self._controls = controls + return BaselineConfiguration( + model=controls.model, + task_ids=controls.task_ids, + benchmark_config_digest=controls.benchmark_config_digest, + verifier=controls.verifier, + environment=controls.environment, + ) + + def start(self, run: BaselineRun) -> int: + if self._controls is None: + raise PreparationFailure(PreparationErrorCode.LAUNCH_FAILED, "controls") + return self._runner.start(_baseline_experiment_run(run, self._controls)) + def summarize(self, run: BaselineRun) -> BaselineSummary | None: root = _finished_job_result(run.job_path) if root is None: @@ -270,6 +355,94 @@ def _trial_results(job_path: Path) -> tuple[_HarborTrialResultWire, ...]: return tuple(_parse_trial_result(path) for path in paths if path.exists()) +def _experiment_trials(run: ExperimentRun) -> tuple[ExperimentTrial, ...]: + trials: list[ExperimentTrial] = [] + for directory in sorted( + (child for child in run.job_path.iterdir() if child.is_dir()), + key=_path_name, + ): + result_path = directory / "result.json" + if result_path.exists(): + trials.append(_experiment_trial(run, directory.name, _parse_trial_result(result_path))) + return _ordered_trials(tuple(trials), run.controls.task_ids) + + +def _path_name(path: Path) -> str: + return path.name + + +def _ordered_trials( + trials: tuple[ExperimentTrial, ...], + task_ids: tuple[str, ...], +) -> tuple[ExperimentTrial, ...]: + ordered: list[ExperimentTrial] = [] + for task_id in task_ids: + matching = _matching_trials(trials, task_id) + if len(matching) != 1: + raise PreparationFailure(PreparationErrorCode.INVALID_BASELINE_RESULT, "task ids") + ordered.append(matching[0]) + if len(ordered) != len(trials): + raise PreparationFailure(PreparationErrorCode.INVALID_BASELINE_RESULT, "task ids") + return tuple(ordered) + + +def _matching_trials( + trials: tuple[ExperimentTrial, ...], + task_id: str, +) -> tuple[ExperimentTrial, ...]: + return tuple(trial for trial in trials if trial.task_id == task_id) + + +def _experiment_trial( + run: ExperimentRun, + directory_name: str, + wire: _HarborTrialResultWire, +) -> ExperimentTrial: + task_name, task_checksum, execution, verifier_wire = _required_trial_fields( + wire, + directory_name, + ) + verifier = wire.verifier_result + reward = None if verifier is None or verifier.rewards is None else verifier.rewards.reward + verdict = None if verifier is None else verifier.verdict + return ExperimentTrial( + task_id=task_name, + task_checksum=task_checksum, + exception=wire.exception_info is not None, + verdict=verdict, + reward=reward, + started_at=execution.started_at, + finished_at=execution.finished_at, + evaluated_at=verifier_wire.finished_at, + evidence=( + f"harbor://{run.run_id}/{directory_name}/result.json", + f"harbor://{run.run_id}/{directory_name}/verifier", + ), + ) + + +def _required_trial_fields( + wire: _HarborTrialResultWire, + directory_name: str, +) -> tuple[str, str, _HarborExecutionWire, _HarborVerifierWire]: + if wire.task_name is None: + raise _invalid_trial(directory_name) + if wire.task_checksum is None: + raise _invalid_trial(directory_name) + if wire.agent_execution is None: + raise _invalid_trial(directory_name) + if wire.verifier is None: + raise _invalid_trial(directory_name) + return wire.task_name, wire.task_checksum, wire.agent_execution, wire.verifier + + +def _invalid_trial(directory_name: str) -> PreparationFailure: + return PreparationFailure( + PreparationErrorCode.INVALID_BASELINE_RESULT, + directory_name, + ) + + def _result_sort_key(path: Path) -> str: return path.parent.name @@ -323,7 +496,7 @@ def _required_any(primary: str, fallback: str) -> str: return value.strip() -def _process_environment(run: BaselineRun) -> dict[str, str]: +def _process_environment(run: ExperimentRun) -> dict[str, str]: credentials = _credentials() environment = dict(os.environ) environment.update( @@ -333,16 +506,50 @@ def _process_environment(run: BaselineRun) -> dict[str, str]: "HERMES_LANGFUSE_PUBLIC_KEY": credentials.langfuse_public_key, "HERMES_LANGFUSE_SECRET_KEY": credentials.langfuse_secret_key, "HERMES_LANGFUSE_BASE_URL": credentials.langfuse_base_url, - "HERMES_LANGFUSE_ENV": "itsm-bench", - "HERMES_LANGFUSE_RELEASE": run.initialization_commit, - "HERMES_LANGFUSE_SESSION_ID": run.experiment_id, - _SOURCE_ENVIRONMENT_NAME: str(run.worktree_path), + "HERMES_LANGFUSE_ENV": run.controls.environment, + "HERMES_LANGFUSE_RELEASE": run.release, + "HERMES_LANGFUSE_SESSION_ID": run.session_id, + _SOURCE_ENVIRONMENT_NAME: str(run.source_root), "PYTHONPATH": _python_path(run.benchmark_root, environment.get("PYTHONPATH")), } ) return environment +def _require_run_controls(runner: HarborExperimentRunner, run: ExperimentRun) -> None: + try: + relative_config = run.harbor_config.resolve(strict=True).relative_to( + run.benchmark_root.resolve(strict=True) + ) + except (OSError, ValueError): + raise PreparationFailure(PreparationErrorCode.INVALID_HARBOR_CONFIG, "controls") from None + actual = runner.validate( + run.benchmark_root, + run.harbor_executable, + relative_config, + ) + if actual != run.controls: + raise PreparationFailure(PreparationErrorCode.INVALID_HARBOR_CONFIG, "controls") + + +def _baseline_experiment_run( + run: BaselineRun, + controls: ExperimentControls, +) -> ExperimentRun: + return ExperimentRun( + run_id=run.experiment_id, + benchmark_root=run.benchmark_root, + harbor_executable=run.harbor_executable, + harbor_config=run.harbor_config, + job_path=run.job_path, + log_path=run.log_path, + source_root=run.worktree_path, + release=run.initialization_commit, + session_id=run.experiment_id, + controls=controls, + ) + + def _python_path(root: Path, existing: str | None) -> str: if existing is None or not existing: return str(root) diff --git a/src/ofw/preparation/templates/base.md b/src/ofw/preparation/templates/base.md index 1fd3954..c440f02 100644 --- a/src/ofw/preparation/templates/base.md +++ b/src/ofw/preparation/templates/base.md @@ -4,8 +4,8 @@ This file is generated by `prepare_workspace`. Do not edit it directly. ## Mission -Record one evidence-backed hypothesis for the connected agent harness under the canonical -experiment policy, then stop before candidate editing. +Record one evidence-backed hypothesis, execute its isolated candidate under the canonical +experiment policy, then stop before admission. The baseline has already been recorded. Begin at step 2; do not rerun the unchanged baseline. Its provenance is recorded in the policy (`baseline_reused` is explicit when an @@ -27,9 +27,9 @@ existing terminal Harbor job was adopted). ## Editable and frozen surfaces -Target only exact paths allowed by the canonical experiment policy. Do not edit them in this -program. Never target the benchmark, held-out tasks, verifier, model, reasoning budget, -observability identity, or this program. +Target only exact paths allowed by the canonical experiment policy. Edit them only in the +candidate worktree returned by `execute_candidate`. Never target the benchmark, held-out tasks, +verifier, model, reasoning budget, observability identity, or this program. Keep one focused hypothesis per iteration. Do not mix prompt, tool, middleware, and control flow changes unless the evidence requires the combination. @@ -51,10 +51,20 @@ between materially different changes. Use `$hypothesis-former` with one curation receipt and group ID plus every exact supported pattern and diagnosis receipt ID in that group, explicit predicted task IDs, and at-risk task IDs, then call -`record_hypothesis`. Retain the stable hypothesis receipt and stop before candidate editing. -Candidate editing requires a later package and must not begin in this prepared program. +`record_hypothesis`. Retain the stable hypothesis receipt before candidate execution. + +### 4. Execute one candidate + +Call `execute_candidate` with the prepared workspace, experiment and hypothesis receipts, sibling +candidate-worktree parent, and Harbor runtime locations. The first call creates the isolated +worktree from the accepted experiment commit. Edit only the exact hypothesis targets in the +returned candidate worktree, then call `execute_candidate` again with the identical request. + +Poll identical requests while the candidate is running. Retain its candidate ID, Git commit, +trace-mapping blockers, and authoritative outcome receipts. Do not copy trace payloads locally, +change frozen controls, rerun an empty candidate, or edit the accepted experiment worktree. ## Package boundary -Report the hypothesis receipt and its exact evidence and target paths. Do not edit, commit, -run, gate, publish, or install a candidate; those capabilities are not part of this program. +Report the hypothesis, candidate, commit, blocker, and outcome receipts. Stop before admission: +do not gate, accept, merge, publish, push, or install the candidate. diff --git a/tests/test_candidate_execution.py b/tests/test_candidate_execution.py new file mode 100644 index 0000000..4b188fa --- /dev/null +++ b/tests/test_candidate_execution.py @@ -0,0 +1,825 @@ +"""Candidate identity, Git isolation, execution, and authoritative receipts.""" + +from __future__ import annotations + +import hashlib +import subprocess +from dataclasses import replace +from datetime import UTC, datetime +from pathlib import Path + +import pytest +from pydantic import ValidationError + +from ofw.contracts import ComponentKind, Sha256Digest +from ofw.evaluation.langfuse import OutcomeScoreSubmission +from ofw.evaluation.outcome import OutcomeEvaluation +from ofw.evolution.candidate import ( + CandidateBlockerCode, + CandidateErrorCode, + CandidateExecutionInput, + CandidateFailure, + CandidateId, + CandidatePhase, + CandidateStatus, + TraceMatch, + TraceMatchRequest, + candidate_policy_digest, +) +from ofw.evolution.candidate_git import CandidateGitGateway +from ofw.evolution.candidate_langfuse import LangfuseCandidateTraceLocator +from ofw.evolution.candidate_service import CandidateExecutionService +from ofw.evolution.hypothesis import ( + FailurePatternReference, + HarnessChangeTarget, + HarnessHypothesis, + HypothesisId, +) +from ofw.evolution.hypothesis_repository import FileHypothesisRepository +from ofw.observability.langfuse.domain import ( + JsonDocument, + ObservationId, + ObservationPage, + ObservationRecord, + ObservationType, + PageCursor, + ProjectId, + ScoreId, + TraceId, +) +from ofw.observability.langfuse.trace_query import ObservationRead +from ofw.preparation.contracts import ( + BaselineConfiguration, + ExperimentControls, + ExperimentRun, + ExperimentSummary, + ExperimentTrial, + PreparationErrorCode, + PreparationFailure, + PreparedGitWorkspace, + PrepareWorkspaceInput, +) +from ofw.preparation.policy import ( + ExperimentPolicySnapshot, + FileExperimentPolicyRepository, + build_experiment_policy, +) + + +def _git(root: Path, *arguments: str, check: bool = True) -> str: + return subprocess.run( + ("git", "-C", str(root), *arguments), + check=check, + capture_output=True, + text=True, + ).stdout.strip() + + +def _authority(tmp_path: Path) -> tuple[Path, ExperimentPolicySnapshot, HarnessHypothesis]: + root = tmp_path / "accepted" + root.mkdir() + (root / "prompt.md").write_text("Original prompt.\n", encoding="utf-8") + (root / "tools.py").write_text("def tool() -> bool:\n return True\n", encoding="utf-8") + (root / ".gitignore").write_text(".env\n", encoding="utf-8") + (root / "PROGRAM.md").write_text("# Managed\n", encoding="utf-8") + (root / "experiment_config.yaml").write_text("managed: true\n", encoding="utf-8") + _git(root, "init", "-q") + _git(root, "config", "user.name", "OpenFlywheel Test") + _git(root, "config", "user.email", "ofw@example.test") + _git(root, "add", ".") + _git(root, "commit", "-qm", "accepted experiment") + _git(root, "branch", "-m", "ofw/experiment-one") + commit = _git(root, "rev-parse", "HEAD") + benchmark = tmp_path / "benchmark" + benchmark.mkdir() + executable = tmp_path / "harbor" + executable.touch(mode=0o700) + request = PrepareWorkspaceInput( + experiment_id="experiment-one", + harness_root=root, + base_ref="HEAD", + worktree_parent=tmp_path, + benchmark_root=benchmark, + harbor_executable=executable, + harbor_config=Path("config.json"), + expected_task_count=2, + editable_paths=(Path("prompt.md"), Path("tools.py")), + goal="Improve verifier-backed quality.", + quality_target=1.0, + max_iterations=3, + no_improvement_limit=2, + max_baseline_seconds=600, + ) + policy = build_experiment_policy( + request, + PreparedGitWorkspace( + branch_name="ofw/experiment-one", + worktree_path=root, + base_commit=commit, + initialization_commit=commit, + program_path=root / "PROGRAM.md", + ), + BaselineConfiguration( + model="openai/gpt-5.4-mini", + task_ids=("task-1", "task-2"), + benchmark_config_digest="sha256:" + "a" * 64, + verifier="itsm-bench", + environment="itsm-bench", + ), + ) + hypothesis = HarnessHypothesis( + id=HypothesisId("sha256:" + "b" * 64), + experiment_id=policy.experiment_id, + source_commit=commit, + patterns=( + FailurePatternReference( + "sha256:" + "c" * 64, + ("00000000-0000-0000-0000-000000000000",), + ), + ), + statement="Require a state check before finalizing.", + rationale="Supported failures share the same cause.", + target=HarnessChangeTarget(ComponentKind.PROMPT, (Path("prompt.md"),)), + expected_effect="The agent verifies completion.", + regression_risks=(), + ) + control = root / ".git/ofw/preparations/experiment-one" + control.mkdir(parents=True) + FileExperimentPolicyRepository().publish(control, policy) + FileHypothesisRepository().store(root, hypothesis) + return root, policy, hypothesis + + +class _FakeRunner: + def __init__(self, controls: ExperimentControls) -> None: + self.controls = controls + self.runs: list[ExperimentRun] = [] + self.summary: ExperimentSummary | None = None + self.failure: PreparationFailure | None = None + + def validate( + self, + benchmark_root: Path, + harbor_executable: Path, + harbor_config: Path, + ) -> ExperimentControls: + if self.failure is not None: + raise self.failure + return self.controls + + def start(self, run: ExperimentRun) -> int: + self.runs.append(run) + return 123 + + def summarize(self, run: ExperimentRun) -> ExperimentSummary | None: + return self.summary + + +class _FakeTraceLocator: + def __init__(self) -> None: + self.requests: list[TraceMatchRequest] = [] + self.blocker: CandidateBlockerCode | None = None + + def locate(self, request: TraceMatchRequest) -> TraceMatch: + self.requests.append(request) + if self.blocker is not None: + return TraceMatch(trace_id=None, blocker=self.blocker) + return TraceMatch(trace_id=f"trace-{request.task_id}", blocker=None) + + +class _FakeOutcomeStore: + def __init__(self) -> None: + self.outcomes: list[OutcomeEvaluation] = [] + self.failure: Exception | None = None + + def store(self, outcome: OutcomeEvaluation) -> OutcomeScoreSubmission: + if self.failure is not None: + raise self.failure + self.outcomes.append(outcome) + return OutcomeScoreSubmission( + ScoreId(f"score-{outcome.task_id.value}"), + outcome.trace_id, + ) + + +class _ObservationReader: + def __init__(self, page: ObservationPage) -> None: + self.page = page + self.queries: list[ObservationRead] = [] + + def read_observations(self, query: ObservationRead) -> ObservationPage: + self.queries.append(query) + return self.page + + +def _root_observation(trace_id: str, suffix: str) -> ObservationRecord: + raw = JsonDocument("{}") + return ObservationRecord( + id=ObservationId(f"observation-{suffix}"), + trace_id=TraceId(trace_id), + start_time=datetime(2026, 9, 2, 10, 1, tzinfo=UTC), + end_time=None, + project_id=ProjectId("project-1"), + parent_observation_id=None, + type=ObservationType.AGENT, + is_root=True, + name="agent", + level=None, + version=None, + environment="itsm-bench", + user_id=None, + session_id="candidate-session", + created_at=None, + updated_at=None, + metadata=None, + usage=None, + costs=None, + total_cost=None, + tags=(), + release="d" * 40, + trace_name=None, + raw=raw, + digest=Sha256Digest("sha256:" + "0" * 64), + ) + + +def _controls(policy: ExperimentPolicySnapshot) -> ExperimentControls: + return ExperimentControls( + model=policy.model, + task_ids=policy.task_ids, + benchmark_config_digest=policy.benchmark_config_digest, + verifier=policy.verifier, + environment=policy.environment, + concurrency=policy.concurrency, + max_retries=policy.max_retries, + ) + + +def _candidate_request( + tmp_path: Path, + root: Path, + hypothesis: HarnessHypothesis, +) -> CandidateExecutionInput: + benchmark = tmp_path / "benchmark" + executable = tmp_path / "harbor" + config = benchmark / "config.json" + config.write_text("{}") + return CandidateExecutionInput( + workspace_root=root, + worktree_parent=tmp_path / "candidates", + benchmark_root=benchmark, + harbor_executable=executable, + harbor_config=config.relative_to(benchmark), + experiment_id=hypothesis.experiment_id, + hypothesis_id=hypothesis.id.value, + ) + + +def test_candidate_id_binds_policy_hypothesis_source_tree_and_controls() -> None: + candidate = CandidateId.build( + policy_digest="sha256:" + "1" * 64, + hypothesis_id="sha256:" + "2" * 64, + source_commit="3" * 40, + candidate_tree="4" * 40, + controls_digest="sha256:" + "5" * 64, + ) + + expected = hashlib.sha256( + ( + '{"schema_version":1,"policy_digest":"sha256:' + + "1" * 64 + + '","hypothesis_id":"sha256:' + + "2" * 64 + + '","source_commit":"' + + "3" * 40 + + '","candidate_tree":"' + + "4" * 40 + + '","controls_digest":"sha256:' + + "5" * 64 + + '"}' + ).encode("utf-8") + ).hexdigest() + + assert candidate.value == f"sha256:{expected}" + assert str(candidate) == candidate.value + + +def test_candidate_id_changes_when_any_authority_input_changes() -> None: + values = { + "policy_digest": "sha256:" + "1" * 64, + "hypothesis_id": "sha256:" + "2" * 64, + "source_commit": "3" * 40, + "candidate_tree": "4" * 40, + "controls_digest": "sha256:" + "5" * 64, + } + baseline = CandidateId.build(**values) + + for field, replacement in ( + ("policy_digest", "sha256:" + "a" * 64), + ("hypothesis_id", "sha256:" + "b" * 64), + ("source_commit", "c" * 40), + ("candidate_tree", "d" * 40), + ("controls_digest", "sha256:" + "e" * 64), + ): + changed = dict(values) + changed[field] = replacement + assert CandidateId.build(**changed) != baseline + + +def test_candidate_input_rejects_extra_relative_and_escaped_paths(tmp_path: Path) -> None: + root, _, hypothesis = _authority(tmp_path) + request = _candidate_request(tmp_path, root, hypothesis) + payload = request.model_dump_json() + + with pytest.raises(ValidationError): + CandidateExecutionInput.model_validate_json(payload[:-1] + ',"unexpected":true}') + with pytest.raises(ValidationError): + CandidateExecutionInput.model_validate_json( + payload.replace(str(request.workspace_root), "relative-workspace") + ) + with pytest.raises(ValidationError): + CandidateExecutionInput.model_validate_json( + payload.replace('"harbor_config":"config.json"', '"harbor_config":"../config.json"') + ) + + +def test_candidate_git_gateway_isolates_validates_and_commits_one_tree(tmp_path: Path) -> None: + root, policy, hypothesis = _authority(tmp_path) + worktree_parent = tmp_path / "candidates" + worktree_parent.mkdir() + gateway = CandidateGitGateway() + workspace = gateway.prepare(root, worktree_parent, policy, hypothesis) + + (workspace.worktree_path / "prompt.md").write_text("Check state before finalizing.\n") + tree = gateway.inspect(workspace, policy, hypothesis) + candidate_id = CandidateId.build( + policy_digest=candidate_policy_digest(policy), + hypothesis_id=hypothesis.id.value, + source_commit=workspace.source_commit, + candidate_tree=tree.tree_id, + controls_digest=policy.controls_digest, + ) + committed = gateway.commit(workspace, tree, candidate_id, policy.experiment_id) + + assert (root / "prompt.md").read_text() == "Original prompt.\n" + assert _git(workspace.worktree_path, "rev-parse", "HEAD^") == policy.initialization_commit + assert _git(workspace.worktree_path, "rev-parse", "HEAD^{tree}") == tree.tree_id + message = _git(workspace.worktree_path, "show", "-s", "--format=%B", committed.commit) + assert message.count("OFW-Experiment: experiment-one") == 1 + assert message.count(f"OFW-Run: {candidate_id.value}") == 1 + + +def test_candidate_git_gateway_rejects_an_empty_candidate(tmp_path: Path) -> None: + root, policy, hypothesis = _authority(tmp_path) + parent = tmp_path / "candidates" + parent.mkdir() + gateway = CandidateGitGateway() + workspace = gateway.prepare(root, parent, policy, hypothesis) + + with pytest.raises(CandidateFailure) as raised: + gateway.inspect(workspace, policy, hypothesis) + + assert raised.value.code is CandidateErrorCode.EMPTY_CANDIDATE + assert _git(workspace.worktree_path, "rev-parse", "HEAD") == policy.initialization_commit + + +def test_candidate_git_gateway_refuses_an_existing_candidate_worktree(tmp_path: Path) -> None: + root, policy, hypothesis = _authority(tmp_path) + parent = tmp_path / "candidates" + parent.mkdir() + gateway = CandidateGitGateway() + gateway.prepare(root, parent, policy, hypothesis) + + with pytest.raises(CandidateFailure) as raised: + gateway.prepare(root, parent, policy, hypothesis) + + assert raised.value.code is CandidateErrorCode.WORKTREE_EXISTS + + +def test_candidate_git_gateway_rechecks_the_tree_before_commit(tmp_path: Path) -> None: + root, policy, hypothesis = _authority(tmp_path) + parent = tmp_path / "candidates" + parent.mkdir() + gateway = CandidateGitGateway() + workspace = gateway.prepare(root, parent, policy, hypothesis) + target = workspace.worktree_path / "prompt.md" + target.write_text("first candidate\n") + tree = gateway.inspect(workspace, policy, hypothesis) + target.write_text("changed after sealing\n") + candidate_id = CandidateId.build( + policy_digest=candidate_policy_digest(policy), + hypothesis_id=hypothesis.id.value, + source_commit=workspace.source_commit, + candidate_tree=tree.tree_id, + controls_digest=policy.controls_digest, + ) + + with pytest.raises(CandidateFailure) as raised: + gateway.commit(workspace, tree, candidate_id, policy.experiment_id) + + assert raised.value.code is CandidateErrorCode.STALE_COMMIT + assert _git(workspace.worktree_path, "rev-parse", "HEAD") == policy.initialization_commit + + +@pytest.mark.parametrize( + ("mutation", "expected"), + ( + ("out-of-scope", CandidateErrorCode.OUT_OF_SCOPE), + ("rename", CandidateErrorCode.UNSAFE_PATH), + ("managed", CandidateErrorCode.MANAGED_PATH), + ("workspace", CandidateErrorCode.MANAGED_PATH), + ("credential", CandidateErrorCode.CREDENTIAL_PATH), + ("symlink", CandidateErrorCode.UNSAFE_PATH), + ), +) +def test_candidate_git_gateway_freezes_everything_except_exact_hypothesis_targets( + tmp_path: Path, + mutation: str, + expected: CandidateErrorCode, +) -> None: + root, policy, hypothesis = _authority(tmp_path) + parent = tmp_path / "candidates" + parent.mkdir() + gateway = CandidateGitGateway() + workspace = gateway.prepare(root, parent, policy, hypothesis) + candidate = workspace.worktree_path + if mutation == "out-of-scope": + (candidate / "tools.py").write_text("unsafe = True\n") + elif mutation == "rename": + _git(candidate, "mv", "prompt.md", "renamed.md") + elif mutation == "managed": + (candidate / "PROGRAM.md").write_text("changed\n") + elif mutation == "workspace": + (candidate / ".workspace").mkdir() + (candidate / ".workspace/state.json").write_text("{}") + elif mutation == "credential": + (candidate / ".env").write_text("TOKEN=secret\n") + else: + (candidate / "prompt.md").unlink() + (candidate / "prompt.md").symlink_to(candidate / "PROGRAM.md") + + with pytest.raises(CandidateFailure) as raised: + gateway.inspect(workspace, policy, hypothesis) + + assert raised.value.code is expected + assert _git(candidate, "rev-parse", "HEAD") == policy.initialization_commit + + +@pytest.mark.parametrize("invalid", ("missing", "file")) +def test_candidate_git_gateway_rejects_invalid_workspace_roots( + tmp_path: Path, + invalid: str, +) -> None: + root, policy, hypothesis = _authority(tmp_path) + parent = tmp_path / "missing" + if invalid == "file": + parent.write_text("not a directory\n") + + with pytest.raises(CandidateFailure) as raised: + CandidateGitGateway().prepare(root, parent, policy, hypothesis) + + assert raised.value.code is CandidateErrorCode.INVALID_WORKSPACE + + +@pytest.mark.parametrize("drift", ("accepted", "candidate")) +def test_candidate_git_gateway_rejects_stale_or_unrelated_ancestry( + tmp_path: Path, + drift: str, +) -> None: + root, policy, hypothesis = _authority(tmp_path) + parent = tmp_path / "candidates" + parent.mkdir() + gateway = CandidateGitGateway() + workspace = gateway.prepare(root, parent, policy, hypothesis) + target = root if drift == "accepted" else workspace.worktree_path + (target / "prompt.md").write_text("changed\n") + _git(target, "add", "prompt.md") + _git(target, "commit", "-qm", "unauthorized commit") + + with pytest.raises(CandidateFailure) as raised: + gateway.inspect(workspace, policy, hypothesis) + + assert raised.value.code is CandidateErrorCode.STALE_COMMIT + + +@pytest.mark.parametrize("drift", ("experiment", "source", "branch", "target")) +def test_candidate_git_gateway_rejects_policy_or_hypothesis_drift( + tmp_path: Path, + drift: str, +) -> None: + root, policy, hypothesis = _authority(tmp_path) + if drift == "experiment": + hypothesis = replace(hypothesis, experiment_id="different-experiment") + elif drift == "source": + hypothesis = replace(hypothesis, source_commit="f" * 40) + elif drift == "branch": + _git(root, "branch", "-m", "wrong-branch") + else: + hypothesis = replace( + hypothesis, + target=HarnessChangeTarget(ComponentKind.TOOL, (Path("missing.py"),)), + ) + + with pytest.raises(CandidateFailure) as raised: + CandidateGitGateway().validate_accepted(root, policy, hypothesis) + + assert raised.value.code in { + CandidateErrorCode.STALE_COMMIT, + CandidateErrorCode.STALE_POLICY, + } + + +def test_candidate_service_creates_launches_polls_and_replays_authoritative_receipts( + tmp_path: Path, +) -> None: + root, policy, hypothesis = _authority(tmp_path) + (tmp_path / "candidates").mkdir() + request = _candidate_request(tmp_path, root, hypothesis) + runner = _FakeRunner(_controls(policy)) + locator = _FakeTraceLocator() + outcomes = _FakeOutcomeStore() + service = CandidateExecutionService( + workspace=CandidateGitGateway(), + hypotheses=FileHypothesisRepository(), + runner=runner, + trace_locator=locator, + outcome_store=outcomes, + ) + + editing = service.execute(request) + assert editing.status is CandidateStatus.WARNING + assert editing.phase is CandidatePhase.EDITING + assert editing.worktree_path is not None + (editing.worktree_path / "prompt.md").write_text("Check state before finalizing.\n") + + running = service.execute(request) + assert running.phase is CandidatePhase.RUNNING + assert running.candidate_id is not None + assert running.candidate_commit is not None + assert len(runner.runs) == 1 + still_running = service.execute(request) + assert still_running.phase is CandidatePhase.RUNNING + assert still_running.candidate_id == running.candidate_id + assert len(runner.runs) == 1 + runner.summary = ExperimentSummary( + trials=( + ExperimentTrial( + task_id="task-1", + task_checksum="checksum-1", + exception=False, + verdict=None, + reward=1.0, + started_at=datetime(2026, 9, 2, 10, 1, tzinfo=UTC), + finished_at=datetime(2026, 9, 2, 10, 1, 30, tzinfo=UTC), + evaluated_at=datetime(2026, 9, 2, 10, 1, 31, tzinfo=UTC), + evidence=("harbor://candidate/task-1/result.json",), + ), + ExperimentTrial( + task_id="task-2", + task_checksum="checksum-2", + exception=False, + verdict=None, + reward=0.0, + started_at=datetime(2026, 9, 2, 10, 2, tzinfo=UTC), + finished_at=datetime(2026, 9, 2, 10, 2, 30, tzinfo=UTC), + evaluated_at=datetime(2026, 9, 2, 10, 2, 31, tzinfo=UTC), + evidence=("harbor://candidate/task-2/result.json",), + ), + ) + ) + + complete = service.execute(request) + repeated = service.execute(request) + + assert complete == repeated + assert complete.status is CandidateStatus.SUCCESS + assert complete.phase is CandidatePhase.COMPLETE + assert complete.verifier_passes == 1 + assert complete.verifier_failures == 1 + assert complete.unverified_trials == 0 + assert tuple(receipt.task_id for receipt in complete.outcome_receipts) == ( + "task-1", + "task-2", + ) + assert len(runner.runs) == 1 + assert len(locator.requests) == 2 + assert len(outcomes.outcomes) == 2 + state = next((root / ".git/ofw/candidates").glob("*/state.json")).read_text() + assert "input" not in state + assert "output" not in state + assert "test-openai-key" not in state + + +@pytest.mark.parametrize( + ("trace_ids", "cursor", "trace_id", "blocker"), + ( + (("trace-1",), None, "trace-1", None), + ((), None, None, CandidateBlockerCode.TRACE_NOT_FOUND), + (("trace-1", "trace-2"), None, None, CandidateBlockerCode.TRACE_AMBIGUOUS), + (("trace-1",), "next", None, CandidateBlockerCode.TRACE_AMBIGUOUS), + ), +) +def test_trace_locator_requires_exactly_one_complete_structural_match( + trace_ids: tuple[str, ...], + cursor: str | None, + trace_id: str | None, + blocker: CandidateBlockerCode | None, +) -> None: + page = ObservationPage( + tuple(_root_observation(value, str(index)) for index, value in enumerate(trace_ids)), + None if cursor is None else PageCursor(cursor), + ) + reader = _ObservationReader(page) + locator = LangfuseCandidateTraceLocator(reader) + request = TraceMatchRequest( + task_id="task-1", + session_id="candidate-session", + environment="itsm-bench", + release="d" * 40, + started_at=datetime(2026, 9, 2, 10, 1, tzinfo=UTC), + finished_at=datetime(2026, 9, 2, 10, 2, tzinfo=UTC), + ) + + assert locator.locate(request) == TraceMatch(trace_id=trace_id, blocker=blocker) + query = reader.queries[0] + assert query.session_id == request.session_id + assert query.environment == request.environment + assert query.release == request.release + assert query.limit == 2 + assert query.is_root_observation is True + assert query.window is not None + assert query.window.start == request.started_at + assert query.window.end == request.finished_at + assert tuple(field.value for field in query.fields) == ("core", "basic", "trace_context") + + +def test_candidate_service_rejects_controls_drift_before_commit_or_launch(tmp_path: Path) -> None: + root, policy, hypothesis = _authority(tmp_path) + (tmp_path / "candidates").mkdir() + request = _candidate_request(tmp_path, root, hypothesis) + drifted = ExperimentControls( + model="different-model", + task_ids=policy.task_ids, + benchmark_config_digest=policy.benchmark_config_digest, + verifier=policy.verifier, + environment=policy.environment, + concurrency=policy.concurrency, + max_retries=policy.max_retries, + ) + runner = _FakeRunner(drifted) + service = CandidateExecutionService( + workspace=CandidateGitGateway(), + hypotheses=FileHypothesisRepository(), + runner=runner, + trace_locator=_FakeTraceLocator(), + outcome_store=_FakeOutcomeStore(), + ) + editing = service.execute(request) + assert editing.worktree_path is not None + (editing.worktree_path / "prompt.md").write_text("changed\n") + + rejected = service.execute(request) + + assert rejected.error_code is CandidateErrorCode.CONTROLS_DRIFT + assert _git(editing.worktree_path, "rev-parse", "HEAD") == policy.initialization_commit + assert runner.runs == [] + + +def test_candidate_service_sanitizes_missing_runtime_credentials(tmp_path: Path) -> None: + root, policy, hypothesis = _authority(tmp_path) + (tmp_path / "candidates").mkdir() + request = _candidate_request(tmp_path, root, hypothesis) + runner = _FakeRunner(_controls(policy)) + runner.failure = PreparationFailure( + PreparationErrorCode.MISSING_ENVIRONMENT, + "OPENAI_API_KEY|AZURE_OPENAI_API_KEY", + ) + service = CandidateExecutionService( + workspace=CandidateGitGateway(), + hypotheses=FileHypothesisRepository(), + runner=runner, + trace_locator=_FakeTraceLocator(), + outcome_store=_FakeOutcomeStore(), + ) + editing = service.execute(request) + assert editing.worktree_path is not None + (editing.worktree_path / "prompt.md").write_text("changed\n") + + rejected = service.execute(request) + + assert rejected.error_code is CandidateErrorCode.MISSING_ENVIRONMENT + assert "secret" not in rejected.summary + + +def test_candidate_service_rejects_a_missing_hypothesis_receipt(tmp_path: Path) -> None: + root, policy, hypothesis = _authority(tmp_path) + request = _candidate_request(tmp_path, root, hypothesis) + missing = CandidateExecutionInput( + workspace_root=request.workspace_root, + worktree_parent=request.worktree_parent, + benchmark_root=request.benchmark_root, + harbor_executable=request.harbor_executable, + harbor_config=request.harbor_config, + experiment_id=request.experiment_id, + hypothesis_id="sha256:" + "f" * 64, + ) + service = CandidateExecutionService( + workspace=CandidateGitGateway(), + hypotheses=FileHypothesisRepository(), + runner=_FakeRunner(_controls(policy)), + trace_locator=_FakeTraceLocator(), + outcome_store=_FakeOutcomeStore(), + ) + + rejected = service.execute(missing) + + assert rejected.error_code is CandidateErrorCode.STALE_POLICY + + +def test_candidate_service_rejects_a_reused_hypothesis_with_different_runtime_paths( + tmp_path: Path, +) -> None: + root, policy, hypothesis = _authority(tmp_path) + (tmp_path / "candidates").mkdir() + request = _candidate_request(tmp_path, root, hypothesis) + service = CandidateExecutionService( + workspace=CandidateGitGateway(), + hypotheses=FileHypothesisRepository(), + runner=_FakeRunner(_controls(policy)), + trace_locator=_FakeTraceLocator(), + outcome_store=_FakeOutcomeStore(), + ) + service.execute(request) + conflicting = CandidateExecutionInput( + workspace_root=request.workspace_root, + worktree_parent=request.worktree_parent, + benchmark_root=request.benchmark_root, + harbor_executable=request.harbor_executable, + harbor_config=Path("different.json"), + experiment_id=request.experiment_id, + hypothesis_id=request.hypothesis_id, + ) + + rejected = service.execute(conflicting) + + assert rejected.error_code is CandidateErrorCode.REQUEST_CONFLICT + + +def test_candidate_service_keeps_unsupported_and_ambiguous_trials_unverified( + tmp_path: Path, +) -> None: + root, policy, hypothesis = _authority(tmp_path) + (tmp_path / "candidates").mkdir() + request = _candidate_request(tmp_path, root, hypothesis) + runner = _FakeRunner(_controls(policy)) + locator = _FakeTraceLocator() + locator.blocker = CandidateBlockerCode.TRACE_AMBIGUOUS + outcomes = _FakeOutcomeStore() + service = CandidateExecutionService( + workspace=CandidateGitGateway(), + hypotheses=FileHypothesisRepository(), + runner=runner, + trace_locator=locator, + outcome_store=outcomes, + ) + editing = service.execute(request) + assert editing.worktree_path is not None + (editing.worktree_path / "prompt.md").write_text("changed\n") + service.execute(request) + runner.summary = ExperimentSummary( + trials=( + ExperimentTrial( + task_id="task-1", + task_checksum="checksum-1", + exception=False, + verdict=None, + reward=0.5, + started_at=datetime(2026, 9, 2, 10, 1, tzinfo=UTC), + finished_at=datetime(2026, 9, 2, 10, 1, 30, tzinfo=UTC), + evaluated_at=datetime(2026, 9, 2, 10, 1, 31, tzinfo=UTC), + evidence=("harbor://candidate/task-1/result.json",), + ), + ExperimentTrial( + task_id="task-2", + task_checksum="checksum-2", + exception=False, + verdict=None, + reward=0.0, + started_at=datetime(2026, 9, 2, 10, 2, tzinfo=UTC), + finished_at=datetime(2026, 9, 2, 10, 2, 30, tzinfo=UTC), + evaluated_at=datetime(2026, 9, 2, 10, 2, 31, tzinfo=UTC), + evidence=("harbor://candidate/task-2/result.json",), + ), + ) + ) + + complete = service.execute(request) + + assert complete.outcome_receipts == () + assert tuple(blocker.code for blocker in complete.blockers) == ( + CandidateBlockerCode.UNSUPPORTED_REWARD, + CandidateBlockerCode.TRACE_AMBIGUOUS, + ) + assert complete.unverified_trials == 2 + assert len(locator.requests) == 1 + assert outcomes.outcomes == [] diff --git a/tests/test_harbor_experiment.py b/tests/test_harbor_experiment.py new file mode 100644 index 0000000..c5f997e --- /dev/null +++ b/tests/test_harbor_experiment.py @@ -0,0 +1,268 @@ +"""Generalized deterministic Harbor execution for candidate and baseline runs.""" + +from __future__ import annotations + +import time +from datetime import UTC, datetime +from pathlib import Path + +import pytest +from pydantic import BaseModel, ConfigDict + +from ofw.preparation.contracts import ( + BaselineRun, + ExperimentControls, + ExperimentRun, + ExperimentSummary, + PreparationErrorCode, + PreparationFailure, +) +from ofw.preparation.harbor import HarborBaselineRunner, HarborExperimentRunner + + +class _EnvironmentCapture(BaseModel): + model_config = ConfigDict(extra="forbid", frozen=True, strict=True) + + source: str + environment: str + release: str + session: str + + +def _credentials(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setenv("OPENAI_API_KEY", "test-openai-key") + monkeypatch.setenv("OPENAI_BASE_URL", "https://example.test/openai/v1") + monkeypatch.setenv("LANGFUSE_PUBLIC_KEY", "pk-lf-test") + monkeypatch.setenv("LANGFUSE_SECRET_KEY", "sk-lf-test") + monkeypatch.setenv("LANGFUSE_BASE_URL", "https://langfuse.example.test") + + +def _benchmark(tmp_path: Path) -> tuple[Path, Path, Path]: + root = tmp_path / "benchmark" + root.mkdir() + adapter = root / "agents/ofw_hermes.py" + adapter.parent.mkdir() + adapter.write_text('SOURCE = "OFW_HERMES_SOURCE"\n') + config = root / "config.json" + config.write_text( + """{ + "agents": [{ + "name": "agents.ofw_hermes:OfwHermes", + "model_name": "openai/gpt-5.4-mini" + }], + "tasks": [{"path": "task-1"}, {"path": "task-2"}] +} +""" + ) + executable = tmp_path / "harbor" + executable.write_text( + """#!/usr/bin/env python3 +import json +import os +import sys +from pathlib import Path + +args = sys.argv[1:] +name = args[args.index("--job-name") + 1] +root = Path(args[args.index("--jobs-dir") + 1]) / name +root.mkdir(parents=True) +(root / "environment.json").write_text(json.dumps({ + "source": os.environ["OFW_HERMES_SOURCE"], + "environment": os.environ["HERMES_LANGFUSE_ENV"], + "release": os.environ["HERMES_LANGFUSE_RELEASE"], + "session": os.environ["HERMES_LANGFUSE_SESSION_ID"], +})) +for index, reward in enumerate((1.0, 0.0), start=1): + trial = root / f"task-{index}__trial" + (trial / "verifier").mkdir(parents=True) + (trial / "result.json").write_text(json.dumps({ + "task_name": f"task-{index}", + "task_checksum": f"checksum-{index}", + "exception_info": None, + "agent_execution": { + "started_at": f"2026-09-02T10:0{index}:00Z", + "finished_at": f"2026-09-02T10:0{index}:30Z" + }, + "verifier": {"finished_at": f"2026-09-02T10:0{index}:31Z"}, + "verifier_result": {"rewards": {"reward": reward}} + })) +(root / "result.json").write_text(json.dumps({ + "finished_at": "2026-09-02T10:03:00Z", + "n_total_trials": 2 +})) +""" + ) + executable.chmod(0o755) + return root, executable, config + + +def test_generalized_harbor_runner_freezes_controls_environment_and_trials( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + benchmark, executable, config = _benchmark(tmp_path) + source = tmp_path / "candidate" + source.mkdir() + _credentials(monkeypatch) + runner = HarborExperimentRunner() + + controls = runner.validate(benchmark, executable, config.relative_to(benchmark)) + run = ExperimentRun( + run_id="candidate-one", + benchmark_root=benchmark, + harbor_executable=executable, + harbor_config=config, + job_path=benchmark / "jobs/candidate-one", + log_path=tmp_path / "candidate.log", + source_root=source, + release="a" * 40, + session_id="sha256-" + "b" * 64, + controls=controls, + ) + runner.start(run) + summary = None + for _ in range(100): + summary = runner.summarize(run) + if summary is not None: + break + time.sleep(0.01) + + assert controls == ExperimentControls( + model="openai/gpt-5.4-mini", + task_ids=("task-1", "task-2"), + benchmark_config_digest=controls.benchmark_config_digest, + verifier="itsm-bench", + environment="itsm-bench", + concurrency=1, + max_retries=0, + ) + assert isinstance(summary, ExperimentSummary) + assert tuple(trial.task_id for trial in summary.trials) == ("task-1", "task-2") + assert summary.trials[0].reward == 1.0 + assert summary.trials[1].reward == 0.0 + assert summary.trials[0].started_at == datetime(2026, 9, 2, 10, 1, tzinfo=UTC) + assert summary.trials[0].evidence == ( + "harbor://candidate-one/task-1__trial/result.json", + "harbor://candidate-one/task-1__trial/verifier", + ) + environment = _EnvironmentCapture.model_validate_json( + (run.job_path / "environment.json").read_text() + ) + assert environment == _EnvironmentCapture( + source=str(source), + environment="itsm-bench", + release="a" * 40, + session="sha256-" + "b" * 64, + ) + + +def test_generalized_harbor_runner_rechecks_controls_immediately_before_launch( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + benchmark, executable, config = _benchmark(tmp_path) + source = tmp_path / "candidate" + source.mkdir() + _credentials(monkeypatch) + runner = HarborExperimentRunner() + controls = runner.validate(benchmark, executable, config.relative_to(benchmark)) + run = ExperimentRun( + run_id="candidate-one", + benchmark_root=benchmark, + harbor_executable=executable, + harbor_config=config, + job_path=benchmark / "jobs/candidate-one", + log_path=tmp_path / "candidate.log", + source_root=source, + release="a" * 40, + session_id="sha256-" + "b" * 64, + controls=controls, + ) + config.write_text(config.read_text().replace("gpt-5.4-mini", "different-model")) + + with pytest.raises(PreparationFailure) as raised: + runner.start(run) + + assert raised.value.code is PreparationErrorCode.INVALID_HARBOR_CONFIG + assert not run.job_path.exists() + + +def test_harbor_runners_reject_existing_jobs_and_unvalidated_baseline_launches( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + benchmark, executable, config = _benchmark(tmp_path) + source = tmp_path / "candidate" + source.mkdir() + _credentials(monkeypatch) + experiment = HarborExperimentRunner() + controls = experiment.validate(benchmark, executable, config.relative_to(benchmark)) + job = benchmark / "jobs/candidate-one" + job.mkdir(parents=True) + run = ExperimentRun( + run_id="candidate-one", + benchmark_root=benchmark, + harbor_executable=executable, + harbor_config=config, + job_path=job, + log_path=tmp_path / "candidate.log", + source_root=source, + release="a" * 40, + session_id="candidate-session", + controls=controls, + ) + + with pytest.raises(PreparationFailure) as existing: + experiment.start(run) + with pytest.raises(PreparationFailure) as unvalidated: + HarborBaselineRunner().start( + BaselineRun( + experiment_id="baseline", + benchmark_root=benchmark, + harbor_executable=executable, + harbor_config=config, + job_path=benchmark / "jobs/baseline", + log_path=tmp_path / "baseline.log", + worktree_path=source, + initialization_commit="a" * 40, + ) + ) + + assert existing.value.code is PreparationErrorCode.LAUNCH_FAILED + assert unvalidated.value.code is PreparationErrorCode.LAUNCH_FAILED + + +def test_generalized_harbor_runner_rejects_trials_without_mapping_fields( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + benchmark, executable, config = _benchmark(tmp_path) + source = tmp_path / "candidate" + source.mkdir() + _credentials(monkeypatch) + runner = HarborExperimentRunner() + controls = runner.validate(benchmark, executable, config.relative_to(benchmark)) + job = benchmark / "jobs/candidate-one" + trial = job / "task-1__trial" + trial.mkdir(parents=True) + (job / "result.json").write_text('{"finished_at":"2026-09-02T10:03:00Z","n_total_trials":1}') + (trial / "result.json").write_text( + '{"exception_info":null,"verifier_result":{"rewards":{"reward":1.0}}}' + ) + run = ExperimentRun( + run_id="candidate-one", + benchmark_root=benchmark, + harbor_executable=executable, + harbor_config=config, + job_path=job, + log_path=tmp_path / "candidate.log", + source_root=source, + release="a" * 40, + session_id="candidate-session", + controls=controls, + ) + + with pytest.raises(PreparationFailure) as raised: + runner.summarize(run) + + assert raised.value.code is PreparationErrorCode.INVALID_BASELINE_RESULT diff --git a/tests/test_openflywheel_mcp.py b/tests/test_openflywheel_mcp.py index 85d39d8..49e5f45 100644 --- a/tests/test_openflywheel_mcp.py +++ b/tests/test_openflywheel_mcp.py @@ -52,6 +52,11 @@ VerifierVerdict, ) from ofw.evolution import ( + CandidateExecutionInput, + CandidateExecutionObservation, + CandidateExecutionService, + CandidatePhase, + CandidateStatus, FailurePatternReferenceInput, HarnessChangeTargetInput, HypothesisObservation, @@ -98,6 +103,8 @@ def _curation_service(self) -> FailureCurationService: ... def _hypothesis_service(self) -> HypothesisService: ... + def _candidate_service(self) -> CandidateExecutionService: ... + def _program_template(self, name: str) -> str: ... def prepare_workspace( @@ -156,6 +163,11 @@ def record_failure_curation( def record_hypothesis(self, request: RecordHypothesisInput) -> HypothesisObservation: ... + def execute_candidate( + self, + request: CandidateExecutionInput, + ) -> CandidateExecutionObservation: ... + class _FakeOutcomeStore: def __init__(self) -> None: @@ -218,6 +230,16 @@ def record(self, request: RecordHypothesisInput) -> HypothesisObservation: return self.observation +class _FakeCandidateService: + def __init__(self, observation: CandidateExecutionObservation) -> None: + self.observation = observation + self.requests: list[CandidateExecutionInput] = [] + + def execute(self, request: CandidateExecutionInput) -> CandidateExecutionObservation: + self.requests.append(request) + return self.observation + + def _module() -> OpenFlywheelMcpModule: return cast(OpenFlywheelMcpModule, importlib.import_module("ofw.mcp")) @@ -397,6 +419,7 @@ def test_mcp_exposes_scoped_read_and_recording_tools() -> None: "mine_failure_patterns", "record_failure_curation", "record_hypothesis", + "execute_candidate", ] assert tuple(map(_annotation_flags, tools)) == ( (False, False, True), @@ -409,6 +432,7 @@ def test_mcp_exposes_scoped_read_and_recording_tools() -> None: (True, False, True), (False, False, True), (False, False, True), + (False, False, True), ) @@ -728,3 +752,38 @@ def test_record_hypothesis_accepts_mcp_json_mapping(tmp_path: Path) -> None: request = _hypothesis_request(tmp_path) assert RecordHypothesisInput.model_validate(_json_request(request)) == request + + +def test_execute_candidate_passes_one_strict_object_to_the_service( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + module = _module() + request = CandidateExecutionInput( + workspace_root=tmp_path / "accepted", + worktree_parent=tmp_path / "candidates", + benchmark_root=tmp_path / "benchmark", + harbor_executable=tmp_path / "harbor", + harbor_config=Path("config.json"), + experiment_id="experiment-one", + hypothesis_id=_HYPOTHESIS_ID, + ) + expected = CandidateExecutionObservation( + status=CandidateStatus.WARNING, + summary="The isolated candidate worktree is ready for the hypothesis edit.", + next_actions=("Edit the candidate, then poll.",), + artifacts=(str(tmp_path / "candidate"),), + phase=CandidatePhase.EDITING, + experiment_id="experiment-one", + hypothesis_id=_HYPOTHESIS_ID, + source_commit=_COMMIT, + worktree_path=tmp_path / "candidate", + outcome_receipts=(), + blockers=(), + ) + service = _FakeCandidateService(expected) + monkeypatch.setattr(module, "_candidate_service", lambda: service) + + assert module.execute_candidate(request) == expected + assert service.requests == [request] + assert CandidateExecutionInput.model_validate(_json_request(request)) == request diff --git a/tests/test_program_templates.py b/tests/test_program_templates.py index d07d9d4..8a6621a 100644 --- a/tests/test_program_templates.py +++ b/tests/test_program_templates.py @@ -60,7 +60,7 @@ def test_failure_pattern_miner_skill_is_packaged() -> None: assert "mine_failure_patterns" in skill.read_text(encoding="utf-8") -def test_hypothesis_former_skill_and_program_stop_before_candidate_editing() -> None: +def test_program_routes_hypothesis_receipt_into_candidate_execution() -> None: root = Path(__file__).parents[1] skill = root / "plugins/openflywheel/skills/hypothesis-former/SKILL.md" program = files("ofw.preparation.templates").joinpath("base.md").read_text(encoding="utf-8") @@ -69,7 +69,9 @@ def test_hypothesis_former_skill_and_program_stop_before_candidate_editing() -> assert "record_hypothesis" in skill.read_text(encoding="utf-8") assert "$hypothesis-former" in program assert "stable hypothesis receipt" in program - assert "stop before candidate editing" in program + assert "execute_candidate" in program + assert "returned candidate worktree" in program + assert "stop before admission" in program def test_base_program_stops_after_repeated_managed_mcp_timeout() -> None: diff --git a/tests/test_typing.py b/tests/test_typing.py index ac0da6e..754e57e 100644 --- a/tests/test_typing.py +++ b/tests/test_typing.py @@ -99,3 +99,16 @@ def test_namespace_exports_policy_and_hypothesis_contracts_without_legacy_owners assert expected <= set(package.__all__) assert {"AssetAccess", "HarnessRevision", "HarnessRevisionId"}.isdisjoint(package.__all__) + + +def test_namespace_exports_candidate_contracts_without_restoring_runtime_plane() -> None: + expected = { + "CandidateExecutionInput", + "CandidateExecutionObservation", + "CandidateId", + "CandidatePhase", + "CandidateStatus", + } + + assert expected <= set(package.__all__) + assert {"E2BSandbox", "CanaryCase", "CommandLoop"}.isdisjoint(package.__all__) From aa6c0ee94ffd23432331d60986fe788685da0364 Mon Sep 17 00:00:00 2001 From: divo12 Date: Thu, 3 Sep 2026 13:30:51 +0530 Subject: [PATCH 13/25] fix: harden candidate execution lifecycle --- src/ofw/evolution/candidate.py | 6 ++ src/ofw/evolution/candidate_git.py | 2 +- src/ofw/evolution/candidate_service.py | 71 +++++++++++-- src/ofw/evolution/hypothesis.py | 23 +++- src/ofw/evolution/hypothesis_repository.py | 5 +- src/ofw/mcp.py | 30 ++++-- src/ofw/preparation/contracts.py | 9 +- src/ofw/preparation/harbor.py | 64 +++++++----- tests/test_candidate_execution.py | 104 ++++++++++++++++++ tests/test_harbor_experiment.py | 116 ++++++++++++++++----- tests/test_hypothesis.py | 21 ++++ tests/test_openflywheel_mcp.py | 36 +++++-- 12 files changed, 403 insertions(+), 84 deletions(-) diff --git a/src/ofw/evolution/candidate.py b/src/ofw/evolution/candidate.py index 1e8c85e..c49f1a5 100644 --- a/src/ofw/evolution/candidate.py +++ b/src/ofw/evolution/candidate.py @@ -25,6 +25,7 @@ from ofw.preparation.policy import ExperimentPolicySnapshot _DIGEST_PATTERN = re.compile(r"sha256:[0-9a-f]{64}") +_IDENTIFIER_PATTERN = re.compile(r"[A-Za-z0-9][A-Za-z0-9._:@/-]*") class _CandidateIdentity(StrictModel): @@ -87,6 +88,7 @@ class CandidateErrorCode(StrEnum): REQUEST_CONFLICT = "request_conflict" MISSING_ENVIRONMENT = "missing_environment" LAUNCH_FAILED = "launch_failed" + CANDIDATE_TIMEOUT = "candidate_timeout" INVALID_RESULT = "invalid_result" OUTCOME_STORE_FAILED = "outcome_store_failed" GIT_FAILED = "git_failed" @@ -222,6 +224,10 @@ class TraceMatch: def __post_init__(self) -> None: if (self.trace_id is None) == (self.blocker is None): raise ValueError("trace match requires exactly one result") + if self.trace_id is not None and ( + len(self.trace_id) > 256 or _IDENTIFIER_PATTERN.fullmatch(self.trace_id) is None + ): + raise CandidateFailure(CandidateErrorCode.INVALID_RESULT, "trace_id") class CandidateWorkspaceGateway(Protocol): diff --git a/src/ofw/evolution/candidate_git.py b/src/ofw/evolution/candidate_git.py index feabeec..f1a1037 100644 --- a/src/ofw/evolution/candidate_git.py +++ b/src/ofw/evolution/candidate_git.py @@ -223,7 +223,7 @@ def _require_regular_path(root: Path, relative: Path) -> None: path.resolve(strict=True).relative_to(root.resolve(strict=True)) except (OSError, ValueError): raise CandidateFailure(CandidateErrorCode.UNSAFE_PATH, relative.as_posix()) from None - if path.is_symlink() or not stat.S_ISREG(metadata.st_mode): + if path.is_symlink() or not stat.S_ISREG(metadata.st_mode) or metadata.st_nlink != 1: raise CandidateFailure(CandidateErrorCode.UNSAFE_PATH, relative.as_posix()) diff --git a/src/ofw/evolution/candidate_service.py b/src/ofw/evolution/candidate_service.py index 3bd0956..247623b 100644 --- a/src/ofw/evolution/candidate_service.py +++ b/src/ofw/evolution/candidate_service.py @@ -72,6 +72,7 @@ class _CandidateState(StrictModel): deadline_at: datetime | None = None outcome_receipts: tuple[CandidateOutcomeReceipt, ...] = Field(max_length=500) blockers: tuple[CandidateBlocker, ...] = Field(max_length=500) + error_code: CandidateErrorCode | None = None @dataclass(frozen=True, slots=True) @@ -117,6 +118,8 @@ def _execute(self, request: CandidateExecutionInput) -> CandidateExecutionObserv return self._prepare(request, policy, hypothesis, control) _validate_state(request, policy, state) self._workspace.validate_accepted(request.workspace_root, policy, hypothesis) + if state.phase is CandidatePhase.FAILED: + return _persisted_failure_observation(request, state) if state.phase is CandidatePhase.COMPLETE: return _complete_observation(request, state) if state.phase is CandidatePhase.RUNNING: @@ -201,7 +204,13 @@ def _launch( policy.max_baseline_seconds, ) _write_state(control, running) - process_id = self._runner.start(run) + try: + process_id = self._runner.start(run) + except PreparationFailure as error: + failure = _runner_failure(error) + failed = _failed_state(running, failure.code) + _write_state(control, failed) + return _persisted_failure_observation(request, failed) launched = _running_state( state, candidate_id, @@ -228,7 +237,10 @@ def _poll( run = _run_from_state(request, state, controls) summary = self._runner.summarize(run) if summary is None: - _require_before_deadline(state) + if _deadline_expired(state): + failed = _failed_state(state, CandidateErrorCode.CANDIDATE_TIMEOUT) + _write_state(control, failed) + return _persisted_failure_observation(request, failed) return _running_observation(request, state) reduction = _record_outcomes( summary, @@ -496,6 +508,28 @@ def _complete_state(state: _CandidateState, reduction: _OutcomeReduction) -> _Ca ) +def _failed_state(state: _CandidateState, code: CandidateErrorCode) -> _CandidateState: + return _CandidateState( + request_digest=state.request_digest, + phase=CandidatePhase.FAILED, + source_commit=state.source_commit, + policy_digest=state.policy_digest, + controls_digest=state.controls_digest, + worktree_path=state.worktree_path, + candidate_id=state.candidate_id, + candidate_tree=state.candidate_tree, + candidate_commit=state.candidate_commit, + job_path=state.job_path, + log_path=state.log_path, + process_id=state.process_id, + started_at=state.started_at, + deadline_at=state.deadline_at, + outcome_receipts=state.outcome_receipts, + blockers=state.blockers, + error_code=code, + ) + + def _validate_state( request: CandidateExecutionInput, policy: ExperimentPolicySnapshot, @@ -509,9 +543,8 @@ def _validate_state( raise CandidateFailure(CandidateErrorCode.CONTROLS_DRIFT, request.experiment_id) -def _require_before_deadline(state: _CandidateState) -> None: - if state.deadline_at is not None and datetime.now(UTC) > state.deadline_at: - raise CandidateFailure(CandidateErrorCode.INVALID_RESULT, "candidate_timeout") +def _deadline_expired(state: _CandidateState) -> bool: + return state.deadline_at is not None and datetime.now(UTC) > state.deadline_at def _editing_observation( @@ -568,7 +601,7 @@ def _complete_observation( failures = sum(item.verdict is VerifierVerdict.FAIL for item in state.outcome_receipts) terminal = len(state.outcome_receipts) + len(state.blockers) return CandidateExecutionObservation( - status=CandidateStatus.SUCCESS, + status=CandidateStatus.WARNING if state.blockers else CandidateStatus.SUCCESS, summary="The candidate run is complete with authoritative outcome receipts.", next_actions=("Retain the candidate and outcome receipts for the admission gate.",), artifacts=_artifacts(state), @@ -609,6 +642,32 @@ def _failure_observation( ) +def _persisted_failure_observation( + request: CandidateExecutionInput, + state: _CandidateState, +) -> CandidateExecutionObservation: + code = state.error_code or CandidateErrorCode.INVALID_RESULT + return CandidateExecutionObservation( + status=CandidateStatus.ERROR, + summary=f"Candidate execution stopped: {code.value}.", + next_actions=("Retain the terminal candidate failure receipt.",), + artifacts=_artifacts(state), + phase=CandidatePhase.FAILED, + experiment_id=request.experiment_id, + hypothesis_id=request.hypothesis_id, + source_commit=state.source_commit, + candidate_id=state.candidate_id, + candidate_tree=state.candidate_tree, + candidate_commit=state.candidate_commit, + worktree_path=state.worktree_path, + job_path=state.job_path, + session_id=state.candidate_id, + outcome_receipts=state.outcome_receipts, + blockers=state.blockers, + error_code=code, + ) + + def _artifacts(state: _CandidateState) -> tuple[str, ...]: values = ( state.candidate_id, diff --git a/src/ofw/evolution/hypothesis.py b/src/ofw/evolution/hypothesis.py index 4b14c28..78682f6 100644 --- a/src/ofw/evolution/hypothesis.py +++ b/src/ofw/evolution/hypothesis.py @@ -245,6 +245,20 @@ def to_hypothesis(self) -> HarnessHypothesis: regression_risks=self.regression_risks, ) + def recomputed_id(self) -> HypothesisId: + return _hypothesis_id( + _HypothesisContent( + experiment_id=self.experiment_id, + source_commit=self.source_commit, + patterns=self.patterns, + statement=self.statement, + rationale=self.rationale, + target=self.target, + expected_effect=self.expected_effect, + regression_risks=self.regression_risks, + ) + ) + class HypothesisStatus(StrEnum): SUCCESS = "success" @@ -498,9 +512,7 @@ def _hypothesis( target = HarnessChangeTarget(request.target.component_kind, paths) risks = tuple(sorted(request.regression_risks)) content = _content(request, patterns, target, risks) - hypothesis_id = HypothesisId( - f"sha256:{hashlib.sha256(content.model_dump_json().encode('utf-8')).hexdigest()}" - ) + hypothesis_id = _hypothesis_id(content) return HarnessHypothesis( hypothesis_id, request.experiment_id, @@ -518,6 +530,11 @@ def _hypothesis( ) +def _hypothesis_id(content: _HypothesisContent) -> HypothesisId: + digest = hashlib.sha256(content.model_dump_json().encode("utf-8")).hexdigest() + return HypothesisId(f"sha256:{digest}") + + def _content( request: RecordHypothesisInput, patterns: tuple[FailurePatternReference, ...], diff --git a/src/ofw/evolution/hypothesis_repository.py b/src/ofw/evolution/hypothesis_repository.py index 1322adc..150d61e 100644 --- a/src/ofw/evolution/hypothesis_repository.py +++ b/src/ofw/evolution/hypothesis_repository.py @@ -79,7 +79,10 @@ def load(self, root: Path, hypothesis_id: str) -> HarnessHypothesis: artifact = HypothesisArtifact.model_validate_json(content) except (FileNotFoundError, SafeFileFailure, ValidationError, ValueError, OSError): raise HypothesisFailure(HypothesisErrorCode.STALE_POLICY, hypothesis_id) from None - if artifact.hypothesis_id != hypothesis_id: + if ( + artifact.hypothesis_id != hypothesis_id + or artifact.recomputed_id().value != hypothesis_id + ): raise HypothesisFailure(HypothesisErrorCode.STALE_POLICY, hypothesis_id) return artifact.to_hypothesis() diff --git a/src/ofw/mcp.py b/src/ofw/mcp.py index c4a4d77..9bdb0fc 100644 --- a/src/ofw/mcp.py +++ b/src/ofw/mcp.py @@ -4,7 +4,8 @@ from __future__ import annotations import os -from collections.abc import Callable +from collections.abc import Callable, Iterator +from contextlib import contextmanager from datetime import datetime from enum import StrEnum from importlib.resources import files @@ -175,15 +176,23 @@ def _hypothesis_service() -> HypothesisService: ) -def _candidate_service() -> CandidateExecutionService: +@contextmanager +def _candidate_service() -> Iterator[CandidateExecutionService]: client = _client() - return CandidateExecutionService( - workspace=CandidateGitGateway(), - hypotheses=FileHypothesisRepository(), - runner=HarborExperimentRunner(), - trace_locator=LangfuseCandidateTraceLocator(client), - outcome_store=_outcome_store(), - ) + try: + store = _outcome_store() + try: + yield CandidateExecutionService( + workspace=CandidateGitGateway(), + hypotheses=FileHypothesisRepository(), + runner=HarborExperimentRunner(), + trace_locator=LangfuseCandidateTraceLocator(client), + outcome_store=store, + ) + finally: + store.close() + finally: + client.close() def _program_template(name: str) -> str: @@ -338,7 +347,8 @@ def execute_candidate( request: CandidateExecutionInput, ) -> CandidateExecutionObservation: """Create, seal, launch, or poll one exact hypothesis candidate.""" - return _candidate_service().execute(request) + with _candidate_service() as service: + return service.execute(request) def main() -> None: diff --git a/src/ofw/preparation/contracts.py b/src/ofw/preparation/contracts.py index 95e4b45..3020c7b 100644 --- a/src/ofw/preparation/contracts.py +++ b/src/ofw/preparation/contracts.py @@ -3,7 +3,7 @@ from __future__ import annotations from dataclasses import dataclass -from datetime import datetime +from datetime import datetime, timedelta from enum import StrEnum from pathlib import Path from typing import Annotated, Protocol @@ -245,6 +245,13 @@ class ExperimentTrial: evaluated_at: datetime evidence: tuple[str, ...] + def __post_init__(self) -> None: + timestamps = (self.started_at, self.finished_at, self.evaluated_at) + if any(value.utcoffset() != timedelta(0) for value in timestamps): + raise ValueError("trial timestamps must be UTC") + if self.started_at >= self.finished_at or self.finished_at > self.evaluated_at: + raise ValueError("trial timestamps must be ordered") + @dataclass(frozen=True, slots=True) class ExperimentSummary: diff --git a/src/ofw/preparation/harbor.py b/src/ofw/preparation/harbor.py index 056ce41..978b7c5 100644 --- a/src/ofw/preparation/harbor.py +++ b/src/ofw/preparation/harbor.py @@ -73,6 +73,7 @@ class _HarborVerifierWire(_WireModel): class _HarborTrialResultWire(_WireModel): + task_id: str | None = None task_name: str | None = None task_checksum: str | None = None exception_info: JsonValue | None = None @@ -201,7 +202,6 @@ class HarborBaselineRunner: def __init__(self) -> None: self._runner = HarborExperimentRunner() - self._controls: ExperimentControls | None = None def validate(self, request: PrepareWorkspaceInput) -> BaselineConfiguration: controls = self._runner.validate( @@ -216,7 +216,6 @@ def validate(self, request: PrepareWorkspaceInput) -> BaselineConfiguration: str(len(controls.task_ids)), ) _credentials() - self._controls = controls return BaselineConfiguration( model=controls.model, task_ids=controls.task_ids, @@ -226,9 +225,12 @@ def validate(self, request: PrepareWorkspaceInput) -> BaselineConfiguration: ) def start(self, run: BaselineRun) -> int: - if self._controls is None: - raise PreparationFailure(PreparationErrorCode.LAUNCH_FAILED, "controls") - return self._runner.start(_baseline_experiment_run(run, self._controls)) + controls = self._runner.validate( + run.benchmark_root, + run.harbor_executable, + _relative_run_config(run.benchmark_root, run.harbor_config), + ) + return self._runner.start(_baseline_experiment_run(run, controls)) def summarize(self, run: BaselineRun) -> BaselineSummary | None: root = _finished_job_result(run.job_path) @@ -405,27 +407,33 @@ def _experiment_trial( verifier = wire.verifier_result reward = None if verifier is None or verifier.rewards is None else verifier.rewards.reward verdict = None if verifier is None else verifier.verdict - return ExperimentTrial( - task_id=task_name, - task_checksum=task_checksum, - exception=wire.exception_info is not None, - verdict=verdict, - reward=reward, - started_at=execution.started_at, - finished_at=execution.finished_at, - evaluated_at=verifier_wire.finished_at, - evidence=( - f"harbor://{run.run_id}/{directory_name}/result.json", - f"harbor://{run.run_id}/{directory_name}/verifier", - ), - ) + try: + return ExperimentTrial( + task_id=task_name, + task_checksum=task_checksum, + exception=wire.exception_info is not None, + verdict=verdict, + reward=reward, + started_at=execution.started_at, + finished_at=execution.finished_at, + evaluated_at=verifier_wire.finished_at, + evidence=( + f"harbor://{run.run_id}/{directory_name}/result.json", + f"harbor://{run.run_id}/{directory_name}/verifier", + ), + ) + except ValueError: + raise PreparationFailure( + PreparationErrorCode.INVALID_BASELINE_RESULT, + "trial timestamps", + ) from None def _required_trial_fields( wire: _HarborTrialResultWire, directory_name: str, ) -> tuple[str, str, _HarborExecutionWire, _HarborVerifierWire]: - if wire.task_name is None: + if wire.task_id is None: raise _invalid_trial(directory_name) if wire.task_checksum is None: raise _invalid_trial(directory_name) @@ -433,7 +441,7 @@ def _required_trial_fields( raise _invalid_trial(directory_name) if wire.verifier is None: raise _invalid_trial(directory_name) - return wire.task_name, wire.task_checksum, wire.agent_execution, wire.verifier + return wire.task_id, wire.task_checksum, wire.agent_execution, wire.verifier def _invalid_trial(directory_name: str) -> PreparationFailure: @@ -517,12 +525,7 @@ def _process_environment(run: ExperimentRun) -> dict[str, str]: def _require_run_controls(runner: HarborExperimentRunner, run: ExperimentRun) -> None: - try: - relative_config = run.harbor_config.resolve(strict=True).relative_to( - run.benchmark_root.resolve(strict=True) - ) - except (OSError, ValueError): - raise PreparationFailure(PreparationErrorCode.INVALID_HARBOR_CONFIG, "controls") from None + relative_config = _relative_run_config(run.benchmark_root, run.harbor_config) actual = runner.validate( run.benchmark_root, run.harbor_executable, @@ -532,6 +535,13 @@ def _require_run_controls(runner: HarborExperimentRunner, run: ExperimentRun) -> raise PreparationFailure(PreparationErrorCode.INVALID_HARBOR_CONFIG, "controls") +def _relative_run_config(benchmark_root: Path, harbor_config: Path) -> Path: + try: + return harbor_config.resolve(strict=True).relative_to(benchmark_root.resolve(strict=True)) + except (OSError, ValueError): + raise PreparationFailure(PreparationErrorCode.INVALID_HARBOR_CONFIG, "controls") from None + + def _baseline_experiment_run( run: BaselineRun, controls: ExperimentControls, diff --git a/tests/test_candidate_execution.py b/tests/test_candidate_execution.py index 4b188fa..a45e9b6 100644 --- a/tests/test_candidate_execution.py +++ b/tests/test_candidate_execution.py @@ -3,6 +3,7 @@ from __future__ import annotations import hashlib +import os import subprocess from dataclasses import replace from datetime import UTC, datetime @@ -11,6 +12,7 @@ import pytest from pydantic import ValidationError +import ofw.evolution.candidate_service as candidate_service_module from ofw.contracts import ComponentKind, Sha256Digest from ofw.evaluation.langfuse import OutcomeScoreSubmission from ofw.evaluation.outcome import OutcomeEvaluation @@ -33,6 +35,7 @@ FailurePatternReference, HarnessChangeTarget, HarnessHypothesis, + HypothesisArtifact, HypothesisId, ) from ofw.evolution.hypothesis_repository import FileHypothesisRepository @@ -143,6 +146,10 @@ def _authority(tmp_path: Path) -> tuple[Path, ExperimentPolicySnapshot, HarnessH expected_effect="The agent verifies completion.", regression_risks=(), ) + hypothesis = replace( + hypothesis, + id=HypothesisArtifact.from_hypothesis(hypothesis).recomputed_id(), + ) control = root / ".git/ofw/preparations/experiment-one" control.mkdir(parents=True) FileExperimentPolicyRepository().publish(control, policy) @@ -156,6 +163,8 @@ def __init__(self, controls: ExperimentControls) -> None: self.runs: list[ExperimentRun] = [] self.summary: ExperimentSummary | None = None self.failure: PreparationFailure | None = None + self.start_failure: PreparationFailure | None = None + self.start_count = 0 def validate( self, @@ -168,6 +177,9 @@ def validate( return self.controls def start(self, run: ExperimentRun) -> int: + self.start_count += 1 + if self.start_failure is not None: + raise self.start_failure self.runs.append(run) return 123 @@ -326,6 +338,14 @@ def test_candidate_id_changes_when_any_authority_input_changes() -> None: assert CandidateId.build(**changed) != baseline +@pytest.mark.parametrize("trace_id", ("", "invalid trace", "x" * 257)) +def test_trace_match_rejects_invalid_trace_identifiers(trace_id: str) -> None: + with pytest.raises(CandidateFailure) as raised: + TraceMatch(trace_id=trace_id, blocker=None) + + assert raised.value.code is CandidateErrorCode.INVALID_RESULT + + def test_candidate_input_rejects_extra_relative_and_escaped_paths(tmp_path: Path) -> None: root, _, hypothesis = _authority(tmp_path) request = _candidate_request(tmp_path, root, hypothesis) @@ -421,6 +441,23 @@ def test_candidate_git_gateway_rechecks_the_tree_before_commit(tmp_path: Path) - assert _git(workspace.worktree_path, "rev-parse", "HEAD") == policy.initialization_commit +def test_candidate_git_gateway_rejects_hard_linked_targets(tmp_path: Path) -> None: + root, policy, hypothesis = _authority(tmp_path) + parent = tmp_path / "candidates" + parent.mkdir() + gateway = CandidateGitGateway() + workspace = gateway.prepare(root, parent, policy, hypothesis) + candidate_target = workspace.worktree_path / "prompt.md" + candidate_target.unlink() + os.link(root / "prompt.md", candidate_target) + candidate_target.write_text("shared mutation\n") + + with pytest.raises(CandidateFailure) as raised: + gateway.inspect(workspace, policy, hypothesis) + + assert raised.value.code is CandidateErrorCode.UNSAFE_PATH + + @pytest.mark.parametrize( ("mutation", "expected"), ( @@ -711,6 +748,72 @@ def test_candidate_service_sanitizes_missing_runtime_credentials(tmp_path: Path) assert "secret" not in rejected.summary +def test_candidate_service_persists_a_terminal_launch_failure(tmp_path: Path) -> None: + root, policy, hypothesis = _authority(tmp_path) + (tmp_path / "candidates").mkdir() + request = _candidate_request(tmp_path, root, hypothesis) + runner = _FakeRunner(_controls(policy)) + runner.start_failure = PreparationFailure(PreparationErrorCode.LAUNCH_FAILED, "harbor") + service = CandidateExecutionService( + workspace=CandidateGitGateway(), + hypotheses=FileHypothesisRepository(), + runner=runner, + trace_locator=_FakeTraceLocator(), + outcome_store=_FakeOutcomeStore(), + ) + editing = service.execute(request) + assert editing.worktree_path is not None + (editing.worktree_path / "prompt.md").write_text("changed\n") + + failed = service.execute(request) + repeated = service.execute(request) + + assert failed == repeated + assert failed.phase is CandidatePhase.FAILED + assert failed.error_code is CandidateErrorCode.LAUNCH_FAILED + assert failed.candidate_commit is not None + assert runner.start_count == 1 + + +def test_candidate_service_persists_timeout_and_ignores_late_results( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + root, policy, hypothesis = _authority(tmp_path) + (tmp_path / "candidates").mkdir() + request = _candidate_request(tmp_path, root, hypothesis) + runner = _FakeRunner(_controls(policy)) + outcomes = _FakeOutcomeStore() + service = CandidateExecutionService( + workspace=CandidateGitGateway(), + hypotheses=FileHypothesisRepository(), + runner=runner, + trace_locator=_FakeTraceLocator(), + outcome_store=outcomes, + ) + editing = service.execute(request) + assert editing.worktree_path is not None + (editing.worktree_path / "prompt.md").write_text("changed\n") + running = service.execute(request) + assert running.phase is CandidatePhase.RUNNING + + class _ExpiredDateTime(datetime): + @classmethod + def now(cls, tz: object = None) -> _ExpiredDateTime: + del tz + return cls(2100, 1, 1, tzinfo=UTC) + + monkeypatch.setattr(candidate_service_module, "datetime", _ExpiredDateTime) + timed_out = service.execute(request) + runner.summary = ExperimentSummary(()) + repeated = service.execute(request) + + assert timed_out == repeated + assert timed_out.phase is CandidatePhase.FAILED + assert timed_out.error_code is CandidateErrorCode.CANDIDATE_TIMEOUT + assert outcomes.outcomes == [] + + def test_candidate_service_rejects_a_missing_hypothesis_receipt(tmp_path: Path) -> None: root, policy, hypothesis = _authority(tmp_path) request = _candidate_request(tmp_path, root, hypothesis) @@ -816,6 +919,7 @@ def test_candidate_service_keeps_unsupported_and_ambiguous_trials_unverified( complete = service.execute(request) assert complete.outcome_receipts == () + assert complete.status is CandidateStatus.WARNING assert tuple(blocker.code for blocker in complete.blockers) == ( CandidateBlockerCode.UNSUPPORTED_REWARD, CandidateBlockerCode.TRACE_AMBIGUOUS, diff --git a/tests/test_harbor_experiment.py b/tests/test_harbor_experiment.py index c5f997e..b86f8fb 100644 --- a/tests/test_harbor_experiment.py +++ b/tests/test_harbor_experiment.py @@ -2,7 +2,7 @@ from __future__ import annotations -import time +import os from datetime import UTC, datetime from pathlib import Path @@ -76,7 +76,8 @@ def _benchmark(tmp_path: Path) -> tuple[Path, Path, Path]: trial = root / f"task-{index}__trial" (trial / "verifier").mkdir(parents=True) (trial / "result.json").write_text(json.dumps({ - "task_name": f"task-{index}", + "task_name": f"display-{index}", + "task_id": f"task-{index}", "task_checksum": f"checksum-{index}", "exception_info": None, "agent_execution": { @@ -119,13 +120,9 @@ def test_generalized_harbor_runner_freezes_controls_environment_and_trials( session_id="sha256-" + "b" * 64, controls=controls, ) - runner.start(run) - summary = None - for _ in range(100): - summary = runner.summarize(run) - if summary is not None: - break - time.sleep(0.01) + pid = runner.start(run) + waited, status = os.waitpid(pid, 0) + summary = runner.summarize(run) assert controls == ExperimentControls( model="openai/gpt-5.4-mini", @@ -136,6 +133,8 @@ def test_generalized_harbor_runner_freezes_controls_environment_and_trials( concurrency=1, max_retries=0, ) + assert waited == pid + assert os.waitstatus_to_exitcode(status) == 0 assert isinstance(summary, ExperimentSummary) assert tuple(trial.task_id for trial in summary.trials) == ("task-1", "task-2") assert summary.trials[0].reward == 1.0 @@ -187,7 +186,7 @@ def test_generalized_harbor_runner_rechecks_controls_immediately_before_launch( assert not run.job_path.exists() -def test_harbor_runners_reject_existing_jobs_and_unvalidated_baseline_launches( +def test_harbor_rejects_existing_jobs_and_baseline_launch_recomputes_controls( tmp_path: Path, monkeypatch: pytest.MonkeyPatch, ) -> None: @@ -214,22 +213,91 @@ def test_harbor_runners_reject_existing_jobs_and_unvalidated_baseline_launches( with pytest.raises(PreparationFailure) as existing: experiment.start(run) - with pytest.raises(PreparationFailure) as unvalidated: - HarborBaselineRunner().start( - BaselineRun( - experiment_id="baseline", - benchmark_root=benchmark, - harbor_executable=executable, - harbor_config=config, - job_path=benchmark / "jobs/baseline", - log_path=tmp_path / "baseline.log", - worktree_path=source, - initialization_commit="a" * 40, - ) - ) + baseline_run = BaselineRun( + experiment_id="baseline", + benchmark_root=benchmark, + harbor_executable=executable, + harbor_config=config, + job_path=benchmark / "jobs/baseline", + log_path=tmp_path / "baseline.log", + worktree_path=source, + initialization_commit="a" * 40, + ) + pid = HarborBaselineRunner().start(baseline_run) + waited, status = os.waitpid(pid, 0) assert existing.value.code is PreparationErrorCode.LAUNCH_FAILED - assert unvalidated.value.code is PreparationErrorCode.LAUNCH_FAILED + assert waited == pid + assert os.waitstatus_to_exitcode(status) == 0 + assert baseline_run.job_path.is_dir() + + +@pytest.mark.parametrize( + ("started_at", "finished_at", "evaluated_at"), + ( + ( + "2026-09-02T10:01:00", + "2026-09-02T10:01:30Z", + "2026-09-02T10:01:31Z", + ), + ( + "2026-09-02T10:01:30Z", + "2026-09-02T10:01:00Z", + "2026-09-02T10:01:31Z", + ), + ( + "2026-09-02T10:01:00Z", + "2026-09-02T10:01:30Z", + "2026-09-02T10:01:29Z", + ), + ), +) +def test_generalized_harbor_runner_maps_invalid_trial_time_to_typed_failure( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + started_at: str, + finished_at: str, + evaluated_at: str, +) -> None: + benchmark, executable, config = _benchmark(tmp_path) + source = tmp_path / "candidate" + source.mkdir() + _credentials(monkeypatch) + runner = HarborExperimentRunner() + controls = runner.validate(benchmark, executable, config.relative_to(benchmark)) + job = benchmark / "jobs/candidate-one" + trial = job / "task-1__trial" + trial.mkdir(parents=True) + (job / "result.json").write_text('{"finished_at":"2026-09-02T10:03:00Z","n_total_trials":1}') + (trial / "result.json").write_text( + f"""{{ + "task_id": "task-1", + "task_name": "display name", + "task_checksum": "checksum-1", + "exception_info": null, + "agent_execution": {{"started_at": "{started_at}", "finished_at": "{finished_at}"}}, + "verifier": {{"finished_at": "{evaluated_at}"}}, + "verifier_result": {{"rewards": {{"reward": 1.0}}}} +}}""" + ) + run = ExperimentRun( + run_id="candidate-one", + benchmark_root=benchmark, + harbor_executable=executable, + harbor_config=config, + job_path=job, + log_path=tmp_path / "candidate.log", + source_root=source, + release="a" * 40, + session_id="candidate-session", + controls=controls, + ) + + with pytest.raises(PreparationFailure) as raised: + runner.summarize(run) + + assert raised.value.code is PreparationErrorCode.INVALID_BASELINE_RESULT + assert raised.value.subject == "trial timestamps" def test_generalized_harbor_runner_rejects_trials_without_mapping_fields( diff --git a/tests/test_hypothesis.py b/tests/test_hypothesis.py index 9a9f433..d8070c6 100644 --- a/tests/test_hypothesis.py +++ b/tests/test_hypothesis.py @@ -724,6 +724,27 @@ def test_hypothesis_conflicting_existing_artifact_is_rejected(tmp_path: Path) -> assert raised.value.code is HypothesisErrorCode.HYPOTHESIS_CONFLICT +def test_hypothesis_reload_rejects_content_tampering_under_the_original_id( + tmp_path: Path, +) -> None: + root, commit = _workspace(tmp_path) + request = _request(root, commit, (_diagnosis(root, "1"), _diagnosis(root, "2"))) + recorded = _service().record(request) + path = root / recorded.relative_path + path.write_text( + path.read_text(encoding="utf-8").replace( + request.statement, + "A different candidate target authority.", + ), + encoding="utf-8", + ) + + with pytest.raises(HypothesisFailure) as raised: + FileHypothesisRepository().load(root, recorded.hypothesis_id) + + assert raised.value.code is HypothesisErrorCode.STALE_POLICY + + @pytest.mark.parametrize("kind", ("symlink", "fifo")) def test_hypothesis_rejects_non_regular_artifact_target(tmp_path: Path, kind: str) -> None: root, commit = _workspace(tmp_path) diff --git a/tests/test_openflywheel_mcp.py b/tests/test_openflywheel_mcp.py index 49e5f45..5d931df 100644 --- a/tests/test_openflywheel_mcp.py +++ b/tests/test_openflywheel_mcp.py @@ -4,6 +4,7 @@ import asyncio import importlib +from contextlib import AbstractContextManager from datetime import UTC, datetime from pathlib import Path from typing import Protocol, cast @@ -103,7 +104,7 @@ def _curation_service(self) -> FailureCurationService: ... def _hypothesis_service(self) -> HypothesisService: ... - def _candidate_service(self) -> CandidateExecutionService: ... + def _candidate_service(self) -> AbstractContextManager[CandidateExecutionService]: ... def _program_template(self, name: str) -> str: ... @@ -230,14 +231,12 @@ def record(self, request: RecordHypothesisInput) -> HypothesisObservation: return self.observation -class _FakeCandidateService: - def __init__(self, observation: CandidateExecutionObservation) -> None: - self.observation = observation - self.requests: list[CandidateExecutionInput] = [] +class _FakeTraceClient: + def __init__(self) -> None: + self.close_count = 0 - def execute(self, request: CandidateExecutionInput) -> CandidateExecutionObservation: - self.requests.append(request) - return self.observation + def close(self) -> None: + self.close_count += 1 def _module() -> OpenFlywheelMcpModule: @@ -781,9 +780,24 @@ def test_execute_candidate_passes_one_strict_object_to_the_service( outcome_receipts=(), blockers=(), ) - service = _FakeCandidateService(expected) - monkeypatch.setattr(module, "_candidate_service", lambda: service) + client = _FakeTraceClient() + store = _FakeOutcomeStore() + requests: list[CandidateExecutionInput] = [] + + def execute( + service: CandidateExecutionService, + candidate_request: CandidateExecutionInput, + ) -> CandidateExecutionObservation: + del service + requests.append(candidate_request) + return expected + + monkeypatch.setattr(module, "_client", lambda: client) + monkeypatch.setattr(module, "_outcome_store", lambda: store) + monkeypatch.setattr(CandidateExecutionService, "execute", execute) assert module.execute_candidate(request) == expected - assert service.requests == [request] + assert requests == [request] + assert client.close_count == 1 + assert store.close_count == 1 assert CandidateExecutionInput.model_validate(_json_request(request)) == request From 36204b3b1adf6076a12e45aefbaa7961976b2bfe Mon Sep 17 00:00:00 2001 From: divo12 Date: Thu, 3 Sep 2026 13:58:12 +0530 Subject: [PATCH 14/25] fix: preserve controls and cancel timed out runs --- src/ofw/evolution/candidate.py | 2 + src/ofw/evolution/candidate_service.py | 1 + src/ofw/preparation/contracts.py | 1 + src/ofw/preparation/harbor.py | 19 +++-- src/ofw/preparation/service.py | 10 +++ tests/test_candidate_execution.py | 6 ++ tests/test_harbor_experiment.py | 104 +++++++++++++++++++++++++ tests/test_harbor_preparation.py | 10 +++ 8 files changed, 147 insertions(+), 6 deletions(-) diff --git a/src/ofw/evolution/candidate.py b/src/ofw/evolution/candidate.py index c49f1a5..753c9cb 100644 --- a/src/ofw/evolution/candidate.py +++ b/src/ofw/evolution/candidate.py @@ -282,6 +282,8 @@ def start(self, run: ExperimentRun) -> int: ... def summarize(self, run: ExperimentRun) -> ExperimentSummary | None: ... + def cancel(self, run: ExperimentRun, process_id: int | None) -> None: ... + class CandidateTraceLocator(Protocol): def locate(self, request: TraceMatchRequest) -> TraceMatch: ... diff --git a/src/ofw/evolution/candidate_service.py b/src/ofw/evolution/candidate_service.py index 247623b..b4b25af 100644 --- a/src/ofw/evolution/candidate_service.py +++ b/src/ofw/evolution/candidate_service.py @@ -238,6 +238,7 @@ def _poll( summary = self._runner.summarize(run) if summary is None: if _deadline_expired(state): + self._runner.cancel(run, state.process_id) failed = _failed_state(state, CandidateErrorCode.CANDIDATE_TIMEOUT) _write_state(control, failed) return _persisted_failure_observation(request, failed) diff --git a/src/ofw/preparation/contracts.py b/src/ofw/preparation/contracts.py index 3020c7b..1b26beb 100644 --- a/src/ofw/preparation/contracts.py +++ b/src/ofw/preparation/contracts.py @@ -197,6 +197,7 @@ class BaselineRun: log_path: Path worktree_path: Path initialization_commit: str + controls: ExperimentControls @dataclass(frozen=True, slots=True) diff --git a/src/ofw/preparation/harbor.py b/src/ofw/preparation/harbor.py index 978b7c5..1c8673e 100644 --- a/src/ofw/preparation/harbor.py +++ b/src/ofw/preparation/harbor.py @@ -4,6 +4,7 @@ import hashlib import os +import signal import subprocess # nosec B404 from dataclasses import dataclass from datetime import datetime @@ -192,6 +193,17 @@ def summarize(self, run: ExperimentRun) -> ExperimentSummary | None: ) return ExperimentSummary(trials) + def cancel(self, run: ExperimentRun, process_id: int | None) -> None: + del run + if process_id is None: + return + try: + os.killpg(process_id, signal.SIGTERM) + except ProcessLookupError: + return + except OSError as error: + raise PreparationFailure(PreparationErrorCode.LAUNCH_FAILED, "cancel") from error + def _trial_ids(trials: tuple[ExperimentTrial, ...]) -> tuple[str, ...]: return tuple(trial.task_id for trial in trials) @@ -225,12 +237,7 @@ def validate(self, request: PrepareWorkspaceInput) -> BaselineConfiguration: ) def start(self, run: BaselineRun) -> int: - controls = self._runner.validate( - run.benchmark_root, - run.harbor_executable, - _relative_run_config(run.benchmark_root, run.harbor_config), - ) - return self._runner.start(_baseline_experiment_run(run, controls)) + return self._runner.start(_baseline_experiment_run(run, run.controls)) def summarize(self, run: BaselineRun) -> BaselineSummary | None: root = _finished_job_result(run.job_path) diff --git a/src/ofw/preparation/service.py b/src/ofw/preparation/service.py index decbecc..adef880 100644 --- a/src/ofw/preparation/service.py +++ b/src/ofw/preparation/service.py @@ -18,6 +18,7 @@ BaselineRun, BaselineRunner, BaselineSummary, + ExperimentControls, PreparationErrorCode, PreparationFailure, PreparationPhase, @@ -396,6 +397,15 @@ def _run_from_state( log_path=state.log_path, worktree_path=state.worktree_path, initialization_commit=state.initialization_commit, + controls=ExperimentControls( + model=state.model, + task_ids=state.task_ids, + benchmark_config_digest=state.benchmark_config_digest, + verifier=state.verifier, + environment=state.environment, + concurrency=1, + max_retries=0, + ), ) diff --git a/tests/test_candidate_execution.py b/tests/test_candidate_execution.py index a45e9b6..c63a201 100644 --- a/tests/test_candidate_execution.py +++ b/tests/test_candidate_execution.py @@ -165,6 +165,7 @@ def __init__(self, controls: ExperimentControls) -> None: self.failure: PreparationFailure | None = None self.start_failure: PreparationFailure | None = None self.start_count = 0 + self.cancelled: list[tuple[ExperimentRun, int | None]] = [] def validate( self, @@ -186,6 +187,9 @@ def start(self, run: ExperimentRun) -> int: def summarize(self, run: ExperimentRun) -> ExperimentSummary | None: return self.summary + def cancel(self, run: ExperimentRun, process_id: int | None) -> None: + self.cancelled.append((run, process_id)) + class _FakeTraceLocator: def __init__(self) -> None: @@ -811,6 +815,8 @@ def now(cls, tz: object = None) -> _ExpiredDateTime: assert timed_out == repeated assert timed_out.phase is CandidatePhase.FAILED assert timed_out.error_code is CandidateErrorCode.CANDIDATE_TIMEOUT + assert len(runner.cancelled) == 1 + assert runner.cancelled[0][1] == 123 assert outcomes.outcomes == [] diff --git a/tests/test_harbor_experiment.py b/tests/test_harbor_experiment.py index b86f8fb..6cf9f9c 100644 --- a/tests/test_harbor_experiment.py +++ b/tests/test_harbor_experiment.py @@ -3,6 +3,8 @@ from __future__ import annotations import os +import subprocess +import sys from datetime import UTC, datetime from pathlib import Path @@ -97,6 +99,29 @@ def _benchmark(tmp_path: Path) -> tuple[Path, Path, Path]: return root, executable, config +def _cancel_run(tmp_path: Path) -> ExperimentRun: + return ExperimentRun( + run_id="candidate-one", + benchmark_root=tmp_path, + harbor_executable=Path(sys.executable), + harbor_config=tmp_path / "config.json", + job_path=tmp_path / "jobs/candidate-one", + log_path=tmp_path / "candidate.log", + source_root=tmp_path, + release="a" * 40, + session_id="candidate-session", + controls=ExperimentControls( + model="model", + task_ids=("task-1",), + benchmark_config_digest="sha256:" + "b" * 64, + verifier="itsm-bench", + environment="itsm-bench", + concurrency=1, + max_retries=0, + ), + ) + + def test_generalized_harbor_runner_freezes_controls_environment_and_trials( tmp_path: Path, monkeypatch: pytest.MonkeyPatch, @@ -222,6 +247,7 @@ def test_harbor_rejects_existing_jobs_and_baseline_launch_recomputes_controls( log_path=tmp_path / "baseline.log", worktree_path=source, initialization_commit="a" * 40, + controls=controls, ) pid = HarborBaselineRunner().start(baseline_run) waited, status = os.waitpid(pid, 0) @@ -232,6 +258,84 @@ def test_harbor_rejects_existing_jobs_and_baseline_launch_recomputes_controls( assert baseline_run.job_path.is_dir() +def test_baseline_launch_rejects_controls_changed_after_the_persisted_snapshot( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + benchmark, executable, config = _benchmark(tmp_path) + source = tmp_path / "candidate" + source.mkdir() + _credentials(monkeypatch) + controls = HarborExperimentRunner().validate( + benchmark, + executable, + config.relative_to(benchmark), + ) + run = BaselineRun( + experiment_id="baseline", + benchmark_root=benchmark, + harbor_executable=executable, + harbor_config=config, + job_path=benchmark / "jobs/baseline", + log_path=tmp_path / "baseline.log", + worktree_path=source, + initialization_commit="a" * 40, + controls=controls, + ) + config.write_text(config.read_text().replace("gpt-5.4-mini", "different-model")) + + with pytest.raises(PreparationFailure) as raised: + HarborBaselineRunner().start(run) + + assert raised.value.code is PreparationErrorCode.INVALID_HARBOR_CONFIG + assert not run.job_path.exists() + + +def test_generalized_harbor_cancel_terminates_and_reaps_the_process_group( + tmp_path: Path, +) -> None: + process = subprocess.Popen( # nosec B603 + (sys.executable, "-c", "import time; time.sleep(30)"), + start_new_session=True, + ) + run = _cancel_run(tmp_path) + try: + HarborExperimentRunner().cancel(run, process.pid) + return_code = process.wait(timeout=5) + finally: + if process.poll() is None: + process.kill() + process.wait(timeout=5) + + assert return_code < 0 + + +def test_generalized_harbor_cancel_handles_absent_and_failed_processes( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + runner = HarborExperimentRunner() + run = _cancel_run(tmp_path) + runner.cancel(run, None) + + def missing(process_id: int, signal_number: int) -> None: + del process_id, signal_number + raise ProcessLookupError + + monkeypatch.setattr(os, "killpg", missing) + runner.cancel(run, 123) + + def denied(process_id: int, signal_number: int) -> None: + del process_id, signal_number + raise PermissionError + + monkeypatch.setattr(os, "killpg", denied) + with pytest.raises(PreparationFailure) as raised: + runner.cancel(run, 123) + + assert raised.value.code is PreparationErrorCode.LAUNCH_FAILED + + @pytest.mark.parametrize( ("started_at", "finished_at", "evaluated_at"), ( diff --git a/tests/test_harbor_preparation.py b/tests/test_harbor_preparation.py index 9fa3abe..db4c8fd 100644 --- a/tests/test_harbor_preparation.py +++ b/tests/test_harbor_preparation.py @@ -7,6 +7,7 @@ from ofw.preparation import ( BaselineRun, BaselineSummary, + ExperimentControls, PreparationErrorCode, PreparationFailure, ) @@ -23,6 +24,15 @@ def _run(tmp_path: Path, job_path: Path) -> BaselineRun: log_path=tmp_path / "baseline.log", worktree_path=tmp_path / "worktree", initialization_commit="0" * 40, + controls=ExperimentControls( + model="openai/gpt-5.4-mini", + task_ids=(), + benchmark_config_digest="sha256:" + "0" * 64, + verifier="itsm-bench", + environment="itsm-bench", + concurrency=1, + max_retries=0, + ), ) From dff3057aa7e79d6780ba4703586270b2c38b76a2 Mon Sep 17 00:00:00 2001 From: divo12 Date: Thu, 3 Sep 2026 14:32:19 +0530 Subject: [PATCH 15/25] refactor: simplify process age parsing --- src/ofw/preparation/harbor.py | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/src/ofw/preparation/harbor.py b/src/ofw/preparation/harbor.py index 1c8673e..8404b30 100644 --- a/src/ofw/preparation/harbor.py +++ b/src/ofw/preparation/harbor.py @@ -4,6 +4,7 @@ import hashlib import os +import re import signal import subprocess # nosec B404 from dataclasses import dataclass @@ -30,6 +31,7 @@ _MAX_ADAPTER_BYTES = 512 * 1024 _AGENT_NAME = "agents.ofw_hermes:OfwHermes" _SOURCE_ENVIRONMENT_NAME = "OFW_HERMES_SOURCE" +_ELAPSED_PATTERN = re.compile(r"(?:(\d+)-)?(?:(\d+):)?(\d+):(\d+)") class _WireModel(BaseModel): @@ -205,6 +207,32 @@ def cancel(self, run: ExperimentRun, process_id: int | None) -> None: raise PreparationFailure(PreparationErrorCode.LAUNCH_FAILED, "cancel") from error +def _process_identity_matches(process_id: int, started_at: datetime | None) -> bool: + if started_at is None: + return False + try: + result = subprocess.run( + ("ps", "-p", str(process_id), "-o", "etime="), + check=False, + capture_output=True, + text=True, + ) + elapsed_seconds = _parse_elapsed_seconds(result.stdout.strip()) + except (OSError, ValueError): + return False + if result.returncode != 0: + return False + age_seconds = (datetime.now(UTC) - started_at).total_seconds() + return abs(age_seconds - elapsed_seconds) <= 2 + + +def _parse_elapsed_seconds(value: str) -> int: + match = _ELAPSED_PATTERN.fullmatch(value) + if match is None: + raise ValueError("invalid process age") + days, hours, minutes, seconds = (int(part or 0) for part in match.groups()) + return days * 86400 + hours * 3600 + minutes * 60 + seconds + def _trial_ids(trials: tuple[ExperimentTrial, ...]) -> tuple[str, ...]: return tuple(trial.task_id for trial in trials) From 270cee67cd57b9326c5a20e21c24807382ebe5b9 Mon Sep 17 00:00:00 2001 From: divo12 Date: Thu, 3 Sep 2026 14:53:37 +0530 Subject: [PATCH 16/25] feat: add durable evaluated run receipt --- src/ofw/__init__.py | 8 + src/ofw/evaluation/__init__.py | 8 + src/ofw/evaluation/outcome.py | 165 +++++++++++++++++++ src/ofw/evolution/candidate.py | 33 +++- src/ofw/evolution/candidate_langfuse.py | 23 ++- src/ofw/evolution/candidate_service.py | 42 ++++- src/ofw/preparation/contracts.py | 4 + tests/test_candidate_execution.py | 45 ++++- tests/test_evaluated_run_receipt.py | 210 ++++++++++++++++++++++++ tests/test_typing.py | 4 + 10 files changed, 525 insertions(+), 17 deletions(-) create mode 100644 tests/test_evaluated_run_receipt.py diff --git a/src/ofw/__init__.py b/src/ofw/__init__.py index 8f642c9..842d3e4 100644 --- a/src/ofw/__init__.py +++ b/src/ofw/__init__.py @@ -13,6 +13,9 @@ from ofw.contracts import ComponentKind, Sha256Digest from ofw.evaluation import ( DeferredFailure, + EvaluatedRunBlocker, + EvaluatedRunReceipt, + EvaluatedTaskReceipt, EvidenceReference, FailureCuration, FailureCurationErrorCode, @@ -39,6 +42,7 @@ OutcomeScoreSubmission, OutcomeStoreObservation, OutcomeStoreStatus, + RunSide, TaskId, VerifierId, VerifierResult, @@ -101,6 +105,9 @@ "ComponentKind", "DeferredFailure", "EvidenceReference", + "EvaluatedRunBlocker", + "EvaluatedRunReceipt", + "EvaluatedTaskReceipt", "ExperimentPolicyErrorCode", "ExperimentPolicyFailure", "ExperimentPolicySnapshot", @@ -143,6 +150,7 @@ "OutcomeScoreSubmission", "OutcomeStoreObservation", "OutcomeStoreStatus", + "RunSide", "PreparationErrorCode", "PreparationPhase", "PreparationStatus", diff --git a/src/ofw/evaluation/__init__.py b/src/ofw/evaluation/__init__.py index d39931e..980e5f9 100644 --- a/src/ofw/evaluation/__init__.py +++ b/src/ofw/evaluation/__init__.py @@ -32,10 +32,14 @@ OutcomeStoreStatus, ) from ofw.evaluation.outcome import ( + EvaluatedRunBlocker, + EvaluatedRunReceipt, + EvaluatedTaskReceipt, EvidenceReference, OutcomeErrorCode, OutcomeEvaluation, OutcomeEvaluationError, + RunSide, TaskId, VerifierId, VerifierResult, @@ -45,6 +49,9 @@ __all__ = [ "DeferredFailure", "EvidenceReference", + "EvaluatedRunBlocker", + "EvaluatedRunReceipt", + "EvaluatedTaskReceipt", "FailureCuration", "FailureCurationErrorCode", "FailureCurationFailure", @@ -67,6 +74,7 @@ "OutcomeErrorCode", "OutcomeEvaluation", "OutcomeEvaluationError", + "RunSide", "OutcomeScoreSubmission", "OutcomeStoreObservation", "OutcomeStoreStatus", diff --git a/src/ofw/evaluation/outcome.py b/src/ofw/evaluation/outcome.py index b7f53c2..d3c3c5f 100644 --- a/src/ofw/evaluation/outcome.py +++ b/src/ofw/evaluation/outcome.py @@ -2,18 +2,26 @@ from __future__ import annotations +import hashlib import math import re from dataclasses import dataclass from datetime import datetime, timedelta from enum import StrEnum +from pydantic import BaseModel, ConfigDict, Field, StrictStr, model_validator + from ofw.observability.langfuse.domain import TraceId _IDENTIFIER_PATTERN = re.compile(r"[A-Za-z0-9][A-Za-z0-9._:@/-]*") _IDENTIFIER_LIMIT = 256 _EVIDENCE_LIMIT = 10 _EVIDENCE_VALUE_LIMIT = 1024 +_RUN_ID_LIMIT = 256 +_COMMIT_PATTERN = r"[0-9a-f]{40}" +_DIGEST_PATTERN = r"sha256:[0-9a-f]{64}" +_RUN_METRIC_LIMIT = 172800.0 +_COST_LIMIT = 1_000_000.0 class OutcomeErrorCode(StrEnum): @@ -43,6 +51,163 @@ class VerifierVerdict(StrEnum): ERROR = "error" +class RunSide(StrEnum): + ACCEPTED = "accepted" + CANDIDATE = "candidate" + + +class _ReceiptModel(BaseModel): + model_config = ConfigDict(extra="forbid", frozen=True, strict=True) + + +class EvaluatedTaskReceipt(_ReceiptModel): + task_id: StrictStr = Field(min_length=1, max_length=_IDENTIFIER_LIMIT) + trace_id: StrictStr = Field(min_length=1, max_length=_IDENTIFIER_LIMIT) + score_id: StrictStr = Field(min_length=1, max_length=_IDENTIFIER_LIMIT) + verdict: VerifierVerdict + verifier_id: StrictStr = Field(min_length=1, max_length=_IDENTIFIER_LIMIT) + normalized_score: float | None + cost_usd: float | None + latency_seconds: float | None + + @model_validator(mode="after") + def validate_metrics(self) -> EvaluatedTaskReceipt: + _validate_run_metric(self.normalized_score, 0.0, 1.0, "normalized_score") + _validate_run_metric(self.cost_usd, 0.0, _COST_LIMIT, "cost_usd") + _validate_run_metric( + self.latency_seconds, + 0.0, + _RUN_METRIC_LIMIT, + "latency_seconds", + ) + _validate_verdict_score(self.verdict, self.normalized_score) + return self + + +class EvaluatedRunBlocker(_ReceiptModel): + task_id: StrictStr = Field(min_length=1, max_length=_IDENTIFIER_LIMIT) + code: StrictStr = Field(min_length=1, max_length=_IDENTIFIER_LIMIT) + subject: StrictStr = Field(min_length=1, max_length=_IDENTIFIER_LIMIT) + + +class EvaluatedRunReceipt(_ReceiptModel): + receipt_id: StrictStr = Field(pattern=_DIGEST_PATTERN) + run_id: StrictStr = Field( + min_length=1, + max_length=_RUN_ID_LIMIT, + pattern=_IDENTIFIER_PATTERN.pattern, + ) + side: RunSide + policy_digest: StrictStr = Field(pattern=_DIGEST_PATTERN) + controls_digest: StrictStr = Field(pattern=_DIGEST_PATTERN) + evaluated_commit: StrictStr = Field(pattern=_COMMIT_PATTERN) + evaluated_tree: StrictStr = Field(pattern=_COMMIT_PATTERN) + task_ids: tuple[StrictStr, ...] = Field(min_length=1, max_length=500) + outcome_receipts: tuple[EvaluatedTaskReceipt, ...] = Field(max_length=500) + blockers: tuple[EvaluatedRunBlocker, ...] = Field(max_length=500) + + @classmethod + def build( + cls, + *, + run_id: str, + side: RunSide, + policy_digest: str, + controls_digest: str, + evaluated_commit: str, + evaluated_tree: str, + task_ids: tuple[str, ...], + outcome_receipts: tuple[EvaluatedTaskReceipt, ...], + blockers: tuple[EvaluatedRunBlocker, ...], + ) -> EvaluatedRunReceipt: + draft = cls.model_construct( + receipt_id="sha256:" + "0" * 64, + run_id=run_id, + side=side, + policy_digest=policy_digest, + controls_digest=controls_digest, + evaluated_commit=evaluated_commit, + evaluated_tree=evaluated_tree, + task_ids=task_ids, + outcome_receipts=outcome_receipts, + blockers=blockers, + ) + return cls( + receipt_id=draft.recomputed_id(), + run_id=run_id, + side=side, + policy_digest=policy_digest, + controls_digest=controls_digest, + evaluated_commit=evaluated_commit, + evaluated_tree=evaluated_tree, + task_ids=task_ids, + outcome_receipts=outcome_receipts, + blockers=blockers, + ) + + def recomputed_id(self) -> str: + canonical = self.model_dump_json(exclude={"receipt_id"}) + return f"sha256:{hashlib.sha256(canonical.encode('utf-8')).hexdigest()}" + + @model_validator(mode="after") + def validate_identity_and_partition(self) -> EvaluatedRunReceipt: + if self.receipt_id != self.recomputed_id(): + raise ValueError("receipt_id does not match canonical receipt content") + task_ids = self.task_ids + _validate_unique_ids(task_ids, "task_ids") + outcome_ids = tuple(item.task_id for item in self.outcome_receipts) + blocker_ids = tuple(item.task_id for item in self.blockers) + all_result_ids = outcome_ids + blocker_ids + _validate_partition(all_result_ids, task_ids) + if not _follows_task_order(all_result_ids, task_ids): + raise ValueError("outcomes and blockers must follow task_ids order") + return self + + +def _validate_verdict_score(verdict: VerifierVerdict, score: float | None) -> None: + expected = ( + 1.0 + if verdict is VerifierVerdict.PASS + else 0.0 + if verdict is VerifierVerdict.FAIL + else None + ) + if score != expected: + raise ValueError("normalized_score does not match verdict") + + +def _validate_partition( + result_ids: tuple[str, ...], + task_ids: tuple[str, ...], +) -> None: + _validate_unique_ids(result_ids, "outcomes and blockers") + if set(result_ids) != set(task_ids): + raise ValueError("outcomes and blockers must partition task_ids") + + +def _validate_unique_ids(values: tuple[str, ...], field: str) -> None: + if len(set(values)) != len(values): + raise ValueError(f"{field} must be unique") + + +def _validate_run_metric( + value: float | None, + minimum: float, + maximum: float, + field: str, +) -> None: + if value is not None and (not math.isfinite(value) or not minimum <= value <= maximum): + raise ValueError(f"{field} is outside its finite bounds") + + +def _follows_task_order( + result_ids: tuple[str, ...], + task_ids: tuple[str, ...], +) -> bool: + positions = tuple(task_ids.index(task_id) for task_id in result_ids) + return positions == tuple(sorted(positions)) + + @dataclass(frozen=True, slots=True) class EvidenceReference: value: str diff --git a/src/ofw/evolution/candidate.py b/src/ofw/evolution/candidate.py index 753c9cb..c591ecb 100644 --- a/src/ofw/evolution/candidate.py +++ b/src/ofw/evolution/candidate.py @@ -3,17 +3,23 @@ from __future__ import annotations import hashlib +import math import re from dataclasses import dataclass from datetime import datetime from enum import StrEnum from pathlib import Path -from typing import Literal, Protocol +from typing import Literal, Protocol, TypeAlias from pydantic import Field, field_validator from ofw.evaluation.langfuse import OutcomeScoreSubmission -from ofw.evaluation.outcome import OutcomeEvaluation, VerifierVerdict +from ofw.evaluation.outcome import ( + EvaluatedRunBlocker, + EvaluatedRunReceipt, + EvaluatedTaskReceipt, + OutcomeEvaluation, +) from ofw.evolution.hypothesis import HarnessHypothesis, StrictModel from ofw.preparation.contracts import ( ExperimentControls, @@ -168,11 +174,7 @@ def validate_config(cls, value: Path) -> Path: return contained_relative_path(value, "harbor_config") -class CandidateOutcomeReceipt(StrictModel): - task_id: str = Field(min_length=1, max_length=256) - trace_id: str = Field(min_length=1, max_length=256) - score_id: str = Field(min_length=1, max_length=256) - verdict: VerifierVerdict +CandidateOutcomeReceipt: TypeAlias = EvaluatedTaskReceipt class CandidateBlocker(StrictModel): @@ -180,6 +182,13 @@ class CandidateBlocker(StrictModel): code: CandidateBlockerCode subject: str = Field(min_length=1, max_length=256) + def to_evaluated(self) -> EvaluatedRunBlocker: + return EvaluatedRunBlocker( + task_id=self.task_id, + code=self.code.value, + subject=self.subject, + ) + class CandidateExecutionObservation(StrictModel): status: CandidateStatus @@ -202,6 +211,7 @@ class CandidateExecutionObservation(StrictModel): unverified_trials: int | None = Field(default=None, ge=0, le=500) outcome_receipts: tuple[CandidateOutcomeReceipt, ...] = Field(max_length=500) blockers: tuple[CandidateBlocker, ...] = Field(max_length=500) + evaluated_run_receipt: EvaluatedRunReceipt | None = None next_poll_after_seconds: int | None = Field(default=None, ge=1, le=300) error_code: CandidateErrorCode | None = None @@ -220,6 +230,7 @@ class TraceMatchRequest: class TraceMatch: trace_id: str | None blocker: CandidateBlockerCode | None + cost_usd: float | None = None def __post_init__(self) -> None: if (self.trace_id is None) == (self.blocker is None): @@ -228,6 +239,14 @@ def __post_init__(self) -> None: len(self.trace_id) > 256 or _IDENTIFIER_PATTERN.fullmatch(self.trace_id) is None ): raise CandidateFailure(CandidateErrorCode.INVALID_RESULT, "trace_id") + _validate_cost(self.cost_usd) + + +def _validate_cost(cost_usd: float | None) -> None: + if cost_usd is not None and ( + not math.isfinite(cost_usd) or not 0.0 <= cost_usd <= 1_000_000.0 + ): + raise CandidateFailure(CandidateErrorCode.INVALID_RESULT, "cost_usd") class CandidateWorkspaceGateway(Protocol): diff --git a/src/ofw/evolution/candidate_langfuse.py b/src/ofw/evolution/candidate_langfuse.py index 616b9ca..8736f00 100644 --- a/src/ofw/evolution/candidate_langfuse.py +++ b/src/ofw/evolution/candidate_langfuse.py @@ -8,7 +8,7 @@ TraceMatchRequest, ) from ofw.observability.langfuse.contracts import TraceWindow -from ofw.observability.langfuse.domain import ObservationRecord, TraceId +from ofw.observability.langfuse.domain import ObservationRecord, PageCursor, TraceId from ofw.observability.langfuse.trace_query import ( ObservationFieldGroup, ObservationRead, @@ -39,12 +39,21 @@ def locate(self, request: TraceMatchRequest) -> TraceMatch: is_root_observation=True, ) ) - trace_ids = _trace_ids(page.records) - if not trace_ids: - return TraceMatch(None, CandidateBlockerCode.TRACE_NOT_FOUND) - if page.cursor is not None or len(set(trace_ids)) != 1: - return TraceMatch(None, CandidateBlockerCode.TRACE_AMBIGUOUS) - return TraceMatch(TraceId(trace_ids[0]).value, None) + records = tuple(record for record in page.records if record.trace_id is not None) + trace_ids = _trace_ids(records) + return _trace_match(trace_ids, page.cursor, records) + + +def _trace_match( + trace_ids: tuple[str, ...], + cursor: PageCursor | None, + records: tuple[ObservationRecord, ...], +) -> TraceMatch: + if not trace_ids: + return TraceMatch(None, CandidateBlockerCode.TRACE_NOT_FOUND) + if cursor is not None or len(set(trace_ids)) != 1: + return TraceMatch(None, CandidateBlockerCode.TRACE_AMBIGUOUS) + return TraceMatch(TraceId(trace_ids[0]).value, None, records[0].total_cost) def _trace_ids(records: tuple[ObservationRecord, ...]) -> tuple[str, ...]: diff --git a/src/ofw/evolution/candidate_service.py b/src/ofw/evolution/candidate_service.py index b4b25af..1f0deea 100644 --- a/src/ofw/evolution/candidate_service.py +++ b/src/ofw/evolution/candidate_service.py @@ -15,8 +15,10 @@ from pydantic import Field from ofw.evaluation.outcome import ( + EvaluatedRunReceipt, EvidenceReference, OutcomeEvaluation, + RunSide, TaskId, VerifierId, VerifierVerdict, @@ -72,6 +74,7 @@ class _CandidateState(StrictModel): deadline_at: datetime | None = None outcome_receipts: tuple[CandidateOutcomeReceipt, ...] = Field(max_length=500) blockers: tuple[CandidateBlocker, ...] = Field(max_length=500) + evaluated_run_receipt: EvaluatedRunReceipt | None = None error_code: CandidateErrorCode | None = None @@ -250,7 +253,8 @@ def _poll( self._trace_locator, self._outcome_store, ) - complete = _complete_state(state, reduction) + evaluated = _evaluated_receipt(state, run, controls, reduction) + complete = _complete_state(state, reduction, evaluated) _write_state(control, complete) return _complete_observation(request, complete) @@ -301,11 +305,36 @@ def _record_outcomes( trace_id=match.trace_id, score_id=submission.score_id.value, verdict=result[0], + verifier_id=outcome.verifier_id.value, + normalized_score=result[1], + cost_usd=match.cost_usd, + latency_seconds=trial.latency_seconds, ) ) return _OutcomeReduction(tuple(receipts), tuple(blockers)) +def _evaluated_receipt( + state: _CandidateState, + run: ExperimentRun, + controls: ExperimentControls, + reduction: _OutcomeReduction, +) -> EvaluatedRunReceipt: + if state.candidate_commit is None or state.candidate_tree is None: + raise CandidateFailure(CandidateErrorCode.INVALID_RESULT, run.run_id) + return EvaluatedRunReceipt.build( + run_id=run.run_id, + side=RunSide.CANDIDATE, + policy_digest=state.policy_digest, + controls_digest=state.controls_digest, + evaluated_commit=state.candidate_commit, + evaluated_tree=state.candidate_tree, + task_ids=controls.task_ids, + outcome_receipts=reduction.receipts, + blockers=tuple(item.to_evaluated() for item in reduction.blockers), + ) + + def _authoritative_result( trial: ExperimentTrial, ) -> tuple[VerifierVerdict, float | None] | CandidateBlocker: @@ -485,10 +514,15 @@ def _running_state( deadline_at=started_at + timedelta(seconds=timeout_seconds), outcome_receipts=(), blockers=(), + evaluated_run_receipt=None, ) -def _complete_state(state: _CandidateState, reduction: _OutcomeReduction) -> _CandidateState: +def _complete_state( + state: _CandidateState, + reduction: _OutcomeReduction, + evaluated: EvaluatedRunReceipt, +) -> _CandidateState: return _CandidateState( request_digest=state.request_digest, phase=CandidatePhase.COMPLETE, @@ -506,6 +540,7 @@ def _complete_state(state: _CandidateState, reduction: _OutcomeReduction) -> _Ca deadline_at=state.deadline_at, outcome_receipts=reduction.receipts, blockers=reduction.blockers, + evaluated_run_receipt=evaluated, ) @@ -527,6 +562,7 @@ def _failed_state(state: _CandidateState, code: CandidateErrorCode) -> _Candidat deadline_at=state.deadline_at, outcome_receipts=state.outcome_receipts, blockers=state.blockers, + evaluated_run_receipt=state.evaluated_run_receipt, error_code=code, ) @@ -622,6 +658,7 @@ def _complete_observation( unverified_trials=len(state.blockers), outcome_receipts=state.outcome_receipts, blockers=state.blockers, + evaluated_run_receipt=state.evaluated_run_receipt, ) @@ -665,6 +702,7 @@ def _persisted_failure_observation( session_id=state.candidate_id, outcome_receipts=state.outcome_receipts, blockers=state.blockers, + evaluated_run_receipt=state.evaluated_run_receipt, error_code=code, ) diff --git a/src/ofw/preparation/contracts.py b/src/ofw/preparation/contracts.py index 1b26beb..3804eae 100644 --- a/src/ofw/preparation/contracts.py +++ b/src/ofw/preparation/contracts.py @@ -253,6 +253,10 @@ def __post_init__(self) -> None: if self.started_at >= self.finished_at or self.finished_at > self.evaluated_at: raise ValueError("trial timestamps must be ordered") + @property + def latency_seconds(self) -> float: + return (self.finished_at - self.started_at).total_seconds() + @dataclass(frozen=True, slots=True) class ExperimentSummary: diff --git a/tests/test_candidate_execution.py b/tests/test_candidate_execution.py index c63a201..8956f9e 100644 --- a/tests/test_candidate_execution.py +++ b/tests/test_candidate_execution.py @@ -195,12 +195,17 @@ class _FakeTraceLocator: def __init__(self) -> None: self.requests: list[TraceMatchRequest] = [] self.blocker: CandidateBlockerCode | None = None + self.cost_usd: float | None = None def locate(self, request: TraceMatchRequest) -> TraceMatch: self.requests.append(request) if self.blocker is not None: return TraceMatch(trace_id=None, blocker=self.blocker) - return TraceMatch(trace_id=f"trace-{request.task_id}", blocker=None) + return TraceMatch( + trace_id=f"trace-{request.task_id}", + blocker=None, + cost_usd=self.cost_usd, + ) class _FakeOutcomeStore: @@ -578,6 +583,7 @@ def test_candidate_service_creates_launches_polls_and_replays_authoritative_rece request = _candidate_request(tmp_path, root, hypothesis) runner = _FakeRunner(_controls(policy)) locator = _FakeTraceLocator() + locator.cost_usd = 0.25 outcomes = _FakeOutcomeStore() service = CandidateExecutionService( workspace=CandidateGitGateway(), @@ -642,6 +648,18 @@ def test_candidate_service_creates_launches_polls_and_replays_authoritative_rece "task-1", "task-2", ) + assert complete.evaluated_run_receipt is not None + assert complete.evaluated_run_receipt.task_ids == policy.task_ids + assert complete.evaluated_run_receipt.outcome_receipts[0].cost_usd == 0.25 + assert complete.evaluated_run_receipt.outcome_receipts[0].latency_seconds == 30.0 + reloaded = CandidateExecutionService( + workspace=CandidateGitGateway(), + hypotheses=FileHypothesisRepository(), + runner=runner, + trace_locator=locator, + outcome_store=outcomes, + ).execute(request) + assert reloaded.evaluated_run_receipt == complete.evaluated_run_receipt assert len(runner.runs) == 1 assert len(locator.requests) == 2 assert len(outcomes.outcomes) == 2 @@ -694,6 +712,25 @@ def test_trace_locator_requires_exactly_one_complete_structural_match( assert tuple(field.value for field in query.fields) == ("core", "basic", "trace_context") +def test_trace_locator_preserves_provider_attributed_cost() -> None: + record = replace(_root_observation("trace-1", "cost"), total_cost=0.75) + reader = _ObservationReader(ObservationPage((record,), None)) + request = TraceMatchRequest( + task_id="task-1", + session_id="candidate-session", + environment="itsm-bench", + release="d" * 40, + started_at=datetime(2026, 9, 2, 10, 1, tzinfo=UTC), + finished_at=datetime(2026, 9, 2, 10, 2, tzinfo=UTC), + ) + + assert LangfuseCandidateTraceLocator(reader).locate(request) == TraceMatch( + trace_id="trace-1", + blocker=None, + cost_usd=0.75, + ) + + def test_candidate_service_rejects_controls_drift_before_commit_or_launch(tmp_path: Path) -> None: root, policy, hypothesis = _authority(tmp_path) (tmp_path / "candidates").mkdir() @@ -926,6 +963,12 @@ def test_candidate_service_keeps_unsupported_and_ambiguous_trials_unverified( assert complete.outcome_receipts == () assert complete.status is CandidateStatus.WARNING + assert complete.evaluated_run_receipt is not None + assert complete.evaluated_run_receipt.task_ids == policy.task_ids + assert tuple(item.task_id for item in complete.evaluated_run_receipt.blockers) == ( + "task-1", + "task-2", + ) assert tuple(blocker.code for blocker in complete.blockers) == ( CandidateBlockerCode.UNSUPPORTED_REWARD, CandidateBlockerCode.TRACE_AMBIGUOUS, diff --git a/tests/test_evaluated_run_receipt.py b/tests/test_evaluated_run_receipt.py new file mode 100644 index 0000000..51bb793 --- /dev/null +++ b/tests/test_evaluated_run_receipt.py @@ -0,0 +1,210 @@ +"""Strict durable evaluated-run receipt contracts.""" + +from __future__ import annotations + +import math + +import pytest +from pydantic import TypeAdapter, ValidationError + +from ofw.evaluation.outcome import ( + EvaluatedRunBlocker, + EvaluatedRunReceipt, + EvaluatedTaskReceipt, + RunSide, + VerifierVerdict, +) + +_DIGEST = "sha256:" + "a" * 64 +_COMMIT = "b" * 40 +_TREE = "c" * 40 +_JSON_OBJECT = TypeAdapter(dict[str, object]) + + +def _task( + task_id: str, + *, + verdict: VerifierVerdict = VerifierVerdict.PASS, + score: float | None = 1.0, +) -> EvaluatedTaskReceipt: + return EvaluatedTaskReceipt( + task_id=task_id, + trace_id=f"trace-{task_id}", + score_id=f"score-{task_id}", + verdict=verdict, + verifier_id="itsm-bench@checksum", + normalized_score=score, + cost_usd=0.25, + latency_seconds=1.5, + ) + + +def _blocker(task_id: str) -> EvaluatedRunBlocker: + return EvaluatedRunBlocker( + task_id=task_id, + code="trace_ambiguous", + subject="trace_mapping", + ) + + +def _receipt( + task_ids: tuple[str, ...] = ("task-1", "task-2"), + outcomes: tuple[EvaluatedTaskReceipt, ...] = (), + blockers: tuple[EvaluatedRunBlocker, ...] = (), +) -> EvaluatedRunReceipt: + return EvaluatedRunReceipt.build( + run_id="run-1", + side=RunSide.CANDIDATE, + policy_digest=_DIGEST, + controls_digest=_DIGEST, + evaluated_commit=_COMMIT, + evaluated_tree=_TREE, + task_ids=task_ids, + outcome_receipts=outcomes, + blockers=blockers, + ) + + +def test_evaluated_run_receipt_is_immutable_and_deterministic() -> None: + first = _receipt(outcomes=(_task("task-1"),), blockers=(_blocker("task-2"),)) + second = _receipt(outcomes=(_task("task-1"),), blockers=(_blocker("task-2"),)) + + assert first == second + assert first.receipt_id == first.recomputed_id() + assert first.receipt_id.startswith("sha256:") + with pytest.raises(ValidationError): + first.run_id = "other" + + +def test_evaluated_run_receipt_rejects_tampered_hash_and_extra_fields() -> None: + receipt = _receipt(outcomes=(_task("task-1"),), blockers=(_blocker("task-2"),)) + payload = _JSON_OBJECT.validate_json(receipt.model_dump_json()) + payload["run_id"] = "tampered" + with pytest.raises(ValidationError): + EvaluatedRunReceipt.model_validate(payload) + + payload = _JSON_OBJECT.validate_json(receipt.model_dump_json()) + payload["unexpected"] = True + with pytest.raises(ValidationError): + EvaluatedRunReceipt.model_validate(payload) + + +@pytest.mark.parametrize( + ("task_ids", "outcomes", "blockers"), + ( + (("task-1", "task-2"), (_task("task-1"),), ()), + (("task-1",), (_task("task-1"),), (_blocker("task-1"),)), + (("task-1",), (_task("task-1"), _task("task-1")), ()), + (("task-1",), (), (_blocker("task-1"), _blocker("task-1"))), + (("task-1",), (), (_blocker("task-2"),)), + ), +) +def test_evaluated_run_receipt_requires_an_exact_unique_partition( + task_ids: tuple[str, ...], + outcomes: tuple[EvaluatedTaskReceipt, ...], + blockers: tuple[EvaluatedRunBlocker, ...], +) -> None: + with pytest.raises((ValidationError, ValueError)): + _receipt(task_ids, outcomes, blockers) + + +def test_evaluated_run_receipt_requires_task_order_within_each_partition() -> None: + with pytest.raises((ValidationError, ValueError)): + _receipt( + task_ids=("task-1", "task-2"), + outcomes=(_task("task-2"),), + blockers=(_blocker("task-1"),), + ) + + +@pytest.mark.parametrize( + ("verdict", "score"), + ( + (VerifierVerdict.PASS, None), + (VerifierVerdict.PASS, 0.5), + (VerifierVerdict.FAIL, 1.0), + (VerifierVerdict.ABSTAIN, 0.0), + ), +) +def test_evaluated_task_receipt_validates_decisive_scores( + verdict: VerifierVerdict, + score: float | None, +) -> None: + with pytest.raises(ValidationError): + _task("task-1", verdict=verdict, score=score) + + +@pytest.mark.parametrize( + "field", + ("normalized_score", "cost_usd", "latency_seconds"), +) +def test_evaluated_task_receipt_rejects_non_finite_or_out_of_bound_metrics( + field: str, +) -> None: + values = { + "normalized_score": 1.0, + "cost_usd": 0.25, + "latency_seconds": 1.5, + } + values[field] = math.inf + with pytest.raises(ValidationError): + EvaluatedTaskReceipt( + task_id="task-1", + trace_id="trace-task-1", + score_id="score-task-1", + verdict=VerifierVerdict.PASS, + verifier_id="itsm-bench@checksum", + **values, + ) + + +def test_evaluated_task_receipt_preserves_missing_optional_metrics() -> None: + receipt = EvaluatedTaskReceipt( + task_id="task-1", + trace_id="trace-task-1", + score_id="score-task-1", + verdict=VerifierVerdict.ABSTAIN, + verifier_id="itsm-bench@checksum", + normalized_score=None, + cost_usd=None, + latency_seconds=None, + ) + + assert receipt.cost_usd is None + assert receipt.latency_seconds is None + + +def test_evaluated_run_receipt_rejects_non_strict_scalar_input() -> None: + payload = _receipt(outcomes=(_task("task-1"),), blockers=(_blocker("task-2"),)) + raw = _JSON_OBJECT.validate_json(payload.model_dump_json()) + raw["run_id"] = 1 + with pytest.raises(ValidationError): + EvaluatedRunReceipt.model_validate(raw) + + +def test_evaluated_run_receipt_identity_changes_with_receipt_content() -> None: + first = _receipt(outcomes=(_task("task-1"),), blockers=(_blocker("task-2"),)) + changed = EvaluatedRunReceipt.build( + run_id=first.run_id, + side=first.side, + policy_digest=first.policy_digest, + controls_digest=first.controls_digest, + evaluated_commit=first.evaluated_commit, + evaluated_tree=first.evaluated_tree, + task_ids=first.task_ids, + outcome_receipts=( + EvaluatedTaskReceipt( + task_id="task-1", + trace_id="trace-task-1", + score_id="score-task-1", + verdict=VerifierVerdict.PASS, + verifier_id="itsm-bench@checksum", + normalized_score=1.0, + cost_usd=0.5, + latency_seconds=1.5, + ), + ), + blockers=first.blockers, + ) + + assert first.receipt_id != changed.receipt_id diff --git a/tests/test_typing.py b/tests/test_typing.py index 754e57e..17f730f 100644 --- a/tests/test_typing.py +++ b/tests/test_typing.py @@ -35,7 +35,11 @@ def test_package_excludes_removed_harness_plane() -> None: def test_namespace_exports_authoritative_outcome_contract() -> None: + assert "EvaluatedRunBlocker" in package.__all__ + assert "EvaluatedRunReceipt" in package.__all__ + assert "EvaluatedTaskReceipt" in package.__all__ assert "OutcomeEvaluation" in package.__all__ + assert "RunSide" in package.__all__ assert "LangfuseOutcomeStore" in package.__all__ assert "EvidenceReference" in package.__all__ assert "TaskId" in package.__all__ From 024085265ea4bc1b7d77551696f87bbe72d4f615 Mon Sep 17 00:00:00 2001 From: divo12 Date: Thu, 3 Sep 2026 14:56:01 +0530 Subject: [PATCH 17/25] fix: default optional receipt metrics --- src/ofw/evaluation/outcome.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/ofw/evaluation/outcome.py b/src/ofw/evaluation/outcome.py index d3c3c5f..3980274 100644 --- a/src/ofw/evaluation/outcome.py +++ b/src/ofw/evaluation/outcome.py @@ -66,9 +66,9 @@ class EvaluatedTaskReceipt(_ReceiptModel): score_id: StrictStr = Field(min_length=1, max_length=_IDENTIFIER_LIMIT) verdict: VerifierVerdict verifier_id: StrictStr = Field(min_length=1, max_length=_IDENTIFIER_LIMIT) - normalized_score: float | None - cost_usd: float | None - latency_seconds: float | None + normalized_score: float | None = None + cost_usd: float | None = None + latency_seconds: float | None = None @model_validator(mode="after") def validate_metrics(self) -> EvaluatedTaskReceipt: From 422a0dc8812dec567996ac5777f09c4cf9042f98 Mon Sep 17 00:00:00 2001 From: divo12 Date: Thu, 3 Sep 2026 15:11:13 +0530 Subject: [PATCH 18/25] fix: harden evaluated receipt invariants --- src/ofw/evaluation/outcome.py | 16 +++++++++++----- tests/test_evaluated_run_receipt.py | 17 ++++++++++++----- 2 files changed, 23 insertions(+), 10 deletions(-) diff --git a/src/ofw/evaluation/outcome.py b/src/ofw/evaluation/outcome.py index 3980274..b938181 100644 --- a/src/ofw/evaluation/outcome.py +++ b/src/ofw/evaluation/outcome.py @@ -18,8 +18,9 @@ _EVIDENCE_LIMIT = 10 _EVIDENCE_VALUE_LIMIT = 1024 _RUN_ID_LIMIT = 256 -_COMMIT_PATTERN = r"[0-9a-f]{40}" -_DIGEST_PATTERN = r"sha256:[0-9a-f]{64}" +_COMMIT_PATTERN = r"^[0-9a-f]{40}$" +_DIGEST_PATTERN = r"^sha256:[0-9a-f]{64}$" +_RUN_ID_PATTERN = r"^[A-Za-z0-9][A-Za-z0-9._:@/-]*$" _RUN_METRIC_LIMIT = 172800.0 _COST_LIMIT = 1_000_000.0 @@ -95,7 +96,7 @@ class EvaluatedRunReceipt(_ReceiptModel): run_id: StrictStr = Field( min_length=1, max_length=_RUN_ID_LIMIT, - pattern=_IDENTIFIER_PATTERN.pattern, + pattern=_RUN_ID_PATTERN, ) side: RunSide policy_digest: StrictStr = Field(pattern=_DIGEST_PATTERN) @@ -159,8 +160,8 @@ def validate_identity_and_partition(self) -> EvaluatedRunReceipt: blocker_ids = tuple(item.task_id for item in self.blockers) all_result_ids = outcome_ids + blocker_ids _validate_partition(all_result_ids, task_ids) - if not _follows_task_order(all_result_ids, task_ids): - raise ValueError("outcomes and blockers must follow task_ids order") + _validate_result_order(outcome_ids, task_ids) + _validate_result_order(blocker_ids, task_ids) return self @@ -190,6 +191,11 @@ def _validate_unique_ids(values: tuple[str, ...], field: str) -> None: raise ValueError(f"{field} must be unique") +def _validate_result_order(result_ids: tuple[str, ...], task_ids: tuple[str, ...]) -> None: + if not _follows_task_order(result_ids, task_ids): + raise ValueError("outcomes and blockers must follow task_ids order") + + def _validate_run_metric( value: float | None, minimum: float, diff --git a/tests/test_evaluated_run_receipt.py b/tests/test_evaluated_run_receipt.py index 51bb793..cf34f53 100644 --- a/tests/test_evaluated_run_receipt.py +++ b/tests/test_evaluated_run_receipt.py @@ -112,8 +112,7 @@ def test_evaluated_run_receipt_requires_task_order_within_each_partition() -> No with pytest.raises((ValidationError, ValueError)): _receipt( task_ids=("task-1", "task-2"), - outcomes=(_task("task-2"),), - blockers=(_blocker("task-1"),), + outcomes=(_task("task-2"), _task("task-1")), ) @@ -135,18 +134,26 @@ def test_evaluated_task_receipt_validates_decisive_scores( @pytest.mark.parametrize( - "field", - ("normalized_score", "cost_usd", "latency_seconds"), + ("field", "value"), + ( + ("normalized_score", 2.0), + ("cost_usd", 1_000_001.0), + ("latency_seconds", 172_801.0), + ("normalized_score", math.inf), + ("cost_usd", math.inf), + ("latency_seconds", math.inf), + ), ) def test_evaluated_task_receipt_rejects_non_finite_or_out_of_bound_metrics( field: str, + value: float, ) -> None: values = { "normalized_score": 1.0, "cost_usd": 0.25, "latency_seconds": 1.5, } - values[field] = math.inf + values[field] = value with pytest.raises(ValidationError): EvaluatedTaskReceipt( task_id="task-1", From cc0503216a420e226321c3253f140a189e3987db Mon Sep 17 00:00:00 2001 From: divo12 Date: Thu, 3 Sep 2026 15:18:26 +0530 Subject: [PATCH 19/25] refactor: trim candidate execution abstractions --- src/ofw/__init__.py | 4 -- src/ofw/evolution/__init__.py | 4 -- src/ofw/evolution/candidate.py | 64 ++------------------ src/ofw/evolution/candidate_service.py | 82 +++++++++++--------------- src/ofw/preparation/harbor.py | 6 +- tests/test_candidate_execution.py | 10 ++-- tests/test_typing.py | 1 + 7 files changed, 46 insertions(+), 125 deletions(-) diff --git a/src/ofw/__init__.py b/src/ofw/__init__.py index 842d3e4..7287b86 100644 --- a/src/ofw/__init__.py +++ b/src/ofw/__init__.py @@ -49,14 +49,12 @@ VerifierVerdict, ) from ofw.evolution import ( - CandidateBlocker, CandidateBlockerCode, CandidateErrorCode, CandidateExecutionInput, CandidateExecutionObservation, CandidateFailure, CandidateId, - CandidateOutcomeReceipt, CandidatePhase, CandidateStatus, FailurePatternReference, @@ -90,14 +88,12 @@ ) __all__ = [ - "CandidateBlocker", "CandidateBlockerCode", "CandidateErrorCode", "CandidateExecutionInput", "CandidateExecutionObservation", "CandidateFailure", "CandidateId", - "CandidateOutcomeReceipt", "CandidatePhase", "CandidateStatus", "CollectionError", diff --git a/src/ofw/evolution/__init__.py b/src/ofw/evolution/__init__.py index 866f0ca..2d6146a 100644 --- a/src/ofw/evolution/__init__.py +++ b/src/ofw/evolution/__init__.py @@ -1,14 +1,12 @@ """Prepared-experiment evolution contracts.""" from ofw.evolution.candidate import ( - CandidateBlocker, CandidateBlockerCode, CandidateErrorCode, CandidateExecutionInput, CandidateExecutionObservation, CandidateFailure, CandidateId, - CandidateOutcomeReceipt, CandidatePhase, CandidateStatus, ) @@ -32,7 +30,6 @@ from ofw.evolution.hypothesis_repository import FileHypothesisRepository __all__ = [ - "CandidateBlocker", "CandidateBlockerCode", "CandidateErrorCode", "CandidateExecutionInput", @@ -41,7 +38,6 @@ "CandidateFailure", "CandidateGitGateway", "CandidateId", - "CandidateOutcomeReceipt", "CandidatePhase", "CandidateStatus", "FailurePatternReference", diff --git a/src/ofw/evolution/candidate.py b/src/ofw/evolution/candidate.py index c591ecb..8f8232d 100644 --- a/src/ofw/evolution/candidate.py +++ b/src/ofw/evolution/candidate.py @@ -9,7 +9,7 @@ from datetime import datetime from enum import StrEnum from pathlib import Path -from typing import Literal, Protocol, TypeAlias +from typing import Literal, Protocol from pydantic import Field, field_validator @@ -20,7 +20,7 @@ EvaluatedTaskReceipt, OutcomeEvaluation, ) -from ofw.evolution.hypothesis import HarnessHypothesis, StrictModel +from ofw.evolution.hypothesis import StrictModel from ofw.preparation.contracts import ( ExperimentControls, ExperimentRun, @@ -174,22 +174,6 @@ def validate_config(cls, value: Path) -> Path: return contained_relative_path(value, "harbor_config") -CandidateOutcomeReceipt: TypeAlias = EvaluatedTaskReceipt - - -class CandidateBlocker(StrictModel): - task_id: str = Field(min_length=1, max_length=256) - code: CandidateBlockerCode - subject: str = Field(min_length=1, max_length=256) - - def to_evaluated(self) -> EvaluatedRunBlocker: - return EvaluatedRunBlocker( - task_id=self.task_id, - code=self.code.value, - subject=self.subject, - ) - - class CandidateExecutionObservation(StrictModel): status: CandidateStatus summary: str = Field(min_length=1, max_length=256) @@ -209,8 +193,8 @@ class CandidateExecutionObservation(StrictModel): verifier_passes: int | None = Field(default=None, ge=0, le=500) verifier_failures: int | None = Field(default=None, ge=0, le=500) unverified_trials: int | None = Field(default=None, ge=0, le=500) - outcome_receipts: tuple[CandidateOutcomeReceipt, ...] = Field(max_length=500) - blockers: tuple[CandidateBlocker, ...] = Field(max_length=500) + outcome_receipts: tuple[EvaluatedTaskReceipt, ...] = Field(max_length=500) + blockers: tuple[EvaluatedRunBlocker, ...] = Field(max_length=500) evaluated_run_receipt: EvaluatedRunReceipt | None = None next_poll_after_seconds: int | None = Field(default=None, ge=1, le=300) error_code: CandidateErrorCode | None = None @@ -249,46 +233,6 @@ def _validate_cost(cost_usd: float | None) -> None: raise CandidateFailure(CandidateErrorCode.INVALID_RESULT, "cost_usd") -class CandidateWorkspaceGateway(Protocol): - def control_directory(self, root: Path, hypothesis_id: str) -> Path: ... - - def prepare( - self, - accepted_root: Path, - worktree_parent: Path, - policy: ExperimentPolicySnapshot, - hypothesis: HarnessHypothesis, - ) -> CandidateWorkspace: ... - - def inspect( - self, - workspace: CandidateWorkspace, - policy: ExperimentPolicySnapshot, - hypothesis: HarnessHypothesis, - ) -> CandidateTree: ... - - def commit( - self, - workspace: CandidateWorkspace, - tree: CandidateTree, - candidate_id: CandidateId, - experiment_id: str, - ) -> CandidateCommit: ... - - def validate_accepted( - self, - root: Path, - policy: ExperimentPolicySnapshot, - hypothesis: HarnessHypothesis, - ) -> None: ... - - -class CandidateHypothesisRepository(Protocol): - def load_policy(self, root: Path, experiment_id: str) -> ExperimentPolicySnapshot: ... - - def load(self, root: Path, hypothesis_id: str) -> HarnessHypothesis: ... - - class CandidateExperimentRunner(Protocol): def validate( self, diff --git a/src/ofw/evolution/candidate_service.py b/src/ofw/evolution/candidate_service.py index 1f0deea..deca884 100644 --- a/src/ofw/evolution/candidate_service.py +++ b/src/ofw/evolution/candidate_service.py @@ -15,7 +15,9 @@ from pydantic import Field from ofw.evaluation.outcome import ( + EvaluatedRunBlocker, EvaluatedRunReceipt, + EvaluatedTaskReceipt, EvidenceReference, OutcomeEvaluation, RunSide, @@ -24,26 +26,24 @@ VerifierVerdict, ) from ofw.evolution.candidate import ( - CandidateBlocker, CandidateBlockerCode, CandidateErrorCode, CandidateExecutionInput, CandidateExecutionObservation, CandidateExperimentRunner, CandidateFailure, - CandidateHypothesisRepository, CandidateId, - CandidateOutcomeReceipt, CandidateOutcomeStore, CandidatePhase, CandidateStatus, CandidateTraceLocator, CandidateWorkspace, - CandidateWorkspaceGateway, TraceMatchRequest, candidate_policy_digest, ) +from ofw.evolution.candidate_git import CandidateGitGateway from ofw.evolution.hypothesis import HarnessHypothesis, HypothesisFailure, StrictModel +from ofw.evolution.hypothesis_repository import FileHypothesisRepository from ofw.observability.langfuse.domain import TraceId from ofw.preparation.contracts import ( ExperimentControls, @@ -72,24 +72,22 @@ class _CandidateState(StrictModel): process_id: int | None = Field(default=None, ge=1) started_at: datetime | None = None deadline_at: datetime | None = None - outcome_receipts: tuple[CandidateOutcomeReceipt, ...] = Field(max_length=500) - blockers: tuple[CandidateBlocker, ...] = Field(max_length=500) evaluated_run_receipt: EvaluatedRunReceipt | None = None error_code: CandidateErrorCode | None = None @dataclass(frozen=True, slots=True) class _OutcomeReduction: - receipts: tuple[CandidateOutcomeReceipt, ...] - blockers: tuple[CandidateBlocker, ...] + receipts: tuple[EvaluatedTaskReceipt, ...] + blockers: tuple[EvaluatedRunBlocker, ...] class CandidateExecutionService: def __init__( self, *, - workspace: CandidateWorkspaceGateway, - hypotheses: CandidateHypothesisRepository, + workspace: CandidateGitGateway, + hypotheses: FileHypothesisRepository, runner: CandidateExperimentRunner, trace_locator: CandidateTraceLocator, outcome_store: CandidateOutcomeStore, @@ -126,7 +124,7 @@ def _execute(self, request: CandidateExecutionInput) -> CandidateExecutionObserv if state.phase is CandidatePhase.COMPLETE: return _complete_observation(request, state) if state.phase is CandidatePhase.RUNNING: - return self._poll(request, policy, hypothesis, control, state) + return self._poll(request, policy, control, state) return self._launch(request, policy, hypothesis, control, state) def _authority( @@ -138,10 +136,6 @@ def _authority( hypothesis = self._hypotheses.load(request.workspace_root, request.hypothesis_id) except (ExperimentPolicyFailure, HypothesisFailure): raise CandidateFailure(CandidateErrorCode.STALE_POLICY, request.experiment_id) from None - if hypothesis.experiment_id != policy.experiment_id: - raise CandidateFailure(CandidateErrorCode.STALE_POLICY, request.hypothesis_id) - if hypothesis.source_commit != policy.initialization_commit: - raise CandidateFailure(CandidateErrorCode.STALE_COMMIT, request.hypothesis_id) return policy, hypothesis def _prepare( @@ -164,8 +158,6 @@ def _prepare( policy_digest=candidate_policy_digest(policy), controls_digest=policy.controls_digest, worktree_path=prepared.worktree_path, - outcome_receipts=(), - blockers=(), ) _write_state(control, state) return _editing_observation(request, state) @@ -231,11 +223,9 @@ def _poll( self, request: CandidateExecutionInput, policy: ExperimentPolicySnapshot, - hypothesis: HarnessHypothesis, control: Path, state: _CandidateState, ) -> CandidateExecutionObservation: - del hypothesis controls = self._validated_controls(request, policy) run = _run_from_state(request, state, controls) summary = self._runner.summarize(run) @@ -254,7 +244,7 @@ def _poll( self._outcome_store, ) evaluated = _evaluated_receipt(state, run, controls, reduction) - complete = _complete_state(state, reduction, evaluated) + complete = _complete_state(state, evaluated) _write_state(control, complete) return _complete_observation(request, complete) @@ -280,11 +270,11 @@ def _record_outcomes( trace_locator: CandidateTraceLocator, outcome_store: CandidateOutcomeStore, ) -> _OutcomeReduction: - receipts: list[CandidateOutcomeReceipt] = [] - blockers: list[CandidateBlocker] = [] + receipts: list[EvaluatedTaskReceipt] = [] + blockers: list[EvaluatedRunBlocker] = [] for trial in summary.trials: result = _authoritative_result(trial) - if isinstance(result, CandidateBlocker): + if isinstance(result, EvaluatedRunBlocker): blockers.append(result) continue match = trace_locator.locate(_trace_request(trial, run, controls)) @@ -300,7 +290,7 @@ def _record_outcomes( trial.task_id, ) from None receipts.append( - CandidateOutcomeReceipt( + EvaluatedTaskReceipt( task_id=trial.task_id, trace_id=match.trace_id, score_id=submission.score_id.value, @@ -331,13 +321,13 @@ def _evaluated_receipt( evaluated_tree=state.candidate_tree, task_ids=controls.task_ids, outcome_receipts=reduction.receipts, - blockers=tuple(item.to_evaluated() for item in reduction.blockers), + blockers=reduction.blockers, ) def _authoritative_result( trial: ExperimentTrial, -) -> tuple[VerifierVerdict, float | None] | CandidateBlocker: +) -> tuple[VerifierVerdict, float | None] | EvaluatedRunBlocker: if trial.exception: return _blocker(trial, CandidateBlockerCode.UNVERIFIED, "agent_exception") reward = _reward_result(trial) @@ -348,7 +338,7 @@ def _authoritative_result( def _reward_result( trial: ExperimentTrial, -) -> tuple[VerifierVerdict, float] | CandidateBlocker | None: +) -> tuple[VerifierVerdict, float] | EvaluatedRunBlocker | None: if trial.reward == 1.0: return VerifierVerdict.PASS, 1.0 if trial.reward == 0.0: @@ -360,7 +350,7 @@ def _reward_result( def _verdict_result( trial: ExperimentTrial, -) -> tuple[VerifierVerdict, None] | CandidateBlocker: +) -> tuple[VerifierVerdict, None] | EvaluatedRunBlocker: if trial.verdict in (VerifierVerdict.ABSTAIN.value, VerifierVerdict.ERROR.value): return VerifierVerdict(trial.verdict), None return _blocker(trial, CandidateBlockerCode.UNVERIFIED, "missing_verifier_result") @@ -384,7 +374,7 @@ def _trace_request( def _trace_blocker( trial: ExperimentTrial, code: CandidateBlockerCode | None, -) -> CandidateBlocker: +) -> EvaluatedRunBlocker: if code is None: raise CandidateFailure(CandidateErrorCode.INVALID_RESULT, trial.task_id) return _blocker(trial, code, "trace_mapping") @@ -394,8 +384,8 @@ def _blocker( trial: ExperimentTrial, code: CandidateBlockerCode, subject: str, -) -> CandidateBlocker: - return CandidateBlocker(task_id=trial.task_id, code=code, subject=subject) +) -> EvaluatedRunBlocker: + return EvaluatedRunBlocker(task_id=trial.task_id, code=code.value, subject=subject) def _outcome( @@ -512,15 +502,12 @@ def _running_state( process_id=process_id, started_at=started_at, deadline_at=started_at + timedelta(seconds=timeout_seconds), - outcome_receipts=(), - blockers=(), evaluated_run_receipt=None, ) def _complete_state( state: _CandidateState, - reduction: _OutcomeReduction, evaluated: EvaluatedRunReceipt, ) -> _CandidateState: return _CandidateState( @@ -538,8 +525,6 @@ def _complete_state( process_id=state.process_id, started_at=state.started_at, deadline_at=state.deadline_at, - outcome_receipts=reduction.receipts, - blockers=reduction.blockers, evaluated_run_receipt=evaluated, ) @@ -560,8 +545,6 @@ def _failed_state(state: _CandidateState, code: CandidateErrorCode) -> _Candidat process_id=state.process_id, started_at=state.started_at, deadline_at=state.deadline_at, - outcome_receipts=state.outcome_receipts, - blockers=state.blockers, evaluated_run_receipt=state.evaluated_run_receipt, error_code=code, ) @@ -634,11 +617,14 @@ def _complete_observation( request: CandidateExecutionInput, state: _CandidateState, ) -> CandidateExecutionObservation: - passes = sum(item.verdict is VerifierVerdict.PASS for item in state.outcome_receipts) - failures = sum(item.verdict is VerifierVerdict.FAIL for item in state.outcome_receipts) - terminal = len(state.outcome_receipts) + len(state.blockers) + receipt = state.evaluated_run_receipt + if receipt is None: + raise CandidateFailure(CandidateErrorCode.INVALID_RESULT, request.hypothesis_id) + passes = sum(item.verdict is VerifierVerdict.PASS for item in receipt.outcome_receipts) + failures = sum(item.verdict is VerifierVerdict.FAIL for item in receipt.outcome_receipts) + terminal = len(receipt.outcome_receipts) + len(receipt.blockers) return CandidateExecutionObservation( - status=CandidateStatus.WARNING if state.blockers else CandidateStatus.SUCCESS, + status=CandidateStatus.WARNING if receipt.blockers else CandidateStatus.SUCCESS, summary="The candidate run is complete with authoritative outcome receipts.", next_actions=("Retain the candidate and outcome receipts for the admission gate.",), artifacts=_artifacts(state), @@ -655,10 +641,10 @@ def _complete_observation( terminal_trials=terminal, verifier_passes=passes, verifier_failures=failures, - unverified_trials=len(state.blockers), - outcome_receipts=state.outcome_receipts, - blockers=state.blockers, - evaluated_run_receipt=state.evaluated_run_receipt, + unverified_trials=len(receipt.blockers), + outcome_receipts=receipt.outcome_receipts, + blockers=receipt.blockers, + evaluated_run_receipt=receipt, ) @@ -700,8 +686,8 @@ def _persisted_failure_observation( worktree_path=state.worktree_path, job_path=state.job_path, session_id=state.candidate_id, - outcome_receipts=state.outcome_receipts, - blockers=state.blockers, + outcome_receipts=(), + blockers=(), evaluated_run_receipt=state.evaluated_run_receipt, error_code=code, ) diff --git a/src/ofw/preparation/harbor.py b/src/ofw/preparation/harbor.py index 8404b30..eb17f45 100644 --- a/src/ofw/preparation/harbor.py +++ b/src/ofw/preparation/harbor.py @@ -188,7 +188,7 @@ def summarize(self, run: ExperimentRun) -> ExperimentSummary | None: PreparationErrorCode.INVALID_BASELINE_RESULT, "terminal trial count", ) - if _trial_ids(trials) != run.controls.task_ids: + if tuple(trial.task_id for trial in trials) != run.controls.task_ids: raise PreparationFailure( PreparationErrorCode.INVALID_BASELINE_RESULT, "task ids", @@ -233,10 +233,6 @@ def _parse_elapsed_seconds(value: str) -> int: days, hours, minutes, seconds = (int(part or 0) for part in match.groups()) return days * 86400 + hours * 3600 + minutes * 60 + seconds -def _trial_ids(trials: tuple[ExperimentTrial, ...]) -> tuple[str, ...]: - return tuple(trial.task_id for trial in trials) - - class HarborBaselineRunner: """Compatibility adapter preserving baseline preparation behavior.""" diff --git a/tests/test_candidate_execution.py b/tests/test_candidate_execution.py index 8956f9e..975a34d 100644 --- a/tests/test_candidate_execution.py +++ b/tests/test_candidate_execution.py @@ -664,8 +664,10 @@ def test_candidate_service_creates_launches_polls_and_replays_authoritative_rece assert len(locator.requests) == 2 assert len(outcomes.outcomes) == 2 state = next((root / ".git/ofw/candidates").glob("*/state.json")).read_text() - assert "input" not in state - assert "output" not in state + assert "\n \"input\"" not in state + assert "\n \"output\"" not in state + assert "\n \"outcome_receipts\"" not in state + assert "\n \"blockers\"" not in state assert "test-openai-key" not in state @@ -970,8 +972,8 @@ def test_candidate_service_keeps_unsupported_and_ambiguous_trials_unverified( "task-2", ) assert tuple(blocker.code for blocker in complete.blockers) == ( - CandidateBlockerCode.UNSUPPORTED_REWARD, - CandidateBlockerCode.TRACE_AMBIGUOUS, + CandidateBlockerCode.UNSUPPORTED_REWARD.value, + CandidateBlockerCode.TRACE_AMBIGUOUS.value, ) assert complete.unverified_trials == 2 assert len(locator.requests) == 1 diff --git a/tests/test_typing.py b/tests/test_typing.py index 17f730f..ef8c7f1 100644 --- a/tests/test_typing.py +++ b/tests/test_typing.py @@ -116,3 +116,4 @@ def test_namespace_exports_candidate_contracts_without_restoring_runtime_plane() assert expected <= set(package.__all__) assert {"E2BSandbox", "CanaryCase", "CommandLoop"}.isdisjoint(package.__all__) + assert {"CandidateBlocker", "CandidateOutcomeReceipt"}.isdisjoint(package.__all__) From 1e56f2992bfa5500b80a8fd488fc5baf38c82ee3 Mon Sep 17 00:00:00 2001 From: divo12 Date: Thu, 3 Sep 2026 17:55:47 +0530 Subject: [PATCH 20/25] fix: rebase candidate lifecycle onto latest policy --- src/ofw/evolution/hypothesis.py | 8 ++++++ src/ofw/preparation/harbor.py | 41 +++++++++++++++++++------------ tests/test_candidate_execution.py | 4 +++ 3 files changed, 37 insertions(+), 16 deletions(-) diff --git a/src/ofw/evolution/hypothesis.py b/src/ofw/evolution/hypothesis.py index 78682f6..3b60975 100644 --- a/src/ofw/evolution/hypothesis.py +++ b/src/ofw/evolution/hypothesis.py @@ -228,6 +228,10 @@ def to_hypothesis(self) -> HarnessHypothesis: id=HypothesisId(self.hypothesis_id), experiment_id=self.experiment_id, source_commit=self.source_commit, + curation_id=self.curation_id, + curation_group_id=self.curation_group_id, + predicted_task_ids=self.predicted_task_ids, + at_risk_task_ids=self.at_risk_task_ids, patterns=tuple( FailurePatternReference( pattern.pattern_id, @@ -250,6 +254,10 @@ def recomputed_id(self) -> HypothesisId: _HypothesisContent( experiment_id=self.experiment_id, source_commit=self.source_commit, + curation_id=self.curation_id, + curation_group_id=self.curation_group_id, + predicted_task_ids=self.predicted_task_ids, + at_risk_task_ids=self.at_risk_task_ids, patterns=self.patterns, statement=self.statement, rationale=self.rationale, diff --git a/src/ofw/preparation/harbor.py b/src/ofw/preparation/harbor.py index eb17f45..9fa3343 100644 --- a/src/ofw/preparation/harbor.py +++ b/src/ofw/preparation/harbor.py @@ -8,7 +8,7 @@ import signal import subprocess # nosec B404 from dataclasses import dataclass -from datetime import datetime +from datetime import UTC, datetime from pathlib import Path from pydantic import BaseModel, ConfigDict, Field, JsonValue, ValidationError @@ -178,21 +178,7 @@ def summarize(self, run: ExperimentRun) -> ExperimentSummary | None: if root is None: return None trials = _experiment_trials(run) - if len(trials) > root.n_total_trials: - raise PreparationFailure( - PreparationErrorCode.INVALID_BASELINE_RESULT, - "trial count", - ) - if len(trials) != root.n_total_trials: - raise PreparationFailure( - PreparationErrorCode.INVALID_BASELINE_RESULT, - "terminal trial count", - ) - if tuple(trial.task_id for trial in trials) != run.controls.task_ids: - raise PreparationFailure( - PreparationErrorCode.INVALID_BASELINE_RESULT, - "task ids", - ) + _validate_experiment_trials(root.n_total_trials, trials, run.controls.task_ids) return ExperimentSummary(trials) def cancel(self, run: ExperimentRun, process_id: int | None) -> None: @@ -233,6 +219,29 @@ def _parse_elapsed_seconds(value: str) -> int: days, hours, minutes, seconds = (int(part or 0) for part in match.groups()) return days * 86400 + hours * 3600 + minutes * 60 + seconds + +def _validate_experiment_trials( + expected_count: int, + trials: tuple[ExperimentTrial, ...], + expected_task_ids: tuple[str, ...], +) -> None: + if len(trials) > expected_count: + raise PreparationFailure( + PreparationErrorCode.INVALID_BASELINE_RESULT, + "trial count", + ) + if len(trials) != expected_count: + raise PreparationFailure( + PreparationErrorCode.INVALID_BASELINE_RESULT, + "terminal trial count", + ) + if tuple(trial.task_id for trial in trials) != expected_task_ids: + raise PreparationFailure( + PreparationErrorCode.INVALID_BASELINE_RESULT, + "task ids", + ) + + class HarborBaselineRunner: """Compatibility adapter preserving baseline preparation behavior.""" diff --git a/tests/test_candidate_execution.py b/tests/test_candidate_execution.py index 975a34d..5a2fbca 100644 --- a/tests/test_candidate_execution.py +++ b/tests/test_candidate_execution.py @@ -134,6 +134,10 @@ def _authority(tmp_path: Path) -> tuple[Path, ExperimentPolicySnapshot, HarnessH id=HypothesisId("sha256:" + "b" * 64), experiment_id=policy.experiment_id, source_commit=commit, + curation_id="00000000-0000-0000-0000-000000000001", + curation_group_id="00000000-0000-0000-0000-000000000002", + predicted_task_ids=("task-1",), + at_risk_task_ids=("task-2",), patterns=( FailurePatternReference( "sha256:" + "c" * 64, From c3b3d6d739fac12c4727dea378164204b49d54f0 Mon Sep 17 00:00:00 2001 From: divo12 Date: Thu, 3 Sep 2026 18:10:08 +0530 Subject: [PATCH 21/25] refactor: remove dead Harbor parsing code --- src/ofw/preparation/harbor.py | 32 +------------------------------- 1 file changed, 1 insertion(+), 31 deletions(-) diff --git a/src/ofw/preparation/harbor.py b/src/ofw/preparation/harbor.py index 9fa3343..8bb7ded 100644 --- a/src/ofw/preparation/harbor.py +++ b/src/ofw/preparation/harbor.py @@ -4,11 +4,10 @@ import hashlib import os -import re import signal import subprocess # nosec B404 from dataclasses import dataclass -from datetime import UTC, datetime +from datetime import datetime from pathlib import Path from pydantic import BaseModel, ConfigDict, Field, JsonValue, ValidationError @@ -31,7 +30,6 @@ _MAX_ADAPTER_BYTES = 512 * 1024 _AGENT_NAME = "agents.ofw_hermes:OfwHermes" _SOURCE_ENVIRONMENT_NAME = "OFW_HERMES_SOURCE" -_ELAPSED_PATTERN = re.compile(r"(?:(\d+)-)?(?:(\d+):)?(\d+):(\d+)") class _WireModel(BaseModel): @@ -77,7 +75,6 @@ class _HarborVerifierWire(_WireModel): class _HarborTrialResultWire(_WireModel): task_id: str | None = None - task_name: str | None = None task_checksum: str | None = None exception_info: JsonValue | None = None agent_execution: _HarborExecutionWire | None = None @@ -193,33 +190,6 @@ def cancel(self, run: ExperimentRun, process_id: int | None) -> None: raise PreparationFailure(PreparationErrorCode.LAUNCH_FAILED, "cancel") from error -def _process_identity_matches(process_id: int, started_at: datetime | None) -> bool: - if started_at is None: - return False - try: - result = subprocess.run( - ("ps", "-p", str(process_id), "-o", "etime="), - check=False, - capture_output=True, - text=True, - ) - elapsed_seconds = _parse_elapsed_seconds(result.stdout.strip()) - except (OSError, ValueError): - return False - if result.returncode != 0: - return False - age_seconds = (datetime.now(UTC) - started_at).total_seconds() - return abs(age_seconds - elapsed_seconds) <= 2 - - -def _parse_elapsed_seconds(value: str) -> int: - match = _ELAPSED_PATTERN.fullmatch(value) - if match is None: - raise ValueError("invalid process age") - days, hours, minutes, seconds = (int(part or 0) for part in match.groups()) - return days * 86400 + hours * 3600 + minutes * 60 + seconds - - def _validate_experiment_trials( expected_count: int, trials: tuple[ExperimentTrial, ...], From 4aa2f2e3e2bf2bf08286c092a42eedd4250dd43f Mon Sep 17 00:00:00 2001 From: divo12 Date: Thu, 3 Sep 2026 18:17:59 +0530 Subject: [PATCH 22/25] fix: describe candidate execution in plugin --- plugins/openflywheel/.codex-plugin/plugin.json | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/plugins/openflywheel/.codex-plugin/plugin.json b/plugins/openflywheel/.codex-plugin/plugin.json index 6393349..dea1915 100644 --- a/plugins/openflywheel/.codex-plugin/plugin.json +++ b/plugins/openflywheel/.codex-plugin/plugin.json @@ -1,7 +1,7 @@ { "name": "openflywheel", "version": "0.8.0", - "description": "Prepare ITSM-bench workspaces and record evidence-backed harness hypotheses.", + "description": "Prepare ITSM-bench workspaces, record hypotheses, and execute isolated candidates.", "author": { "name": "OpenFlyWheel" }, @@ -10,14 +10,14 @@ "skills": "./skills/", "interface": { "displayName": "OpenFlyWheel", - "shortDescription": "Record evidence-backed harness hypotheses", - "longDescription": "Initialize an ITSM-bench agent-harness optimization workspace, inspect bounded Langfuse evidence, record authoritative outcomes and compact diagnoses, mine exact recurring patterns, and persist one evidence-backed hypothesis before candidate editing without copying trace payloads.", + "shortDescription": "Execute isolated harness candidates", + "longDescription": "Initialize an ITSM-bench agent-harness optimization workspace, inspect bounded Langfuse evidence, record authoritative outcomes and compact diagnoses, mine exact recurring patterns, persist one evidence-backed hypothesis, execute one isolated candidate, and stop before admission without copying trace payloads.", "developerName": "OpenFlyWheel", "category": "Productivity", "capabilities": ["Read", "Write"], "defaultPrompt": [ "Read this repository, identify the primary agent harness the user wants to evaluate with ITSM-bench, and use $workspace-init to collect its experiment configuration and perform the initial setup.", - "When PROGRAM.md is ready, read it, record one evidence-backed hypothesis, and stop before candidate editing." + "When PROGRAM.md is ready, read it, record one evidence-backed hypothesis, execute one isolated candidate, and stop before admission." ] }, "mcpServers": "./.mcp.json" From c5c08c0c793ed635ba885d87981c9757423173cd Mon Sep 17 00:00:00 2001 From: divo12 Date: Thu, 3 Sep 2026 18:18:15 +0530 Subject: [PATCH 23/25] chore: pin plugin to candidate runtime --- plugins/openflywheel/.mcp.json | 2 +- tests/test_plugin_packaging.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/plugins/openflywheel/.mcp.json b/plugins/openflywheel/.mcp.json index 1078b2a..9f9c703 100644 --- a/plugins/openflywheel/.mcp.json +++ b/plugins/openflywheel/.mcp.json @@ -4,7 +4,7 @@ "command": "uvx", "args": [ "--from", - "git+https://github.com/divo12/OpenFlyWheel.git@fe5d0972ef4733553454814e96910ae51ab6a5a3", + "git+https://github.com/divo12/OpenFlyWheel.git@4aa2f2e3e2bf2bf08286c092a42eedd4250dd43f", "--with", "mcp>=1.13,<2", "openflywheel-mcp" diff --git a/tests/test_plugin_packaging.py b/tests/test_plugin_packaging.py index 0826b94..6fc9036 100644 --- a/tests/test_plugin_packaging.py +++ b/tests/test_plugin_packaging.py @@ -28,7 +28,7 @@ def test_openflywheel_mcp_uses_pinned_portable_runtime() -> None: manifest = _McpManifest.model_validate_json(path.read_text(encoding="utf-8")) server = manifest.mcpServers["openflywheel"] assert ( - "git+https://github.com/divo12/OpenFlyWheel.git@fe5d0972ef4733553454814e96910ae51ab6a5a3" + "git+https://github.com/divo12/OpenFlyWheel.git@4aa2f2e3e2bf2bf08286c092a42eedd4250dd43f" in server.args ) runtime = (root / "src/ofw/mcp.py").read_text(encoding="utf-8") From 9041db3c08a89df0fe9f8f2476a303b46dd2812a Mon Sep 17 00:00:00 2001 From: divo12 Date: Thu, 3 Sep 2026 19:07:26 +0530 Subject: [PATCH 24/25] fix: accept Harbor task id objects --- src/ofw/preparation/harbor.py | 13 ++++++++++--- tests/test_harbor_experiment.py | 2 +- 2 files changed, 11 insertions(+), 4 deletions(-) diff --git a/src/ofw/preparation/harbor.py b/src/ofw/preparation/harbor.py index 8bb7ded..383cca3 100644 --- a/src/ofw/preparation/harbor.py +++ b/src/ofw/preparation/harbor.py @@ -45,6 +45,10 @@ class _HarborTaskWire(_WireModel): path: str = Field(min_length=1, max_length=256) +class _HarborTaskIdWire(_WireModel): + path: str = Field(min_length=1, max_length=256) + + class _HarborConfigWire(_WireModel): agents: tuple[_HarborAgentWire, ...] = Field(min_length=1, max_length=1) tasks: tuple[_HarborTaskWire, ...] = Field(min_length=1, max_length=500) @@ -74,7 +78,7 @@ class _HarborVerifierWire(_WireModel): class _HarborTrialResultWire(_WireModel): - task_id: str | None = None + task_id: str | _HarborTaskIdWire | None = None task_checksum: str | None = None exception_info: JsonValue | None = None agent_execution: _HarborExecutionWire | None = None @@ -443,15 +447,18 @@ def _required_trial_fields( wire: _HarborTrialResultWire, directory_name: str, ) -> tuple[str, str, _HarborExecutionWire, _HarborVerifierWire]: - if wire.task_id is None: + task_id = wire.task_id + if task_id is None: raise _invalid_trial(directory_name) + if isinstance(task_id, _HarborTaskIdWire): + task_id = task_id.path if wire.task_checksum is None: raise _invalid_trial(directory_name) if wire.agent_execution is None: raise _invalid_trial(directory_name) if wire.verifier is None: raise _invalid_trial(directory_name) - return wire.task_id, wire.task_checksum, wire.agent_execution, wire.verifier + return task_id, wire.task_checksum, wire.agent_execution, wire.verifier def _invalid_trial(directory_name: str) -> PreparationFailure: diff --git a/tests/test_harbor_experiment.py b/tests/test_harbor_experiment.py index 6cf9f9c..6095abf 100644 --- a/tests/test_harbor_experiment.py +++ b/tests/test_harbor_experiment.py @@ -79,7 +79,7 @@ def _benchmark(tmp_path: Path) -> tuple[Path, Path, Path]: (trial / "verifier").mkdir(parents=True) (trial / "result.json").write_text(json.dumps({ "task_name": f"display-{index}", - "task_id": f"task-{index}", + "task_id": {"path": f"task-{index}"}, "task_checksum": f"checksum-{index}", "exception_info": None, "agent_execution": { From b9df0485d8fcf62c3974a6dcf3f2fb7e7a154fbc Mon Sep 17 00:00:00 2001 From: divo12 Date: Thu, 3 Sep 2026 19:07:42 +0530 Subject: [PATCH 25/25] chore: pin plugin to Harbor compatibility fix --- plugins/openflywheel/.mcp.json | 2 +- tests/test_plugin_packaging.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/plugins/openflywheel/.mcp.json b/plugins/openflywheel/.mcp.json index 9f9c703..b2cfe01 100644 --- a/plugins/openflywheel/.mcp.json +++ b/plugins/openflywheel/.mcp.json @@ -4,7 +4,7 @@ "command": "uvx", "args": [ "--from", - "git+https://github.com/divo12/OpenFlyWheel.git@4aa2f2e3e2bf2bf08286c092a42eedd4250dd43f", + "git+https://github.com/divo12/OpenFlyWheel.git@9041db3c08a89df0fe9f8f2476a303b46dd2812a", "--with", "mcp>=1.13,<2", "openflywheel-mcp" diff --git a/tests/test_plugin_packaging.py b/tests/test_plugin_packaging.py index 6fc9036..b2bb8bb 100644 --- a/tests/test_plugin_packaging.py +++ b/tests/test_plugin_packaging.py @@ -28,7 +28,7 @@ def test_openflywheel_mcp_uses_pinned_portable_runtime() -> None: manifest = _McpManifest.model_validate_json(path.read_text(encoding="utf-8")) server = manifest.mcpServers["openflywheel"] assert ( - "git+https://github.com/divo12/OpenFlyWheel.git@4aa2f2e3e2bf2bf08286c092a42eedd4250dd43f" + "git+https://github.com/divo12/OpenFlyWheel.git@9041db3c08a89df0fe9f8f2476a303b46dd2812a" in server.args ) runtime = (root / "src/ofw/mcp.py").read_text(encoding="utf-8")